add functions to check for directory and filename

1. add validation for directory and filename
2. refactor the file operations about path
3. add in CMakeLists
4. add logger in nn_ops

mindspore/ccsrc/backend/optimizer/somas/somas.cc

@@ -381,9 +381,9 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) {
 #ifdef ENABLE_DUMP_IR
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
-  std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
-  filename = "somas_offline_log_" + std::to_string(graph->graph_id()) + ".ir";
+  filename = "somas_offline_log_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, Offline(), filename);
 #endif
 
@@ -673,7 +673,7 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) {
 #ifdef ENABLE_DUMP_IR
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
-  std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
 #endif
 

mindspore/ccsrc/debug/CMakeLists.txt

@@ -10,6 +10,7 @@ set(_DEBUG_SRC_LIST
 
 if(ENABLE_DUMP_IR)
     list(APPEND _DEBUG_SRC_LIST
+        "${CMAKE_CURRENT_SOURCE_DIR}/rdr/base_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc"

mindspore/ccsrc/debug/common.cc

@@ -170,4 +170,105 @@ std::optional<std::string> Common::GetEnvConfigFile() {
   }
   return config_file;
 }
+
+bool Common::IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message,
+                              const bool &print_str) {
+  const int len_str = str.length();
+  if (len_str > length_limit) {
+    std::ostringstream msg;
+    if (print_str) {
+      msg << error_message << "The string is " << str << ", its length is " << str.length();
+    } else {
+      msg << error_message << "The length is " << str.length();
+    }
+    msg << ", exceeding the limit of " << length_limit << ".";
+    MS_LOG(WARNING) << msg.str();
+    return false;
+  }
+  return true;
+}
+
+bool Common::IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message) {
+  int left_pos = 0;
+  int len_path = path.length();
+  for (int i = 0; i < len_path; i++) {
+    if (i != 0) {
+      if (path[i] == '\\' || path[i] == '/') {
+        int cur_len = i - left_pos;
+        if (cur_len > length_limit) {
+          MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is "
+                          << cur_len << ". It is out of the limit which is " << length_limit << ".";
+          return false;
+        }
+        left_pos = i + 1;
+      }
+    }
+  }
+  if (!(path[len_path - 1] == '\\' || path[len_path - 1] == '/')) {
+    int cur_len = len_path - left_pos;
+    if (cur_len > length_limit) {
+      MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is " << cur_len
+                      << ". It is out of the limit which is " << length_limit << ".";
+      return false;
+    }
+  }
+  return true;
+}
+
+bool Common::IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message,
+                         const bool &print_str) {
+  std::string err_msg = "Detail: ";
+  if (!error_message.empty()) {
+    err_msg = error_message + " " + err_msg;
+  }
+
+  if (path.empty()) {
+    MS_LOG(WARNING) << err_msg << "The path is empty.";
+    return false;
+  }
+
+  if (!IsStrLengthValid(path, length_limit, err_msg, print_str)) {
+    return false;
+  }
+
+  if (!std::all_of(path.begin(), path.end(),
+                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) {
+    MS_LOG(WARNING) << err_msg << "The path only support alphabets, digit or {'-', '_', '/'}, but got:" << path << ".";
+    return false;
+  }
+
+  if (path[0] != '/') {
+    MS_LOG(WARNING) << err_msg << "The path only support absolute path and should start with '/'.";
+    return false;
+  }
+
+  if (!IsEveryFilenameValid(path, maxOSFilenameLength, err_msg)) {
+    return false;
+  }
+  return true;
+}
+
+bool Common::IsFilenameValid(const std::string &filename, const int &length_limit, const std::string &error_message) {
+  std::string err_msg = "Detail: ";
+  if (!error_message.empty()) {
+    err_msg = error_message + " " + err_msg;
+  }
+
+  if (filename.empty()) {
+    MS_LOG(WARNING) << err_msg << "The filename is empty.";
+    return false;
+  }
+
+  if (!IsStrLengthValid(filename, length_limit, err_msg)) {
+    return false;
+  }
+
+  if (!std::all_of(filename.begin(), filename.end(),
+                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '.'; })) {
+    MS_LOG(WARNING) << err_msg << "The filename only support alphabets, digit or {'-', '_', '.'}, but got:" << filename
+                    << ".";
+    return false;
+  }
+  return true;
+}
 }  // namespace mindspore

mindspore/ccsrc/debug/common.h

@@ -22,6 +22,9 @@
 #include "utils/contract.h"
 
 namespace mindspore {
+static const int maxDirectoryLength = 1024;
+static const int maxFilenameLength = 128;
+static const int maxOSFilenameLength = 255;
 class Common {
  public:
   Common() = default;
@@ -29,9 +32,16 @@ class Common {
   static std::optional<std::string> GetRealPath(const std::string &input_path);
   static std::optional<std::string> GetConfigFile(const std::string &env);
   static std::optional<std::string> GetEnvConfigFile();
+  static bool IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message = "",
+                               const bool &print_str = true);
+  static bool IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message = "",
+                          const bool &print_str = true);
+  static bool IsFilenameValid(const std::string &filename, const int &length_limit,
+                              const std::string &error_message = "");
 
  private:
   static bool CreateNotExistDirs(const std::string &path);
+  static bool IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message);
 };
 }  // namespace mindspore
 #endif  // MINDSPORE_CCSRC_DEBUG_COMMON_H_

mindspore/ccsrc/debug/env_config_parser.cc

@@ -116,28 +116,20 @@ void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) {
 }
 
 void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
+  std::string err_msg = "RDR path parse failed. The RDR path will be a default value: '" + rdr_path_ +
+                        "'. Please check the settings about '" + kRdrSettings + "' in config file '" + config_file_ +
+                        "' set by 'env_config_path' in context.";
+
   if (!CheckJsonStringType(content, kRdrSettings, kPath)) {
-    MS_LOG(WARNING) << "The RDR path will be a default value: '" << rdr_path_ << "'.";
+    MS_LOG(WARNING) << err_msg;
     return;
   }
+
   std::string path = content;
-  if (!std::all_of(path.begin(), path.end(),
-                   [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings
-                    << " only support alphabets, digit or {'-', '_', '/'}, but got:" << path << "."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
-    return;
-  }
-  if (path.empty()) {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings << " is empty."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
-    return;
-  }
-  if (path[0] != '/') {
-    MS_LOG(WARNING) << "The path in " << kRdrSettings << " only support absolute path and should start with '/'."
-                    << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
+  if (!Common::IsPathValid(path, maxDirectoryLength, err_msg, false)) {
     return;
   }
+
   if (path.back() != '/') {
     path += '/';
   }
@@ -146,7 +138,7 @@ void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) {
 
 void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) {
   if (!content.is_boolean()) {
-    MS_LOG(WARNING) << "Json Parse Failed. 'enable' in " << kRdrSettings << " should be boolean."
+    MS_LOG(WARNING) << "Json parse failed. 'enable' in " << kRdrSettings << " should be boolean."
                     << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context.";
     rdr_enabled_ = false;
     return;

mindspore/ccsrc/debug/rdr/base_recorder.cc

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/base_recorder.h"
+#include <sys/stat.h>
+#include <fstream>
+#include "debug/common.h"
+#include "utils/utils.h"
+
+namespace mindspore {
+void BaseRecorder::SetDirectory(const std::string &directory) {
+  std::string error_message = module_ + ":" + tag_ + " set directory failed.";
+  if (Common::IsPathValid(directory, maxDirectoryLength, error_message)) {
+    directory_ = directory;
+    if (directory_.back() != '/') {
+      directory_ += "/";
+    }
+  }
+}
+
+void BaseRecorder::SetFilename(const std::string &filename) {
+  std::string error_message = module_ + ":" + tag_ + " set filename failed.";
+  if (Common::IsFilenameValid(filename, maxDirectoryLength, error_message)) {
+    filename_ = filename;
+  }
+}
+
+std::optional<std::string> BaseRecorder::GetFileRealPath() {
+  if (filename_.empty()) {
+    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+  }
+  std::string file_path = directory_ + filename_;
+  auto realpath = Common::GetRealPath(file_path);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path failed. "
+                  << "Info: module=" << module_ << ", tag=" << tag_ << ", "
+                  << "path=" << file_path << ".";
+  }
+
+  return realpath;
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/base_recorder.h

@@ -21,7 +21,11 @@
 #include <sstream>
 #include <chrono>
 #include <iomanip>
+#include "debug/common.h"
 #include "debug/env_config_parser.h"
+#include "mindspore/core/utils/log_adapter.h"
+
+const int maxTagLength = 32;
 namespace mindspore {
 class BaseRecorder {
  public:
@@ -31,6 +35,17 @@ class BaseRecorder {
     config_parser_ptr.Parse();
     directory_ = config_parser_ptr.rdr_path();
 
+    if (tag.length() > maxTagLength) {
+      tag_ = tag.substr(0, maxTagLength);
+      MS_LOG(WARNING) << "The tag length is " << tag.length() << ", exceeding the limit " << maxTagLength
+                      << ". It will be intercepted as '" << tag_ << "'.";
+    }
+
+    std::string err_msg = module_ + ":" + tag_ + " set filename failed.";
+    if (!filename_.empty() && !Common::IsFilenameValid(filename_, maxFilenameLength, err_msg)) {
+      filename_ = "";
+    }
+
     auto sys_time = std::chrono::system_clock::now();
     auto t_time = std::chrono::system_clock::to_time_t(sys_time);
     std::tm *l_time = std::localtime(&t_time);
@@ -47,9 +62,10 @@ class BaseRecorder {
   std::string GetModule() const { return module_; }
   std::string GetTag() const { return tag_; }
   std::string GetTimeStamp() const { return timestamp_; }
+  std::optional<std::string> GetFileRealPath();
 
-  void SetDirectory(const std::string &directory) { directory_ = directory; }
-  std::string GetDirectory() const { return directory_; }
+  void SetDirectory(const std::string &directory);
+  void SetFilename(const std::string &filename);
   virtual void Export() {}
 
  protected:

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc

@@ -46,10 +46,11 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr>
 }  // namespace
 
 void GraphExecOrderRecorder::Export() {
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+  auto realpath = GetFileRealPath();
+  if (!realpath.has_value()) {
+    return;
   }
-  std::string filename = directory_ + filename_ + ".txt";
-  DumpGraphExeOrder(filename, exec_order_);
+  std::string real_file_path = realpath.value() + ".txt";
+  DumpGraphExeOrder(real_file_path, exec_order_);
 }
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_recorder.cc

@@ -25,33 +25,22 @@
 namespace mindspore {
 namespace protobuf {
 #ifdef ENABLE_DUMP_IR
-void DumpIRProto(const std::string &filename, const FuncGraphPtr &func_graph) {
+void DumpIRProto(const std::string &real_path, const FuncGraphPtr &func_graph) {
   if (func_graph == nullptr) {
-    MS_LOG(ERROR) << "Func graph is nullptr";
+    MS_LOG(ERROR) << "Func graph is nullptr.";
     return;
   }
 
-  if (filename.size() > PATH_MAX) {
-    MS_LOG(ERROR) << "File path " << filename << " is too long.";
-    return;
-  }
-
-  auto real_path = Common::GetRealPath(filename);
-  if (!real_path.has_value()) {
-    MS_LOG(ERROR) << "Get real path failed. path=" << filename;
-    return;
-  }
-  ChangeFileMode(real_path.value(), S_IRWXU);
   // write to pb file
-  std::ofstream ofs(real_path.value());
+  std::ofstream ofs(real_path);
   if (!ofs.is_open()) {
-    MS_LOG(ERROR) << "Open file '" << real_path.value() << "' failed!";
+    MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
     return;
   }
   ofs << GetFuncGraphProtoString(func_graph);
   ofs.close();
   // set file mode to read only by user
-  ChangeFileMode(real_path.value(), S_IRUSR);
+  ChangeFileMode(real_path, S_IRUSR);
 }
 #else
 void DumpIRProto(const std::string &, const FuncGraphPtr &) {
@@ -68,31 +57,35 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) {
 
 void GraphRecorder::Export() {
   bool save_flag = false;
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_;
+
+  auto tmp_realpath = GetFileRealPath();
+  if (!tmp_realpath.has_value()) {
+    return;
   }
-  std::string file_path = directory_ + filename_ + std::to_string(id_);
+
+  std::string realpath = tmp_realpath.value() + std::to_string(id_);
+
   if (graph_type_.find(".dat") != std::string::npos) {
     save_flag = true;
-    AnfExporter exporter("");
-    std::string real_path = file_path + ".dat";
-    ChangeFileMode(real_path, S_IRWXU);
-    exporter.ExportFuncGraph(real_path, func_graph_);
-    ChangeFileMode(real_path, S_IRUSR);
+    AnfExporter exporter(std::to_string(id_));
+    std::string realpath_dat = realpath + ".dat";
+    ChangeFileMode(realpath_dat, S_IRWXU);
+    exporter.ExportFuncGraph(realpath_dat, func_graph_);
+    ChangeFileMode(realpath_dat, S_IRUSR);
   }
   if (graph_type_.find(".ir") != std::string::npos) {
     save_flag = true;
-    std::string real_path = file_path + ".ir";
+    std::string realpath_ir = realpath + ".ir";
     if (full_name_) {
-      DumpIRForRDR(real_path, func_graph_, true, kTopStack);
+      DumpIRForRDR(realpath_ir, func_graph_, true, kTopStack);
     } else {
-      DumpIRForRDR(real_path, func_graph_, false, kOff);
+      DumpIRForRDR(realpath_ir, func_graph_, false, kOff);
     }
   }
   if (graph_type_.find(".pb") != std::string::npos) {
     save_flag = true;
-    std::string real_path = file_path + ".pb";
-    protobuf::DumpIRProto(real_path, func_graph_);  // save *.pb file
+
+    protobuf::DumpIRProto(realpath + ".pb", func_graph_);  // save *.pb file
   }
   if (!save_flag) {
     MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_;

mindspore/ccsrc/debug/rdr/string_recorder.cc

@@ -22,23 +22,13 @@
 
 namespace mindspore {
 void StringRecorder::Export() {
-  if (directory_.back() != '/') {
-    directory_ += "/";
-  }
-
-  if (filename_.empty()) {
-    filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt";
-  }
-  std::string file_path = directory_ + filename_;
-
-  auto realpath = Common::GetRealPath(file_path);
+  auto realpath = GetFileRealPath();
   if (!realpath.has_value()) {
-    MS_LOG(ERROR) << "Get real path failed. path=" << file_path;
     return;
   }
-
-  ChangeFileMode(realpath.value(), S_IRWXU);
-  std::ofstream fout(realpath.value(), std::ofstream::app);
+  std::string file_path = realpath.value() + ".txt";
+  ChangeFileMode(file_path, S_IRWXU);
+  std::ofstream fout(file_path, std::ofstream::app);
   if (!fout.is_open()) {
     MS_LOG(WARNING) << "Open file for saving string failed.";
     return;
@@ -46,6 +36,6 @@ void StringRecorder::Export() {
   fout << data_;
   fout.close();
   // set file mode to read only by user
-  ChangeFileMode(realpath.value(), S_IRUSR);
+  ChangeFileMode(file_path, S_IRUSR);
 }
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/string_recorder.h

@@ -25,15 +25,15 @@ class StringRecorder : public BaseRecorder {
  public:
   StringRecorder() : BaseRecorder() {}
   StringRecorder(const std::string &module, const std::string &tag, const std::string &data,
-                 const std::string &file_type)
-      : BaseRecorder(module, tag), data_(data) {}
+                 const std::string &filename)
+      : BaseRecorder(module, tag), data_(data), filename_(filename) {}
   ~StringRecorder() {}
   void SetModule(const std::string &module) { module_ = module; }
-  void SetFilename(const std::string &filename) { filename_ = filename; }
   virtual void Export();
 
  private:
   std::string data_;
+  std::string filename_;
 };
 using StringRecorderPtr = std::shared_ptr<StringRecorder>;
 }  // namespace mindspore

mindspore/ccsrc/runtime/device/memory_manager.cc

@@ -76,10 +76,10 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) {
   SubModuleId module = SubModuleId::SM_OPTIMIZER;
   std::string tag = "somas";
 
-  std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir";
+  std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename);
 
-  filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir";
+  filename = "somas_mem_info_" + std::to_string(graph->graph_id());
   mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename);
 #endif
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);

mindspore/ops/operations/nn_ops.py

@@ -18,6 +18,7 @@
 import math
 import operator
 from functools import reduce, partial
+from mindspore import log as logger
 from mindspore._checkparam import _check_3d_int_or_tuple
 import numpy as np
 from ... import context