From b4d56f635857c59a49915a8fc13c6915d85e5e03 Mon Sep 17 00:00:00 2001 From: luopengting Date: Tue, 2 Feb 2021 22:06:25 +0800 Subject: [PATCH] add functions to check for directory and filename 1. add validation for directory and filename 2. refactor the file operations about path 3. add in CMakeLists 4. add logger in nn_ops --- .../ccsrc/backend/optimizer/somas/somas.cc | 6 +- mindspore/ccsrc/debug/CMakeLists.txt | 1 + mindspore/ccsrc/debug/common.cc | 101 ++++++++++++++++++ mindspore/ccsrc/debug/common.h | 10 ++ mindspore/ccsrc/debug/env_config_parser.cc | 26 ++--- mindspore/ccsrc/debug/rdr/base_recorder.cc | 54 ++++++++++ mindspore/ccsrc/debug/rdr/base_recorder.h | 20 +++- .../debug/rdr/graph_exec_order_recorder.cc | 9 +- mindspore/ccsrc/debug/rdr/graph_recorder.cc | 51 ++++----- mindspore/ccsrc/debug/rdr/string_recorder.cc | 20 +--- mindspore/ccsrc/debug/rdr/string_recorder.h | 6 +- .../ccsrc/runtime/device/memory_manager.cc | 4 +- mindspore/ops/operations/nn_ops.py | 1 + 13 files changed, 234 insertions(+), 75 deletions(-) create mode 100644 mindspore/ccsrc/debug/rdr/base_recorder.cc diff --git a/mindspore/ccsrc/backend/optimizer/somas/somas.cc b/mindspore/ccsrc/backend/optimizer/somas/somas.cc index 7269099a09..bbec091a92 100644 --- a/mindspore/ccsrc/backend/optimizer/somas/somas.cc +++ b/mindspore/ccsrc/backend/optimizer/somas/somas.cc @@ -381,9 +381,9 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) { #ifdef ENABLE_DUMP_IR SubModuleId module = SubModuleId::SM_OPTIMIZER; std::string tag = "somas"; - std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; + std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()); mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); - filename = "somas_offline_log_" + std::to_string(graph->graph_id()) + ".ir"; + filename = "somas_offline_log_" + std::to_string(graph->graph_id()); mindspore::RDR::RecordString(module, tag, Offline(), filename); #endif @@ -673,7 +673,7 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) { #ifdef ENABLE_DUMP_IR SubModuleId module = SubModuleId::SM_OPTIMIZER; std::string tag = "somas"; - std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()) + ".ir"; + std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()); mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); #endif diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index aed1675895..a21688bb3d 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -10,6 +10,7 @@ set(_DEBUG_SRC_LIST if(ENABLE_DUMP_IR) list(APPEND _DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/base_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc" diff --git a/mindspore/ccsrc/debug/common.cc b/mindspore/ccsrc/debug/common.cc index dbdb33aeba..2b1a9687ea 100644 --- a/mindspore/ccsrc/debug/common.cc +++ b/mindspore/ccsrc/debug/common.cc @@ -170,4 +170,105 @@ std::optional Common::GetEnvConfigFile() { } return config_file; } + +bool Common::IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message, + const bool &print_str) { + const int len_str = str.length(); + if (len_str > length_limit) { + std::ostringstream msg; + if (print_str) { + msg << error_message << "The string is " << str << ", its length is " << str.length(); + } else { + msg << error_message << "The length is " << str.length(); + } + msg << ", exceeding the limit of " << length_limit << "."; + MS_LOG(WARNING) << msg.str(); + return false; + } + return true; +} + +bool Common::IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message) { + int left_pos = 0; + int len_path = path.length(); + for (int i = 0; i < len_path; i++) { + if (i != 0) { + if (path[i] == '\\' || path[i] == '/') { + int cur_len = i - left_pos; + if (cur_len > length_limit) { + MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is " + << cur_len << ". It is out of the limit which is " << length_limit << "."; + return false; + } + left_pos = i + 1; + } + } + } + if (!(path[len_path - 1] == '\\' || path[len_path - 1] == '/')) { + int cur_len = len_path - left_pos; + if (cur_len > length_limit) { + MS_LOG(WARNING) << error_message << "The name length of '" << path.substr(left_pos, cur_len) << "' is " << cur_len + << ". It is out of the limit which is " << length_limit << "."; + return false; + } + } + return true; +} + +bool Common::IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message, + const bool &print_str) { + std::string err_msg = "Detail: "; + if (!error_message.empty()) { + err_msg = error_message + " " + err_msg; + } + + if (path.empty()) { + MS_LOG(WARNING) << err_msg << "The path is empty."; + return false; + } + + if (!IsStrLengthValid(path, length_limit, err_msg, print_str)) { + return false; + } + + if (!std::all_of(path.begin(), path.end(), + [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) { + MS_LOG(WARNING) << err_msg << "The path only support alphabets, digit or {'-', '_', '/'}, but got:" << path << "."; + return false; + } + + if (path[0] != '/') { + MS_LOG(WARNING) << err_msg << "The path only support absolute path and should start with '/'."; + return false; + } + + if (!IsEveryFilenameValid(path, maxOSFilenameLength, err_msg)) { + return false; + } + return true; +} + +bool Common::IsFilenameValid(const std::string &filename, const int &length_limit, const std::string &error_message) { + std::string err_msg = "Detail: "; + if (!error_message.empty()) { + err_msg = error_message + " " + err_msg; + } + + if (filename.empty()) { + MS_LOG(WARNING) << err_msg << "The filename is empty."; + return false; + } + + if (!IsStrLengthValid(filename, length_limit, err_msg)) { + return false; + } + + if (!std::all_of(filename.begin(), filename.end(), + [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '.'; })) { + MS_LOG(WARNING) << err_msg << "The filename only support alphabets, digit or {'-', '_', '.'}, but got:" << filename + << "."; + return false; + } + return true; +} } // namespace mindspore diff --git a/mindspore/ccsrc/debug/common.h b/mindspore/ccsrc/debug/common.h index 7fbd9f31f1..e1913af5e4 100644 --- a/mindspore/ccsrc/debug/common.h +++ b/mindspore/ccsrc/debug/common.h @@ -22,6 +22,9 @@ #include "utils/contract.h" namespace mindspore { +static const int maxDirectoryLength = 1024; +static const int maxFilenameLength = 128; +static const int maxOSFilenameLength = 255; class Common { public: Common() = default; @@ -29,9 +32,16 @@ class Common { static std::optional GetRealPath(const std::string &input_path); static std::optional GetConfigFile(const std::string &env); static std::optional GetEnvConfigFile(); + static bool IsStrLengthValid(const std::string &str, const int &length_limit, const std::string &error_message = "", + const bool &print_str = true); + static bool IsPathValid(const std::string &path, const int &length_limit, const std::string &error_message = "", + const bool &print_str = true); + static bool IsFilenameValid(const std::string &filename, const int &length_limit, + const std::string &error_message = ""); private: static bool CreateNotExistDirs(const std::string &path); + static bool IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message); }; } // namespace mindspore #endif // MINDSPORE_CCSRC_DEBUG_COMMON_H_ diff --git a/mindspore/ccsrc/debug/env_config_parser.cc b/mindspore/ccsrc/debug/env_config_parser.cc index 4abb585874..15f3d7c61a 100644 --- a/mindspore/ccsrc/debug/env_config_parser.cc +++ b/mindspore/ccsrc/debug/env_config_parser.cc @@ -116,28 +116,20 @@ void EnvConfigParser::ParseRdrSetting(const nlohmann::json &content) { } void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) { + std::string err_msg = "RDR path parse failed. The RDR path will be a default value: '" + rdr_path_ + + "'. Please check the settings about '" + kRdrSettings + "' in config file '" + config_file_ + + "' set by 'env_config_path' in context."; + if (!CheckJsonStringType(content, kRdrSettings, kPath)) { - MS_LOG(WARNING) << "The RDR path will be a default value: '" << rdr_path_ << "'."; + MS_LOG(WARNING) << err_msg; return; } + std::string path = content; - if (!std::all_of(path.begin(), path.end(), - [](char c) { return ::isalpha(c) || ::isdigit(c) || c == '-' || c == '_' || c == '/'; })) { - MS_LOG(WARNING) << "The path in " << kRdrSettings - << " only support alphabets, digit or {'-', '_', '/'}, but got:" << path << "." - << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."; - return; - } - if (path.empty()) { - MS_LOG(WARNING) << "The path in " << kRdrSettings << " is empty." - << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."; - return; - } - if (path[0] != '/') { - MS_LOG(WARNING) << "The path in " << kRdrSettings << " only support absolute path and should start with '/'." - << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."; + if (!Common::IsPathValid(path, maxDirectoryLength, err_msg, false)) { return; } + if (path.back() != '/') { path += '/'; } @@ -146,7 +138,7 @@ void EnvConfigParser::ParseRdrPath(const nlohmann::json &content) { void EnvConfigParser::ParseRdrEnable(const nlohmann::json &content) { if (!content.is_boolean()) { - MS_LOG(WARNING) << "Json Parse Failed. 'enable' in " << kRdrSettings << " should be boolean." + MS_LOG(WARNING) << "Json parse failed. 'enable' in " << kRdrSettings << " should be boolean." << " Please check the config file '" << config_file_ << "' set by 'env_config_path' in context."; rdr_enabled_ = false; return; diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.cc b/mindspore/ccsrc/debug/rdr/base_recorder.cc new file mode 100644 index 0000000000..4c3f22894a --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/base_recorder.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/base_recorder.h" +#include +#include +#include "debug/common.h" +#include "utils/utils.h" + +namespace mindspore { +void BaseRecorder::SetDirectory(const std::string &directory) { + std::string error_message = module_ + ":" + tag_ + " set directory failed."; + if (Common::IsPathValid(directory, maxDirectoryLength, error_message)) { + directory_ = directory; + if (directory_.back() != '/') { + directory_ += "/"; + } + } +} + +void BaseRecorder::SetFilename(const std::string &filename) { + std::string error_message = module_ + ":" + tag_ + " set filename failed."; + if (Common::IsFilenameValid(filename, maxDirectoryLength, error_message)) { + filename_ = filename; + } +} + +std::optional BaseRecorder::GetFileRealPath() { + if (filename_.empty()) { + filename_ = module_ + "_" + tag_ + "_" + timestamp_; + } + std::string file_path = directory_ + filename_; + auto realpath = Common::GetRealPath(file_path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed. " + << "Info: module=" << module_ << ", tag=" << tag_ << ", " + << "path=" << file_path << "."; + } + + return realpath; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.h b/mindspore/ccsrc/debug/rdr/base_recorder.h index 9f16c3c7eb..fd9f5faeeb 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.h +++ b/mindspore/ccsrc/debug/rdr/base_recorder.h @@ -21,7 +21,11 @@ #include #include #include +#include "debug/common.h" #include "debug/env_config_parser.h" +#include "mindspore/core/utils/log_adapter.h" + +const int maxTagLength = 32; namespace mindspore { class BaseRecorder { public: @@ -31,6 +35,17 @@ class BaseRecorder { config_parser_ptr.Parse(); directory_ = config_parser_ptr.rdr_path(); + if (tag.length() > maxTagLength) { + tag_ = tag.substr(0, maxTagLength); + MS_LOG(WARNING) << "The tag length is " << tag.length() << ", exceeding the limit " << maxTagLength + << ". It will be intercepted as '" << tag_ << "'."; + } + + std::string err_msg = module_ + ":" + tag_ + " set filename failed."; + if (!filename_.empty() && !Common::IsFilenameValid(filename_, maxFilenameLength, err_msg)) { + filename_ = ""; + } + auto sys_time = std::chrono::system_clock::now(); auto t_time = std::chrono::system_clock::to_time_t(sys_time); std::tm *l_time = std::localtime(&t_time); @@ -47,9 +62,10 @@ class BaseRecorder { std::string GetModule() const { return module_; } std::string GetTag() const { return tag_; } std::string GetTimeStamp() const { return timestamp_; } + std::optional GetFileRealPath(); - void SetDirectory(const std::string &directory) { directory_ = directory; } - std::string GetDirectory() const { return directory_; } + void SetDirectory(const std::string &directory); + void SetFilename(const std::string &filename); virtual void Export() {} protected: diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc index a1d093b18e..168a018640 100644 --- a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc @@ -46,10 +46,11 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector } // namespace void GraphExecOrderRecorder::Export() { - if (filename_.empty()) { - filename_ = module_ + "_" + tag_ + "_" + timestamp_; + auto realpath = GetFileRealPath(); + if (!realpath.has_value()) { + return; } - std::string filename = directory_ + filename_ + ".txt"; - DumpGraphExeOrder(filename, exec_order_); + std::string real_file_path = realpath.value() + ".txt"; + DumpGraphExeOrder(real_file_path, exec_order_); } } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc index c509356acb..8d0ae8e4bd 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -25,33 +25,22 @@ namespace mindspore { namespace protobuf { #ifdef ENABLE_DUMP_IR -void DumpIRProto(const std::string &filename, const FuncGraphPtr &func_graph) { +void DumpIRProto(const std::string &real_path, const FuncGraphPtr &func_graph) { if (func_graph == nullptr) { - MS_LOG(ERROR) << "Func graph is nullptr"; + MS_LOG(ERROR) << "Func graph is nullptr."; return; } - if (filename.size() > PATH_MAX) { - MS_LOG(ERROR) << "File path " << filename << " is too long."; - return; - } - - auto real_path = Common::GetRealPath(filename); - if (!real_path.has_value()) { - MS_LOG(ERROR) << "Get real path failed. path=" << filename; - return; - } - ChangeFileMode(real_path.value(), S_IRWXU); // write to pb file - std::ofstream ofs(real_path.value()); + std::ofstream ofs(real_path); if (!ofs.is_open()) { - MS_LOG(ERROR) << "Open file '" << real_path.value() << "' failed!"; + MS_LOG(ERROR) << "Open file '" << real_path << "' failed!"; return; } ofs << GetFuncGraphProtoString(func_graph); ofs.close(); // set file mode to read only by user - ChangeFileMode(real_path.value(), S_IRUSR); + ChangeFileMode(real_path, S_IRUSR); } #else void DumpIRProto(const std::string &, const FuncGraphPtr &) { @@ -68,31 +57,35 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) { void GraphRecorder::Export() { bool save_flag = false; - if (filename_.empty()) { - filename_ = module_ + "_" + tag_ + "_" + timestamp_; + + auto tmp_realpath = GetFileRealPath(); + if (!tmp_realpath.has_value()) { + return; } - std::string file_path = directory_ + filename_ + std::to_string(id_); + + std::string realpath = tmp_realpath.value() + std::to_string(id_); + if (graph_type_.find(".dat") != std::string::npos) { save_flag = true; - AnfExporter exporter(""); - std::string real_path = file_path + ".dat"; - ChangeFileMode(real_path, S_IRWXU); - exporter.ExportFuncGraph(real_path, func_graph_); - ChangeFileMode(real_path, S_IRUSR); + AnfExporter exporter(std::to_string(id_)); + std::string realpath_dat = realpath + ".dat"; + ChangeFileMode(realpath_dat, S_IRWXU); + exporter.ExportFuncGraph(realpath_dat, func_graph_); + ChangeFileMode(realpath_dat, S_IRUSR); } if (graph_type_.find(".ir") != std::string::npos) { save_flag = true; - std::string real_path = file_path + ".ir"; + std::string realpath_ir = realpath + ".ir"; if (full_name_) { - DumpIRForRDR(real_path, func_graph_, true, kTopStack); + DumpIRForRDR(realpath_ir, func_graph_, true, kTopStack); } else { - DumpIRForRDR(real_path, func_graph_, false, kOff); + DumpIRForRDR(realpath_ir, func_graph_, false, kOff); } } if (graph_type_.find(".pb") != std::string::npos) { save_flag = true; - std::string real_path = file_path + ".pb"; - protobuf::DumpIRProto(real_path, func_graph_); // save *.pb file + + protobuf::DumpIRProto(realpath + ".pb", func_graph_); // save *.pb file } if (!save_flag) { MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_; diff --git a/mindspore/ccsrc/debug/rdr/string_recorder.cc b/mindspore/ccsrc/debug/rdr/string_recorder.cc index 9d75c8620c..9c8094adf5 100644 --- a/mindspore/ccsrc/debug/rdr/string_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/string_recorder.cc @@ -22,23 +22,13 @@ namespace mindspore { void StringRecorder::Export() { - if (directory_.back() != '/') { - directory_ += "/"; - } - - if (filename_.empty()) { - filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt"; - } - std::string file_path = directory_ + filename_; - - auto realpath = Common::GetRealPath(file_path); + auto realpath = GetFileRealPath(); if (!realpath.has_value()) { - MS_LOG(ERROR) << "Get real path failed. path=" << file_path; return; } - - ChangeFileMode(realpath.value(), S_IRWXU); - std::ofstream fout(realpath.value(), std::ofstream::app); + std::string file_path = realpath.value() + ".txt"; + ChangeFileMode(file_path, S_IRWXU); + std::ofstream fout(file_path, std::ofstream::app); if (!fout.is_open()) { MS_LOG(WARNING) << "Open file for saving string failed."; return; @@ -46,6 +36,6 @@ void StringRecorder::Export() { fout << data_; fout.close(); // set file mode to read only by user - ChangeFileMode(realpath.value(), S_IRUSR); + ChangeFileMode(file_path, S_IRUSR); } } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/string_recorder.h b/mindspore/ccsrc/debug/rdr/string_recorder.h index 578251adeb..a6ba4b0e1f 100644 --- a/mindspore/ccsrc/debug/rdr/string_recorder.h +++ b/mindspore/ccsrc/debug/rdr/string_recorder.h @@ -25,15 +25,15 @@ class StringRecorder : public BaseRecorder { public: StringRecorder() : BaseRecorder() {} StringRecorder(const std::string &module, const std::string &tag, const std::string &data, - const std::string &file_type) - : BaseRecorder(module, tag), data_(data) {} + const std::string &filename) + : BaseRecorder(module, tag), data_(data), filename_(filename) {} ~StringRecorder() {} void SetModule(const std::string &module) { module_ = module; } - void SetFilename(const std::string &filename) { filename_ = filename; } virtual void Export(); private: std::string data_; + std::string filename_; }; using StringRecorderPtr = std::shared_ptr; } // namespace mindspore diff --git a/mindspore/ccsrc/runtime/device/memory_manager.cc b/mindspore/ccsrc/runtime/device/memory_manager.cc index 697c9afd33..46d0ca8b30 100644 --- a/mindspore/ccsrc/runtime/device/memory_manager.cc +++ b/mindspore/ccsrc/runtime/device/memory_manager.cc @@ -76,10 +76,10 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) { SubModuleId module = SubModuleId::SM_OPTIMIZER; std::string tag = "somas"; - std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir"; + std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()); mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename); - filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir"; + filename = "somas_mem_info_" + std::to_string(graph->graph_id()); mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename); #endif bool save_graphs = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG); diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 5180a9990c..1f0fa4fdbd 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -18,6 +18,7 @@ import math import operator from functools import reduce, partial +from mindspore import log as logger from mindspore._checkparam import _check_3d_int_or_tuple import numpy as np from ... import context