From: @luopengting Reviewed-by: @ouwenchang,@lixiaohui33 Signed-off-by: @lixiaohui33tags/v1.2.0-rc1
| @@ -399,8 +399,8 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern | |||
| kernel_graph->SetExecOrderByDefault(); | |||
| #ifdef ENABLE_DUMP_IR | |||
| const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order(); | |||
| std::string exec_order_tag = "graph_exec_order"; | |||
| mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, exec_order_tag, exec_order, kernel_graph->graph_id()); | |||
| std::string exec_order_name = "graph_exec_order." + std::to_string(kernel_graph->graph_id()); | |||
| mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, exec_order_name, exec_order); | |||
| #endif | |||
| if (save_graphs) { | |||
| std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir"; | |||
| @@ -379,11 +379,10 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) { | |||
| #ifdef ENABLE_DUMP_IR | |||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||
| std::string tag = "somas"; | |||
| std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); | |||
| filename = "somas_offline_log_" + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, tag, Offline(), filename); | |||
| std::string name = "somas_pre_processed_info." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, name, SomasInfo()); | |||
| name = "somas_offline_log." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, name, Offline()); | |||
| #endif | |||
| if (save_graphs_) { | |||
| @@ -671,9 +670,8 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) { | |||
| #ifdef ENABLE_DUMP_IR | |||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||
| std::string tag = "somas"; | |||
| std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); | |||
| std::string name = "somas_initial_info." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, name, SomasInfo()); | |||
| #endif | |||
| save_graphs_ = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | |||
| @@ -1010,8 +1010,8 @@ void AscendSession::DumpAllGraphs(const std::vector<KernelGraphPtr> &all_graphs) | |||
| uint32_t device_id = kernel_runtime->device_id(); | |||
| for (auto &graph : all_graphs) { | |||
| MS_EXCEPTION_IF_NULL(graph); | |||
| std::string tag = "graph_build"; | |||
| mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph, true, ".ir;.pb"); | |||
| std::string name = "graph_build." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph, true, ".ir;.pb"); | |||
| if (save_graphs) { | |||
| std::string file_name = "graph_build_" + std::to_string(graph->graph_id()) + ".ir"; | |||
| DumpIR(file_name, graph, true, kWholeStack); | |||
| @@ -372,8 +372,8 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) { | |||
| // Build kernel if node is cnode | |||
| BuildKernel(graph); | |||
| #ifdef ENABLE_DUMP_IR | |||
| std::string tag = "graph_build"; | |||
| mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, ".ir,.pb"); | |||
| std::string name = "graph_build"; | |||
| mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, name, graph, false, ".ir,.pb"); | |||
| #endif | |||
| // Get summary nodes. | |||
| SetSummaryNodes(graph.get()); | |||
| @@ -21,7 +21,7 @@ | |||
| namespace mindspore { | |||
| void BaseRecorder::SetDirectory(const std::string &directory) { | |||
| std::string error_message = module_ + ":" + tag_ + " set directory failed."; | |||
| std::string error_message = module_ + ":" + name_ + " set directory failed."; | |||
| if (Common::IsPathValid(directory, maxDirectoryLength, error_message)) { | |||
| directory_ = directory; | |||
| if (directory_.back() != '/') { | |||
| @@ -31,7 +31,7 @@ void BaseRecorder::SetDirectory(const std::string &directory) { | |||
| } | |||
| void BaseRecorder::SetFilename(const std::string &filename) { | |||
| std::string error_message = module_ + ":" + tag_ + " set filename failed."; | |||
| std::string error_message = module_ + ":" + name_ + " set filename failed."; | |||
| if (Common::IsFilenameValid(filename, maxDirectoryLength, error_message)) { | |||
| filename_ = filename; | |||
| } | |||
| @@ -40,7 +40,7 @@ void BaseRecorder::SetFilename(const std::string &filename) { | |||
| std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suffix) { | |||
| std::string filename; | |||
| if (filename_.empty()) { | |||
| filename = module_ + delimiter_ + tag_; | |||
| filename = module_ + delimiter_ + name_; | |||
| if (!suffix.empty()) { | |||
| filename += delimiter_ + suffix; | |||
| } | |||
| @@ -55,7 +55,7 @@ std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suff | |||
| auto realpath = Common::GetRealPath(file_path); | |||
| if (!realpath.has_value()) { | |||
| MS_LOG(ERROR) << "Get real path failed. " | |||
| << "Info: module=" << module_ << ", tag=" << tag_ << ", " | |||
| << "Info: module=" << module_ << ", name=" << name_ << ", " | |||
| << "path=" << file_path << "."; | |||
| } | |||
| @@ -24,21 +24,21 @@ | |||
| #include "debug/env_config_parser.h" | |||
| #include "mindspore/core/utils/log_adapter.h" | |||
| const int maxTagLength = 32; | |||
| const int maxNameLength = 32; | |||
| namespace mindspore { | |||
| class BaseRecorder { | |||
| public: | |||
| BaseRecorder() : module_(""), tag_(""), directory_(""), filename_(""), timestamp_("") {} | |||
| BaseRecorder(const std::string &module, const std::string &tag) : module_(module), tag_(tag), filename_("") { | |||
| BaseRecorder() : module_(""), name_(""), directory_(""), filename_(""), timestamp_("") {} | |||
| BaseRecorder(const std::string &module, const std::string &name) : module_(module), name_(name), filename_("") { | |||
| directory_ = mindspore::EnvConfigParser::GetInstance().rdr_path(); | |||
| if (tag.length() > maxTagLength) { | |||
| tag_ = tag.substr(0, maxTagLength); | |||
| MS_LOG(WARNING) << "The tag length is " << tag.length() << ", exceeding the limit " << maxTagLength | |||
| << ". It will be intercepted as '" << tag_ << "'."; | |||
| if (name.length() > maxNameLength) { | |||
| name_ = name.substr(0, maxNameLength); | |||
| MS_LOG(WARNING) << "The name length is " << name.length() << ", exceeding the limit " << maxNameLength | |||
| << ". It will be intercepted as '" << name_ << "'."; | |||
| } | |||
| std::string err_msg = module_ + ":" + tag_ + " set filename failed."; | |||
| std::string err_msg = module_ + ":" + name_ + " set filename failed."; | |||
| if (!filename_.empty() && !Common::IsFilenameValid(filename_, maxFilenameLength, err_msg)) { | |||
| filename_ = ""; | |||
| } | |||
| @@ -57,7 +57,7 @@ class BaseRecorder { | |||
| virtual ~BaseRecorder() {} | |||
| std::string GetModule() const { return module_; } | |||
| std::string GetTag() const { return tag_; } | |||
| std::string GetName() const { return name_; } | |||
| std::string GetTimeStamp() const { return timestamp_; } | |||
| std::optional<std::string> GetFileRealPath(const std::string &suffix = ""); | |||
| @@ -65,10 +65,11 @@ class BaseRecorder { | |||
| void SetFilename(const std::string &filename); | |||
| void SetModule(const std::string &module) { module_ = module; } | |||
| virtual void Export() {} | |||
| virtual void UpdateInfo(const BaseRecorder &recorder) {} | |||
| protected: | |||
| std::string module_; | |||
| std::string tag_; | |||
| std::string name_; | |||
| std::string directory_; | |||
| std::string filename_; | |||
| std::string timestamp_; // year,month,day,hour,minute,second | |||
| @@ -49,7 +49,7 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr> | |||
| } // namespace | |||
| void GraphExecOrderRecorder::Export() { | |||
| auto realpath = GetFileRealPath(std::to_string(graph_id_)); | |||
| auto realpath = GetFileRealPath(); | |||
| if (!realpath.has_value()) { | |||
| return; | |||
| } | |||
| @@ -27,16 +27,15 @@ using CNodePtr = std::shared_ptr<CNode>; | |||
| class GraphExecOrderRecorder : public BaseRecorder { | |||
| public: | |||
| GraphExecOrderRecorder() : BaseRecorder() {} | |||
| GraphExecOrderRecorder(const std::string &module, const std::string &tag, | |||
| const std::vector<CNodePtr> &final_exec_order, int graph_id) | |||
| : BaseRecorder(module, tag), exec_order_(final_exec_order), graph_id_(graph_id) {} | |||
| GraphExecOrderRecorder(const std::string &module, const std::string &name, | |||
| const std::vector<CNodePtr> &final_exec_order) | |||
| : BaseRecorder(module, name), exec_order_(final_exec_order) {} | |||
| ~GraphExecOrderRecorder() {} | |||
| void SetExecOrder(const std::vector<CNodePtr> &final_exec_order) { exec_order_ = final_exec_order; } | |||
| virtual void Export(); | |||
| private: | |||
| std::vector<CNodePtr> exec_order_; | |||
| int graph_id_; | |||
| }; | |||
| using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>; | |||
| } // namespace mindspore | |||
| @@ -58,13 +58,7 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) { | |||
| void GraphRecorder::Export() { | |||
| bool save_flag = false; | |||
| int graph_id = -1; | |||
| if (func_graph_->isa<session::KernelGraph>()) { | |||
| auto kernel_graph = func_graph_->cast<KernelGraphPtr>(); | |||
| graph_id = kernel_graph->graph_id(); | |||
| } | |||
| std::string suffix = graph_id >= 0 ? std::to_string(graph_id) : ""; | |||
| auto tmp_realpath = GetFileRealPath(suffix); | |||
| auto tmp_realpath = GetFileRealPath(); | |||
| if (!tmp_realpath.has_value()) { | |||
| return; | |||
| } | |||
| @@ -27,9 +27,9 @@ using FuncGraphPtr = std::shared_ptr<FuncGraph>; | |||
| class GraphRecorder : public BaseRecorder { | |||
| public: | |||
| GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {} | |||
| GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph, | |||
| GraphRecorder(const std::string &module, const std::string &name, const FuncGraphPtr &graph, | |||
| const std::string &file_type) | |||
| : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type) {} | |||
| : BaseRecorder(module, name), func_graph_(graph), graph_type_(file_type) {} | |||
| ~GraphRecorder() {} | |||
| void SetGraphType(const std::string &file_type) { graph_type_ = file_type; } | |||
| void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; } | |||
| @@ -32,10 +32,6 @@ std::string MemInfo2String(const std::string &label, const AddressPtrList &info) | |||
| return ss.str(); | |||
| } | |||
| } // namespace | |||
| MemAddressRecorder &MemAddressRecorder::Instance() { | |||
| static MemAddressRecorder recorder; | |||
| return recorder; | |||
| } | |||
| void MemAddressRecorder::SaveMemInfo(const std::string &op_name, const GPUMemInfo &mem_info) { | |||
| std::lock_guard<std::mutex> lock(mtx_); | |||
| @@ -48,6 +44,7 @@ void MemAddressRecorder::SaveMemInfo(const std::string &op_name, const GPUMemInf | |||
| auto outputs = mem_info.outputs_; | |||
| mem_info_stream << MemInfo2String("kernel_outputs", *outputs); | |||
| mem_info_stream << std::endl; | |||
| std::string mem_info_str = mem_info_stream.str(); | |||
| mem_info_container_[op_name] = mem_info_str; | |||
| } | |||
| @@ -71,4 +68,12 @@ void MemAddressRecorder::Export() { | |||
| fout.close(); | |||
| ChangeFileMode(file_path, S_IRUSR); | |||
| } | |||
| void MemAddressRecorder::UpdateInfo(const BaseRecorder &recorder) { | |||
| const MemAddressRecorder *mem_recorder = reinterpret_cast<const MemAddressRecorder *>(&recorder); | |||
| std::map<std::string, std::string> mem_info = mem_recorder->MemInfo(); | |||
| for (const auto &info : mem_info) { | |||
| mem_info_container_[info.first] = info.second; | |||
| } | |||
| } | |||
| } // namespace mindspore | |||
| @@ -36,17 +36,16 @@ struct GPUMemInfo { | |||
| }; | |||
| class MemAddressRecorder : public BaseRecorder { | |||
| public: | |||
| static MemAddressRecorder &Instance(); | |||
| MemAddressRecorder() {} | |||
| MemAddressRecorder(const std::string &module, const std::string &name) : BaseRecorder(module, name) {} | |||
| ~MemAddressRecorder() {} | |||
| virtual void Export(); | |||
| void SaveMemInfo(const std::string &op_name, const GPUMemInfo &mem_info); | |||
| void SetTag(const std::string &tag) { tag_ = tag; } | |||
| void UpdateInfo(const BaseRecorder &recorder); | |||
| std::map<std::string, std::string> MemInfo() const { return mem_info_container_; } | |||
| private: | |||
| MemAddressRecorder() {} | |||
| MemAddressRecorder(const MemAddressRecorder &recorder); | |||
| ~MemAddressRecorder() {} | |||
| MemAddressRecorder &operator=(const MemAddressRecorder &recorder); | |||
| mutable std::mutex mtx_; | |||
| std::map<std::string, std::string> mem_info_container_; | |||
| @@ -42,7 +42,7 @@ void RecorderManager::UpdateRdrEnable() { | |||
| updated = true; | |||
| } | |||
| bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) { | |||
| bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder, const bool &replace) { | |||
| if (!rdr_enable_) { | |||
| return false; | |||
| } | |||
| @@ -52,8 +52,20 @@ bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) { | |||
| return false; | |||
| } | |||
| std::string module = recorder->GetModule(); | |||
| std::string name = recorder->GetName(); | |||
| std::pair<std::string, std::string> recorder_key(module, name); | |||
| std::lock_guard<std::mutex> lock(mtx_); | |||
| recorder_container_[module].push_back(std::move(recorder)); | |||
| if (replace) { | |||
| recorder_container_[recorder_key] = recorder; | |||
| return true; | |||
| } | |||
| std::unordered_map<std::pair<std::string, std::string>, BaseRecorderPtr, pair_hash>::iterator item = | |||
| recorder_container_.find(recorder_key); | |||
| if (item == recorder_container_.end()) { | |||
| recorder_container_[recorder_key] = recorder; | |||
| } else { | |||
| recorder_container_[recorder_key]->UpdateInfo(*recorder); | |||
| } | |||
| return true; | |||
| } | |||
| @@ -65,10 +77,8 @@ void RecorderManager::TriggerAll() { | |||
| bool trigger = false; | |||
| std::lock_guard<std::mutex> lock(mtx_); | |||
| for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) { | |||
| for (auto &recorder : iter->second) { | |||
| recorder->Export(); | |||
| trigger = true; | |||
| } | |||
| iter->second->Export(); | |||
| trigger = true; | |||
| } | |||
| if (!trigger) { | |||
| MS_LOG(WARNING) << "There is no recorder to export."; | |||
| @@ -22,11 +22,36 @@ | |||
| #include <unordered_map> | |||
| #include <memory> | |||
| #include <mutex> | |||
| #include <utility> | |||
| namespace mindspore { | |||
| template <typename T> | |||
| inline void hash_combine(std::size_t *seed, const T &val) { | |||
| // The number is the reciprocal of the golden ratio. | |||
| unsigned int magic_constant = 0x9e3779b9; | |||
| int shift_left = 6; | |||
| int shift_right = 2; | |||
| *seed ^= std::hash<T>()(val) + magic_constant + (*seed << shift_left) + (*seed >> shift_right); | |||
| } | |||
| template <typename T1, typename T2> | |||
| inline std::size_t hash_seed(const T1 &val1, const T2 &val2) { | |||
| std::size_t seed = 0; | |||
| hash_combine(&seed, val1); | |||
| hash_combine(&seed, val2); | |||
| return seed; | |||
| } | |||
| struct pair_hash { | |||
| template <class T1, class T2> | |||
| std::size_t operator()(const std::pair<T1, T2> &p) const { | |||
| return hash_seed(p.first, p.second); | |||
| } | |||
| }; | |||
| class BaseRecorder; | |||
| using BaseRecorderPtr = std::shared_ptr<BaseRecorder>; | |||
| using BaseRecorderPtrList = std::vector<BaseRecorderPtr>; | |||
| class RecorderManager { | |||
| public: | |||
| static RecorderManager &Instance() { | |||
| @@ -37,7 +62,7 @@ class RecorderManager { | |||
| void UpdateRdrEnable(); | |||
| bool RdrEnable() const { return rdr_enable_; } | |||
| bool RecordObject(const BaseRecorderPtr &recorder); | |||
| bool RecordObject(const BaseRecorderPtr &recorder, const bool &replace = true); | |||
| void TriggerAll(); | |||
| void ClearAll(); | |||
| @@ -48,8 +73,8 @@ class RecorderManager { | |||
| bool rdr_enable_{false}; | |||
| mutable std::mutex mtx_; | |||
| // module, BaserRecorderPtrList | |||
| std::unordered_map<std::string, BaseRecorderPtrList> recorder_container_; | |||
| // <module, name>, BaserRecorderPtr | |||
| std::unordered_map<std::pair<std::string, std::string>, BaseRecorderPtr, pair_hash> recorder_container_; | |||
| }; | |||
| } // namespace mindspore | |||
| #endif // MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_ | |||
| @@ -65,84 +65,78 @@ static const char *GetSubModuleName(SubModuleId module_id) { | |||
| } // namespace | |||
| namespace RDR { | |||
| #ifdef ENABLE_D | |||
| bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info_list, int graph_id) { | |||
| bool RecordTaskDebugInfo(SubModuleId module, const std::string &name, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info_list) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| TaskDebugInfoRecorderPtr task_debug_info_recorder = | |||
| std::make_shared<TaskDebugInfoRecorder>(submodule_name, tag, task_debug_info_list, graph_id); | |||
| std::make_shared<TaskDebugInfoRecorder>(submodule_name, name, task_debug_info_list); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(task_debug_info_recorder)); | |||
| return ans; | |||
| } | |||
| #endif // ENABLE_D | |||
| bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, | |||
| bool RecordAnfGraph(const SubModuleId module, const std::string &name, const FuncGraphPtr &graph, bool full_name, | |||
| const std::string &file_type) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, tag, graph, file_type); | |||
| GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, name, graph, file_type); | |||
| graph_recorder->SetDumpFlag(full_name); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder)); | |||
| return ans; | |||
| } | |||
| bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, | |||
| const std::vector<CNodePtr> &final_exec_order, int graph_id) { | |||
| bool RecordGraphExecOrder(const SubModuleId module, const std::string &name, | |||
| const std::vector<CNodePtr> &final_exec_order) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| GraphExecOrderRecorderPtr graph_exec_order_recorder = | |||
| std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order, graph_id); | |||
| std::make_shared<GraphExecOrderRecorder>(submodule_name, name, final_exec_order); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder)); | |||
| return ans; | |||
| } | |||
| bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { | |||
| bool RecordString(SubModuleId module, const std::string &name, const std::string &data) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, tag, data, filename); | |||
| string_recorder->SetFilename(filename); | |||
| StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, name, data); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(string_recorder)); | |||
| return ans; | |||
| } | |||
| bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id, | |||
| const std::vector<CNodePtr> &exec_order) { | |||
| bool RecordStreamExecOrder(const SubModuleId module, const std::string &name, const std::vector<CNodePtr> &exec_order) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| StreamExecOrderRecorderPtr stream_exec_order_recorder = | |||
| std::make_shared<StreamExecOrderRecorder>(submodule_name, tag, graph_id, exec_order); | |||
| std::make_shared<StreamExecOrderRecorder>(submodule_name, name, exec_order); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(stream_exec_order_recorder)); | |||
| return ans; | |||
| } | |||
| bool RecordMemAddressInfo(const SubModuleId module, const std::string &tag, const std::string &op_name, | |||
| bool RecordMemAddressInfo(const SubModuleId module, const std::string &name, const std::string &op_name, | |||
| const GPUMemInfo &mem_info) { | |||
| if (!mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| return false; | |||
| } | |||
| std::string submodule_name = std::string(GetSubModuleName(module)); | |||
| std::string directory = mindspore::EnvConfigParser::GetInstance().rdr_path(); | |||
| MemAddressRecorder::Instance().SetModule(submodule_name); | |||
| MemAddressRecorder::Instance().SetFilename(tag); // set filename using tag | |||
| MemAddressRecorder::Instance().SetDirectory(directory); | |||
| MemAddressRecorder::Instance().SaveMemInfo(op_name, mem_info); | |||
| return true; | |||
| } | |||
| void TriggerAll() { | |||
| mindspore::RecorderManager::Instance().TriggerAll(); | |||
| MemAddressRecorder::Instance().Export(); | |||
| MemAddressRecorderPtr mem_info_recorder = std::make_shared<MemAddressRecorder>(submodule_name, name); | |||
| mem_info_recorder->SaveMemInfo(op_name, mem_info); | |||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(mem_info_recorder), false); | |||
| return ans; | |||
| } | |||
| void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } | |||
| void ClearAll() { mindspore::RecorderManager::Instance().ClearAll(); } | |||
| } // namespace RDR | |||
| } // namespace mindspore | |||
| @@ -44,19 +44,17 @@ using TaskDebugInfoPtr = std::shared_ptr<device::ascend::tasksink::TaskDebugInfo | |||
| #endif // ENABLE_D | |||
| namespace RDR { | |||
| bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, | |||
| bool RecordAnfGraph(const SubModuleId module, const std::string &name, const FuncGraphPtr &graph, bool full_name, | |||
| const std::string &file_type = ".ir;.pb;.dat"); | |||
| bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, | |||
| const std::vector<CNodePtr> &final_exec_order, int graph_id = 0); | |||
| bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, | |||
| const std::string &filename = ""); | |||
| bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id, | |||
| const std::vector<CNodePtr> &exec_order); | |||
| bool RecordMemAddressInfo(const SubModuleId module, const std::string &tag, const std::string &op_name, | |||
| bool RecordGraphExecOrder(const SubModuleId module, const std::string &name, | |||
| const std::vector<CNodePtr> &final_exec_order); | |||
| bool RecordString(SubModuleId module, const std::string &name, const std::string &data); | |||
| bool RecordStreamExecOrder(const SubModuleId module, const std::string &name, const std::vector<CNodePtr> &exec_order); | |||
| bool RecordMemAddressInfo(const SubModuleId module, const std::string &name, const std::string &op_name, | |||
| const GPUMemInfo &mem_info); | |||
| #ifdef ENABLE_D | |||
| bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info_list, int graph_id = 0); | |||
| bool RecordTaskDebugInfo(SubModuleId module, const std::string &name, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info_list); | |||
| #endif // ENABLE_D | |||
| void TriggerAll(); | |||
| void ClearAll(); | |||
| @@ -49,8 +49,7 @@ json ExecNode::ExecNode2Json() { | |||
| } | |||
| void StreamExecOrderRecorder::Export() { | |||
| std::string file_suffix = std::to_string(graph_id_); | |||
| auto realpath = GetFileRealPath(file_suffix); | |||
| auto realpath = GetFileRealPath(); | |||
| if (!realpath.has_value()) { | |||
| return; | |||
| } | |||
| @@ -61,9 +61,8 @@ using CNodePtr = std::shared_ptr<CNode>; | |||
| class StreamExecOrderRecorder : public BaseRecorder { | |||
| public: | |||
| StreamExecOrderRecorder() : BaseRecorder() {} | |||
| StreamExecOrderRecorder(const std::string &module, const std::string &tag, const int &graph_id, | |||
| const std::vector<CNodePtr> &exec_order) | |||
| : BaseRecorder(module, tag), graph_id_(graph_id) { | |||
| StreamExecOrderRecorder(const std::string &module, const std::string &name, const std::vector<CNodePtr> &exec_order) | |||
| : BaseRecorder(module, name) { | |||
| // Extract information from execute order. | |||
| for (size_t i = 0; i < exec_order.size(); i++) { | |||
| CNodePtr cur_cnode_ptr = exec_order[i]; | |||
| @@ -104,7 +103,6 @@ class StreamExecOrderRecorder : public BaseRecorder { | |||
| private: | |||
| std::vector<ExecNodePtr> exec_order_; | |||
| int graph_id_{0}; | |||
| }; | |||
| using StreamExecOrderRecorderPtr = std::shared_ptr<StreamExecOrderRecorder>; | |||
| } // namespace mindspore | |||
| @@ -24,11 +24,8 @@ namespace mindspore { | |||
| class StringRecorder : public BaseRecorder { | |||
| public: | |||
| StringRecorder() : BaseRecorder() {} | |||
| StringRecorder(const std::string &module, const std::string &tag, const std::string &data, | |||
| const std::string &filename) | |||
| : BaseRecorder(module, tag), data_(data) { | |||
| SetFilename(filename); | |||
| } | |||
| StringRecorder(const std::string &module, const std::string &name, const std::string &data) | |||
| : BaseRecorder(module, name), data_(data) {} | |||
| ~StringRecorder() {} | |||
| virtual void Export(); | |||
| @@ -18,7 +18,7 @@ | |||
| namespace mindspore { | |||
| void TaskDebugInfoRecorder::Export() { | |||
| auto realpath = GetFileRealPath(std::to_string(graph_id_)); | |||
| auto realpath = GetFileRealPath(); | |||
| if (!realpath.has_value()) { | |||
| return; | |||
| } | |||
| @@ -34,14 +34,13 @@ using TaskDebugInfoPtr = std::shared_ptr<device::ascend::tasksink::TaskDebugInfo | |||
| class TaskDebugInfoRecorder : public BaseRecorder { | |||
| public: | |||
| TaskDebugInfoRecorder() {} | |||
| TaskDebugInfoRecorder(const std::string &module, const std::string &tag, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info, int graph_id) | |||
| : BaseRecorder(module, tag), graph_id_(graph_id), task_debug_info_(task_debug_info) {} | |||
| TaskDebugInfoRecorder(const std::string &module, const std::string &name, | |||
| const std::vector<TaskDebugInfoPtr> &task_debug_info) | |||
| : BaseRecorder(module, name), task_debug_info_(task_debug_info) {} | |||
| ~TaskDebugInfoRecorder() {} | |||
| virtual void Export(); | |||
| private: | |||
| int graph_id_; | |||
| std::vector<TaskDebugInfoPtr> task_debug_info_; | |||
| }; | |||
| using TaskDebugInfoRecorderPtr = std::shared_ptr<TaskDebugInfoRecorder>; | |||
| @@ -737,11 +737,11 @@ void Pipeline::Run() { | |||
| #ifdef ENABLE_DUMP_IR | |||
| if (mindspore::RecorderManager::Instance().RdrEnable()) { | |||
| MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR."; | |||
| std::string tag = GetBaseNameForIR(i, action.first); | |||
| std::string name = GetBaseNameForIR(i, action.first); | |||
| if (graph != nullptr) { | |||
| auto graph_clone = BasicClone(graph); | |||
| if (graph_clone != nullptr) { | |||
| mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph_clone, false, ".ir"); | |||
| mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph_clone, false, ".ir"); | |||
| } else { | |||
| MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR."; | |||
| } | |||
| @@ -229,9 +229,9 @@ void AscendStreamAssign::AssignStream(const NotNull<KernelGraphPtr> &graph_ptr) | |||
| MS_LOG(INFO) << "After finish stream assign"; | |||
| #ifdef ENABLE_DUMP_IR | |||
| SubModuleId module = SubModuleId::SM_SESSION; | |||
| std::string tag = "assign_stream"; | |||
| std::string name = "assign_stream." + std::to_string(graph_ptr->graph_id()); | |||
| const std::vector<CNodePtr> &exec_order = graph_ptr->execution_order(); | |||
| mindspore::RDR::RecordStreamExecOrder(module, tag, graph_ptr->graph_id(), exec_order); | |||
| mindspore::RDR::RecordStreamExecOrder(module, name, exec_order); | |||
| #endif | |||
| graph_ptr->PrintGraphExecuteOrder(); | |||
| @@ -40,8 +40,8 @@ bool TaskGenerator::GenTasks(const std::vector<CNodePtr> &anf_node_list, std::ve | |||
| } | |||
| MS_LOG(INFO) << "GenTasks end..."; | |||
| #ifdef ENABLE_DUMP_IR | |||
| string task_info_tag = "task_info_graph"; | |||
| mindspore::RDR::RecordTaskDebugInfo(SUBMODULE_ID, task_info_tag, task_debug_info_list_, graph_id); | |||
| string task_info_name = "task_info_graph." + std::to_string(graph_id); | |||
| mindspore::RDR::RecordTaskDebugInfo(SUBMODULE_ID, task_info_name, task_debug_info_list_); | |||
| #endif | |||
| auto context_ptr = MsContext::GetInstance(); | |||
| MS_EXCEPTION_IF_NULL(context_ptr); | |||
| @@ -650,8 +650,8 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo | |||
| auto &kernels = graph->execution_order(); | |||
| int exec_order = 1; | |||
| #ifdef ENABLE_DUMP_IR | |||
| std::string exec_order_tag = "graph_exec_order"; | |||
| mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_KERNEL, exec_order_tag, kernels, graph->graph_id()); | |||
| std::string exec_order_name = "graph_exec_order." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_KERNEL, exec_order_name, kernels); | |||
| #endif | |||
| auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); | |||
| MS_EXCEPTION_IF_NULL(profiler_inst); | |||
| @@ -695,9 +695,9 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo | |||
| } | |||
| #ifdef ENABLE_DUMP_IR | |||
| GPUMemInfo mem_info = {&kernel_inputs, &kernel_workspaces, &kernel_outputs}; | |||
| std::string tag = "mem_address_list"; | |||
| std::string name = "mem_address_list"; | |||
| std::string op_name = kernel->fullname_with_scope(); | |||
| mindspore::RDR::RecordMemAddressInfo(SubModuleId::SM_KERNEL, tag, op_name, mem_info); | |||
| mindspore::RDR::RecordMemAddressInfo(SubModuleId::SM_KERNEL, name, op_name, mem_info); | |||
| #endif | |||
| if (!mock) { | |||
| if (!profiling) { | |||
| @@ -74,13 +74,12 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) { | |||
| MS_EXCEPTION_IF_NULL(context_ptr); | |||
| #ifdef ENABLE_DUMP_IR | |||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||
| std::string tag = "somas"; | |||
| std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename); | |||
| std::string name = "somas_allocate_info." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, name, somas_reuse_util_ptr_->SomasInfo()); | |||
| filename = "somas_mem_info_" + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename); | |||
| name = "somas_mem_info." + std::to_string(graph->graph_id()); | |||
| mindspore::RDR::RecordString(module, name, somas_reuse_util_ptr_->SomasMemory()); | |||
| #endif | |||
| bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | |||
| auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH); | |||