diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc b/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc index e9953ac5b3..4f4829fed9 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -122,7 +122,9 @@ #include "utils/config_manager.h" #include "debug/anf_ir_dump.h" #include "debug/dump_proto.h" - +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif namespace mindspore { namespace opt { namespace { @@ -262,6 +264,10 @@ void AscendBackendIRFusionOptimization(const std::shared_ptrget_param(MS_CTX_SAVE_GRAPHS_FLAG); +#ifdef ENABLE_DUMP_IR + std::string tag = "before_hwopt"; + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph); +#endif if (save_graphs) { std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir"; DumpIR(file_name, kernel_graph); @@ -380,7 +386,13 @@ void AscendBackendOptimization(const std::shared_ptr &kern optimizer2->AddPassManager(other2_pm); (void)optimizer2->Optimize(kernel_graph); kernel_graph->SetExecOrderByDefault(); - +#ifdef ENABLE_DUMP_IR + std::string tag = "hwopt_d_end"; + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph); + const std::vector &exec_order = kernel_graph->execution_order(); + tag = "graph_exec_order"; + mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(exec_order)); +#endif if (save_graphs) { std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir"; DumpIR(file_name, kernel_graph, true, kTopStack); diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index 452542091c..ce639bbf65 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019-2020 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -62,6 +62,9 @@ #include "debug/anf_ir_dump.h" #include "debug/dump_proto.h" #include "toolchain/adx_datadump_server.h" +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif #if ENABLE_CPU && ENABLE_D #include "ps/util.h" #include "ps/ps_cache/ps_cache_manager.h" @@ -129,7 +132,6 @@ std::vector GetCNodes(const std::vector &anf_nodes) { } return cnodes; } - void InsertMakeTupleForOutput(NotNull root_graph) { auto return_node = root_graph->get_return(); MS_EXCEPTION_IF_NULL(return_node); @@ -715,7 +717,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull func_graph) { InitRuntimeResource(); return root_graph->graph_id(); } - // create parameter for multiple branch std::set memo; CreateMultiBranchOutput(NOT_NULL(root_graph), NOT_NULL(&memo)); @@ -726,10 +727,8 @@ GraphId AscendSession::CompileGraphImpl(NotNull func_graph) { MultiCallGraphOptimize(NOT_NULL(root_graph)); // resource initialize InitRuntimeResource(); - IrFusionPass(NOT_NULL(root_graph), NOT_NULL(&memo)); memo.clear(); - SelectKernel(NOT_NULL(root_graph)); memo.clear(); @@ -740,7 +739,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull func_graph) { LoadGraphsToDbg(NOT_NULL(root_graph), NOT_NULL(&memo)); } memo.clear(); - UpdateRefOutputMap(NOT_NULL(root_graph), NOT_NULL(&memo)); memo.clear(); // add make_tuple to the output graph @@ -757,7 +755,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull func_graph) { // adjust kernel AdjustKernel(root_graph); - // reorder send/recv auto execution_order = root_graph->execution_order(); ReorderSendRecv(&execution_order); @@ -1333,6 +1330,9 @@ void AscendSession::Execute(const std::shared_ptr &kernel_graph, bo bool ret_ok = runtime_instance->Run(kernel_graph.get(), is_task_sink); Dump(kernel_graph); if (!ret_ok) { +#ifdef ENABLE_DUMP_IR + mindspore::RDR::TriggerAll(); +#endif MS_LOG(EXCEPTION) << "run task error!"; } MS_LOG(INFO) << "Finish!"; diff --git a/mindspore/ccsrc/backend/session/executor.cc b/mindspore/ccsrc/backend/session/executor.cc index c0b5faa736..e4f53d4705 100644 --- a/mindspore/ccsrc/backend/session/executor.cc +++ b/mindspore/ccsrc/backend/session/executor.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,9 @@ #include "runtime/device/kernel_runtime_manager.h" #include "utils/comm_manager.h" #include "utils/scoped_long_running.h" +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) #include "ps/ps_cache/ps_cache_manager.h" #endif @@ -114,6 +117,11 @@ void RunGraphTask::Run() { MS_LOG(ERROR) << "Invalid graph id " << graph_id_; return; } +#ifdef ENABLE_DUMP_IR + std::string tag = "run_graph"; + std::string file_type = ".ir;.pb"; + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type); +#endif graph->ResetGraphRunningStatus(); try { session_->RunGraphImpl(graph_id_, input_tensors_, &outputs_); diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 77008f3283..2974b93ca5 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019-2020 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,7 +39,9 @@ #include "debug/anf_ir_dump.h" #include "mindspore/core/base/base_ref_utils.h" #include "utils/trace_base.h" - +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) #include "ps/ps_cache/ps_cache_manager.h" #include "ps/common.h" @@ -1089,6 +1091,11 @@ std::shared_ptr SessionBasic::ConstructKernelGraph(const FuncGraphP auto node_list = TopoSort(func_graph->get_return()); auto graph = NewKernelGraph(); MS_EXCEPTION_IF_NULL(graph); +#ifdef ENABLE_DUMP_IR + std::string tag = "constructed_kernel_graph"; + std::string file_type = ".ir;.pb"; + mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type); +#endif front_backend_graph_map_[func_graph] = graph; MS_LOG(INFO) << "Create graph: " << graph->graph_id(); for (const auto &node : node_list) { diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 2d860a732b..2a262e27f3 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -5,6 +5,11 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc" "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc" "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc" ) if(ENABLE_DEBUGGER) diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.h b/mindspore/ccsrc/debug/rdr/base_recorder.h new file mode 100644 index 0000000000..76a88ffcc3 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/base_recorder.h @@ -0,0 +1,47 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_BASE_RECORDER_H_ +#define MINDSPORE_BASE_RECORDER_H_ + +#include +#include + +namespace mindspore { +class BaseRecorder { + public: + BaseRecorder() : module_(""), tag_(""), directory_("./rdr/"), filename_(""), timestamp_("") {} + BaseRecorder(const std::string &module, const std::string &tag) + : module_(module), tag_(tag), directory_("./rdr/"), filename_(""), timestamp_("") {} + ~BaseRecorder() {} + + std::string GetModule() { return module_; } + std::string GetTag() { return tag_; } + + void SetDirectory(const std::string &directory) { directory_ = directory; } + + virtual void Export() {} + + protected: + std::string module_; + std::string tag_; + std::string directory_; + std::string filename_; + std::string timestamp_; // year,month,day,hour,minute,second +}; + +using BaseRecorderPtr = std::shared_ptr; +} // namespace mindspore +#endif // MINDSPORE_BASE_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc new file mode 100644 index 0000000000..c52fbf1f77 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/graph_exec_order_recorder.h" +#include +#include +#include "mindspore/core/ir/anf.h" +#include "mindspore/core/utils/log_adapter.h" +#include "backend/session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace { +bool DumpGraphExeOrder(const std::string &filename, const std::vector &execution_order) { + std::ofstream fout(filename, std::ofstream::app); + if (!fout.is_open()) { + MS_LOG(WARNING) << "Open file for saving graph exec order failed."; + return false; + } + fout << "================== execution order ==================\n"; + fout << "execution_order size: " << execution_order.size() << "\n"; + int i = 0; + for (auto &cnode : execution_order) { + MS_EXCEPTION_IF_NULL(cnode); + fout << i << ":\n"; + fout << "\t" << cnode->DebugString() << "\n"; + fout << "\t" << AnfAlgo::GetStreamDistinctionLabel(cnode.get()) << "\n"; + fout << "\t" << AnfAlgo::GetGraphId(cnode.get()) << "\n"; + i++; + } + fout << "================== execution order ==================\n"; + fout.close(); + return true; +} +} // namespace + +void GraphExecOrderRecorder::Export() { + if (filename_.empty()) { + filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_ + ".txt"; + } + DumpGraphExeOrder(filename_, exec_order_); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h new file mode 100644 index 0000000000..652833a2b4 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h @@ -0,0 +1,42 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRAPH_EXEC_ORDER_RENDER_H_ +#define GRAPH_EXEC_ORDER_RENDER_H_ +#include +#include +#include + +#include "debug/rdr/base_recorder.h" + +namespace mindspore { +class CNode; +using CNodePtr = std::shared_ptr; +class GraphExecOrderRecorder : public BaseRecorder { + public: + GraphExecOrderRecorder() : BaseRecorder() {} + GraphExecOrderRecorder(const std::string &module, const std::string &tag, + const std::vector &final_exec_order) + : BaseRecorder(module, tag), exec_order_(final_exec_order) {} + void SetModule(const std::string &module) { module_ = module; } + void SetExecOrder(const std::vector &final_exec_order) { exec_order_ = final_exec_order; } + virtual void Export(); + + private: + std::vector exec_order_; +}; +using GraphExecOrderRecorderPtr = std::shared_ptr; +} // namespace mindspore +#endif // GRAPH_EXEC_ORDER_RENDER_H diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc new file mode 100644 index 0000000000..b61b93a3ab --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/graph_recorder.h" +#include "mindspore/core/base/base.h" +#include "mindspore/core/ir/func_graph.h" +#include "mindspore/core/utils/log_adapter.h" +#include "debug/anf_ir_dump.h" +#include "debug/anf_ir_utils.h" +#include "debug/dump_proto.h" + +namespace mindspore { +void GraphRecorder::Export() { + bool save_flag = false; + if (filename_.empty()) { + filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_; + } + if (graph_type_.find(".dat") != std::string::npos) { + save_flag = true; + ExportIR(filename_ + ".dat", std::to_string(id_), func_graph_); // saving *.dat file + } + if (graph_type_.find(".ir") != std::string::npos) { + save_flag = true; + DumpIR(filename_ + ".ir", func_graph_); // saving *.ir file + } + if (graph_type_.find(".pb") != std::string::npos) { + save_flag = true; + DumpIRProto(func_graph_, filename_); // save *.pb file + } + if (!save_flag) { + MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_; + } +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.h b/mindspore/ccsrc/debug/rdr/graph_recorder.h new file mode 100644 index 0000000000..2d50699f12 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.h @@ -0,0 +1,47 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_GRAPH_RECORDER_H_ +#define MINDSPORE_GRAPH_RECORDER_H_ + +#include +#include +#include + +#include "debug/rdr/base_recorder.h" +namespace mindspore { +class FuncGraph; +using FuncGraphPtr = std::shared_ptr; +class GraphRecorder : public BaseRecorder { + public: + GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {} + GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph, + const std::string &file_type, const int graph_id) + : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), id_(graph_id) {} + ~GraphRecorder() {} + void SetModule(const std::string &module) { module_ = module; } + void SetGraphType(const std::string &file_type) { graph_type_ = file_type; } + void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; } + void SetNodeId(int id) { id_ = id; } + virtual void Export(); + + private: + FuncGraphPtr func_graph_; + std::string graph_type_; + int id_{0}; +}; +using GraphRecorderPtr = std::shared_ptr; +} // namespace mindspore +#endif // MINDSPORE_GRAPH_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/recorder_manager.cc b/mindspore/ccsrc/debug/rdr/recorder_manager.cc new file mode 100644 index 0000000000..d2f640455e --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/recorder_manager.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/recorder_manager.h" +#include +#include "debug/rdr/base_recorder.h" +#include "mindspore/core/base/base.h" +#include "mindspore/core/ir/func_graph.h" + +namespace mindspore { +bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) { + if (recorder == nullptr) { + MS_LOG(ERROR) << "register recorder module with nullptr."; + return false; + } + std::string module = recorder->GetModule(); + std::lock_guard lock(mtx_); + recorder_container_[module].push_back(std::move(recorder)); + return true; +} + +void RecorderManager::TriggerAll() { + bool trigger = false; + std::lock_guard lock(mtx_); + for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) { + for (auto &recorder : iter->second) { + recorder->Export(); + trigger = true; + } + } + if (!trigger) { + MS_LOG(WARNING) << "There is no recorder to export."; + } +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/recorder_manager.h b/mindspore/ccsrc/debug/rdr/recorder_manager.h new file mode 100644 index 0000000000..632bcc521e --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/recorder_manager.h @@ -0,0 +1,49 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_RECORDER_MANAGER_H_ +#define MINDSPORE_RECORDER_MANAGER_H_ +#include +#include +#include +#include +#include +#include +#include + +namespace mindspore { +class BaseRecorder; +using BaseRecorderPtr = std::shared_ptr; +using BaseRecorderPtrList = std::vector; +class RecorderManager { + public: + static RecorderManager &Instance() { + static RecorderManager manager; + return manager; + } + + bool RecordObject(const BaseRecorderPtr &recorder); + void TriggerAll(); + + private: + RecorderManager() {} + ~RecorderManager() {} + + mutable std::mutex mtx_; + // module, BaserRecorderPtrList + std::unordered_map recorder_container_; +}; +} // namespace mindspore +#endif // MINDSPORE_RECORDER_MANAGER_H diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc new file mode 100644 index 0000000000..b9c1a406f3 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -0,0 +1,129 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/running_data_recorder.h" +#include +#include "debug/rdr/graph_recorder.h" +#include "debug/rdr/somas_recorder.h" +#include "debug/rdr/graph_exec_order_recorder.h" +#include "debug/rdr/recorder_manager.h" +#include "mindspore/core/ir/func_graph.h" +#include "mindspore/core/ir/anf.h" + +namespace mindspore { +namespace { +static const char *GetSubModuleName(SubModuleId module_id) { + static const char *sub_module_names[NUM_SUBMODUES] = { + "UNKNOWN", // SM_UNKNOWN + "CORE", // SM_CORE + "ANALYZER", // SM_ANALYZER + "COMMON", // SM_COMMON + "DEBUG", // SM_DEBUG + "DEVICE", // SM_DEVICE + "GE_ADPT", // SM_GE_ADPT + "IR", // SM_IR + "KERNEL", // SM_KERNEL + "MD", // SM_MD + "ME", // SM_ME + "EXPRESS", // SM_EXPRESS + "OPTIMIZER", // SM_OPTIMIZER + "PARALLEL", // SM_PARALLEL + "PARSER", // SM_PARSER + "PIPELINE", // SM_PIPELINE + "PRE_ACT", // SM_PRE_ACT + "PYNATIVE", // SM_PYNATIVE + "SESSION", // SM_SESSION + "UTILS", // SM_UTILS + "VM", // SM_VM + "PROFILER", // SM_PROFILER + "PS", // SM_PS + "LITE", // SM_LITE + "HCCL_ADPT" // SM_HCCL_ADPT + }; + + return sub_module_names[module_id % NUM_SUBMODUES]; +} +} // namespace +namespace RDR { +#ifdef __linux__ +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, + const std::string &file_type, int graph_id) { + std::string submodule_name = std::string(GetSubModuleName(module)); + GraphRecorderPtr graph_recorder = std::make_shared(submodule_name, tag, graph, file_type, graph_id); + bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder)); + return ans; +} + +bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, + const std::vector &&final_exec_order) { + std::string submodule_name = std::string(GetSubModuleName(module)); + GraphExecOrderRecorderPtr graph_exec_order_recorder = + std::make_shared(submodule_name, tag, final_exec_order); + bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder)); + return ans; +} + +bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { + std::string submodule_name = std::string(GetSubModuleName(module)); + SomasRecorderPtr somas_recorder = std::make_shared(submodule_name, tag, somas_ptr, graph_id); + somas_recorder->GenString(); + bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder)); + return ans; +} + +void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } +#else +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, + const std::string &file_type, int graph_id) { + static bool already_printed = false; + std::string submodule_name = std::string(GetSubModuleName(module)); + if (already_printed) { + return false; + } + already_printed = true; + MS_LOG(WARNING) << "The RDR presently only support linux os " << submodule_name; + return false; +} + +bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std::vector &&final_exec_order) { + static bool already_printed = false; + if (already_printed) { + return false; + } + already_printed = true; + MS_LOG(WARNING) << "The RDR presently only support linux os."; + return false; +} + +bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { + static bool already_printed = false; + if (already_printed) { + return false; + } + already_printed = true; + MS_LOG(WARNING) << "The RDR presently only support linux os."; + return false; +} +void TriggerAll() { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The RDR presently only support linux os."; +} +#endif // __linux__ +} // namespace RDR +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.h b/mindspore/ccsrc/debug/rdr/running_data_recorder.h new file mode 100644 index 0000000000..36241b99fa --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.h @@ -0,0 +1,40 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNNING_DATA_RECORDER_H_ +#define RUNNING_DATA_RECORDER_H_ + +#include +#include +#include + +#include "backend/optimizer/somas/somas.h" +#include "mindspore/core/utils/log_adapter.h" +namespace mindspore { +class FuncGraph; +class CNode; +using FuncGraphPtr = std::shared_ptr; +using CNodePtr = std::shared_ptr; +using SomasPtr = std::shared_ptr; +namespace RDR { +bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, + const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0); +bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, + const std::vector &&final_exec_order); +bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id); +void TriggerAll(); +} // namespace RDR +} // namespace mindspore +#endif // RUNNING_DATA_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/somas_recorder.cc b/mindspore/ccsrc/debug/rdr/somas_recorder.cc new file mode 100644 index 0000000000..9b0b4cbc69 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/somas_recorder.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/somas_recorder.h" +#include "backend/optimizer/somas/somas.h" +namespace mindspore { +void SomasRecorder::Export() { + if (filename_.empty()) { + filename_ = directory_ + module_ + "_" + tag_; + } + std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; + somas_reuse_util_ptr_->DumpSomasInfoIR(filename); + std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; + somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename); +} + +bool SomasRecorder::GenString() { return true; } +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/somas_recorder.h b/mindspore/ccsrc/debug/rdr/somas_recorder.h new file mode 100644 index 0000000000..32fb3f8a53 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/somas_recorder.h @@ -0,0 +1,46 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SOMAS_RECORDER_H_ +#define SOMAS_RECORDER_H_ + +#include +#include +#include +#include "debug/rdr/base_recorder.h" + +namespace mindspore { +namespace somas { +class Somas; +using SomasPtr = std::shared_ptr; +} // namespace somas +class SomasRecorder : public BaseRecorder { + public: + SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {} + SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr, + int graph_id) + : BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {} + void SetModule(const std::string &module) { module_ = module; } + void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; } + bool GenString(); + virtual void Export(); + + private: + somas::SomasPtr somas_reuse_util_ptr_; + int graph_id_{0}; +}; +using SomasRecorderPtr = std::shared_ptr; +} // namespace mindspore +#endif // SOMAS_RECORDER_H