!11461 Add RDR module for debugging run task error

From: @louie5
Reviewed-by: @ouwenchang,@ouwenchang
Signed-off-by:

mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -122,7 +122,9 @@
 #include "utils/config_manager.h"
 #include "debug/anf_ir_dump.h"
 #include "debug/dump_proto.h"
-
+#ifdef ENABLE_DUMP_IR
+#include "debug/rdr/running_data_recorder.h"
+#endif
 namespace mindspore {
 namespace opt {
 namespace {
@@ -262,6 +264,10 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr<session::KernelGrap
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
+#ifdef ENABLE_DUMP_IR
+  std::string tag = "before_hwopt";
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
+#endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
     DumpIR(file_name, kernel_graph);
@@ -380,7 +386,13 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern
   optimizer2->AddPassManager(other2_pm);
   (void)optimizer2->Optimize(kernel_graph);
   kernel_graph->SetExecOrderByDefault();
-
+#ifdef ENABLE_DUMP_IR
+  std::string tag = "hwopt_d_end";
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph);
+  const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order();
+  tag = "graph_exec_order";
+  mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(exec_order));
+#endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
     DumpIR(file_name, kernel_graph, true, kTopStack);

mindspore/ccsrc/backend/session/ascend_session.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -62,6 +62,9 @@
 #include "debug/anf_ir_dump.h"
 #include "debug/dump_proto.h"
 #include "toolchain/adx_datadump_server.h"
+#ifdef ENABLE_DUMP_IR
+#include "debug/rdr/running_data_recorder.h"
+#endif
 #if ENABLE_CPU && ENABLE_D
 #include "ps/util.h"
 #include "ps/ps_cache/ps_cache_manager.h"
@@ -129,7 +132,6 @@ std::vector<CNodePtr> GetCNodes(const std::vector<AnfNodePtr> &anf_nodes) {
   }
   return cnodes;
 }
-
 void InsertMakeTupleForOutput(NotNull<KernelGraphPtr> root_graph) {
   auto return_node = root_graph->get_return();
   MS_EXCEPTION_IF_NULL(return_node);
@@ -715,7 +717,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
     InitRuntimeResource();
     return root_graph->graph_id();
   }
-
   // create parameter for multiple branch
   std::set<KernelGraphPtr> memo;
   CreateMultiBranchOutput(NOT_NULL(root_graph), NOT_NULL(&memo));
@@ -726,10 +727,8 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
   MultiCallGraphOptimize(NOT_NULL(root_graph));
   // resource initialize
   InitRuntimeResource();
-
   IrFusionPass(NOT_NULL(root_graph), NOT_NULL(&memo));
   memo.clear();
-
   SelectKernel(NOT_NULL(root_graph));
   memo.clear();
 
@@ -740,7 +739,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
     LoadGraphsToDbg(NOT_NULL(root_graph), NOT_NULL(&memo));
   }
   memo.clear();
-
   UpdateRefOutputMap(NOT_NULL(root_graph), NOT_NULL(&memo));
   memo.clear();
   // add make_tuple to the output graph
@@ -757,7 +755,6 @@ GraphId AscendSession::CompileGraphImpl(NotNull<FuncGraphPtr> func_graph) {
 
   // adjust kernel
   AdjustKernel(root_graph);
-
   // reorder send/recv
   auto execution_order = root_graph->execution_order();
   ReorderSendRecv(&execution_order);
@@ -1333,6 +1330,9 @@ void AscendSession::Execute(const std::shared_ptr<KernelGraph> &kernel_graph, bo
   bool ret_ok = runtime_instance->Run(kernel_graph.get(), is_task_sink);
   Dump(kernel_graph);
   if (!ret_ok) {
+#ifdef ENABLE_DUMP_IR
+    mindspore::RDR::TriggerAll();
+#endif
     MS_LOG(EXCEPTION) << "run task error!";
   }
   MS_LOG(INFO) << "Finish!";

mindspore/ccsrc/backend/session/executor.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,9 @@
 #include "runtime/device/kernel_runtime_manager.h"
 #include "utils/comm_manager.h"
 #include "utils/scoped_long_running.h"
+#ifdef ENABLE_DUMP_IR
+#include "debug/rdr/running_data_recorder.h"
+#endif
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/ps_cache/ps_cache_manager.h"
 #endif
@@ -114,6 +117,11 @@ void RunGraphTask::Run() {
     MS_LOG(ERROR) << "Invalid graph id " << graph_id_;
     return;
   }
+#ifdef ENABLE_DUMP_IR
+  std::string tag = "run_graph";
+  std::string file_type = ".ir;.pb";
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
+#endif
   graph->ResetGraphRunningStatus();
   try {
     session_->RunGraphImpl(graph_id_, input_tensors_, &outputs_);

mindspore/ccsrc/backend/session/session_basic.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -39,7 +39,9 @@
 #include "debug/anf_ir_dump.h"
 #include "mindspore/core/base/base_ref_utils.h"
 #include "utils/trace_base.h"
-
+#ifdef ENABLE_DUMP_IR
+#include "debug/rdr/running_data_recorder.h"
+#endif
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/ps_cache/ps_cache_manager.h"
 #include "ps/common.h"
@@ -1089,6 +1091,11 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
   auto node_list = TopoSort(func_graph->get_return());
   auto graph = NewKernelGraph();
   MS_EXCEPTION_IF_NULL(graph);
+#ifdef ENABLE_DUMP_IR
+  std::string tag = "constructed_kernel_graph";
+  std::string file_type = ".ir;.pb";
+  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, file_type);
+#endif
   front_backend_graph_map_[func_graph] = graph;
   MS_LOG(INFO) << "Create graph: " << graph->graph_id();
   for (const auto &node : node_list) {

mindspore/ccsrc/debug/CMakeLists.txt

@@ -5,6 +5,11 @@ set(_DEBUG_SRC_LIST
     "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc"
 )
 
 if(ENABLE_DEBUGGER)

mindspore/ccsrc/debug/rdr/base_recorder.h

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_BASE_RECORDER_H_
+#define MINDSPORE_BASE_RECORDER_H_
+
+#include <memory>
+#include <string>
+
+namespace mindspore {
+class BaseRecorder {
+ public:
+  BaseRecorder() : module_(""), tag_(""), directory_("./rdr/"), filename_(""), timestamp_("") {}
+  BaseRecorder(const std::string &module, const std::string &tag)
+      : module_(module), tag_(tag), directory_("./rdr/"), filename_(""), timestamp_("") {}
+  ~BaseRecorder() {}
+
+  std::string GetModule() { return module_; }
+  std::string GetTag() { return tag_; }
+
+  void SetDirectory(const std::string &directory) { directory_ = directory; }
+
+  virtual void Export() {}
+
+ protected:
+  std::string module_;
+  std::string tag_;
+  std::string directory_;
+  std::string filename_;
+  std::string timestamp_;  // year,month,day,hour,minute,second
+};
+
+using BaseRecorderPtr = std::shared_ptr<BaseRecorder>;
+}  // namespace mindspore
+#endif  // MINDSPORE_BASE_RECORDER_H_

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/graph_exec_order_recorder.h"
+#include <sstream>
+#include <fstream>
+#include "mindspore/core/ir/anf.h"
+#include "mindspore/core/utils/log_adapter.h"
+#include "backend/session/anf_runtime_algorithm.h"
+
+namespace mindspore {
+namespace {
+bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr> &execution_order) {
+  std::ofstream fout(filename, std::ofstream::app);
+  if (!fout.is_open()) {
+    MS_LOG(WARNING) << "Open file for saving graph exec order failed.";
+    return false;
+  }
+  fout << "================== execution order ==================\n";
+  fout << "execution_order size: " << execution_order.size() << "\n";
+  int i = 0;
+  for (auto &cnode : execution_order) {
+    MS_EXCEPTION_IF_NULL(cnode);
+    fout << i << ":\n";
+    fout << "\t" << cnode->DebugString() << "\n";
+    fout << "\t" << AnfAlgo::GetStreamDistinctionLabel(cnode.get()) << "\n";
+    fout << "\t" << AnfAlgo::GetGraphId(cnode.get()) << "\n";
+    i++;
+  }
+  fout << "================== execution order ==================\n";
+  fout.close();
+  return true;
+}
+}  // namespace
+
+void GraphExecOrderRecorder::Export() {
+  if (filename_.empty()) {
+    filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_ + ".txt";
+  }
+  DumpGraphExeOrder(filename_, exec_order_);
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h

@@ -0,0 +1,42 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef GRAPH_EXEC_ORDER_RENDER_H_
+#define GRAPH_EXEC_ORDER_RENDER_H_
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "debug/rdr/base_recorder.h"
+
+namespace mindspore {
+class CNode;
+using CNodePtr = std::shared_ptr<CNode>;
+class GraphExecOrderRecorder : public BaseRecorder {
+ public:
+  GraphExecOrderRecorder() : BaseRecorder() {}
+  GraphExecOrderRecorder(const std::string &module, const std::string &tag,
+                         const std::vector<CNodePtr> &final_exec_order)
+      : BaseRecorder(module, tag), exec_order_(final_exec_order) {}
+  void SetModule(const std::string &module) { module_ = module; }
+  void SetExecOrder(const std::vector<CNodePtr> &final_exec_order) { exec_order_ = final_exec_order; }
+  virtual void Export();
+
+ private:
+  std::vector<CNodePtr> exec_order_;
+};
+using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>;
+}  // namespace mindspore
+#endif  // GRAPH_EXEC_ORDER_RENDER_H

mindspore/ccsrc/debug/rdr/graph_recorder.cc

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/graph_recorder.h"
+#include "mindspore/core/base/base.h"
+#include "mindspore/core/ir/func_graph.h"
+#include "mindspore/core/utils/log_adapter.h"
+#include "debug/anf_ir_dump.h"
+#include "debug/anf_ir_utils.h"
+#include "debug/dump_proto.h"
+
+namespace mindspore {
+void GraphRecorder::Export() {
+  bool save_flag = false;
+  if (filename_.empty()) {
+    filename_ = directory_ + module_ + "_" + tag_ + "_" + timestamp_;
+  }
+  if (graph_type_.find(".dat") != std::string::npos) {
+    save_flag = true;
+    ExportIR(filename_ + ".dat", std::to_string(id_), func_graph_);  // saving  *.dat file
+  }
+  if (graph_type_.find(".ir") != std::string::npos) {
+    save_flag = true;
+    DumpIR(filename_ + ".ir", func_graph_);  // saving *.ir file
+  }
+  if (graph_type_.find(".pb") != std::string::npos) {
+    save_flag = true;
+    DumpIRProto(func_graph_, filename_);  // save *.pb file
+  }
+  if (!save_flag) {
+    MS_LOG(WARNING) << "Unknown save graph type: " << graph_type_;
+  }
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_recorder.h

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_GRAPH_RECORDER_H_
+#define MINDSPORE_GRAPH_RECORDER_H_
+
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "debug/rdr/base_recorder.h"
+namespace mindspore {
+class FuncGraph;
+using FuncGraphPtr = std::shared_ptr<FuncGraph>;
+class GraphRecorder : public BaseRecorder {
+ public:
+  GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {}
+  GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph,
+                const std::string &file_type, const int graph_id)
+      : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), id_(graph_id) {}
+  ~GraphRecorder() {}
+  void SetModule(const std::string &module) { module_ = module; }
+  void SetGraphType(const std::string &file_type) { graph_type_ = file_type; }
+  void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; }
+  void SetNodeId(int id) { id_ = id; }
+  virtual void Export();
+
+ private:
+  FuncGraphPtr func_graph_;
+  std::string graph_type_;
+  int id_{0};
+};
+using GraphRecorderPtr = std::shared_ptr<GraphRecorder>;
+}  // namespace mindspore
+#endif  // MINDSPORE_GRAPH_RECORDER_H_

mindspore/ccsrc/debug/rdr/recorder_manager.cc

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/recorder_manager.h"
+#include <utility>
+#include "debug/rdr/base_recorder.h"
+#include "mindspore/core/base/base.h"
+#include "mindspore/core/ir/func_graph.h"
+
+namespace mindspore {
+bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) {
+  if (recorder == nullptr) {
+    MS_LOG(ERROR) << "register recorder module with nullptr.";
+    return false;
+  }
+  std::string module = recorder->GetModule();
+  std::lock_guard<std::mutex> lock(mtx_);
+  recorder_container_[module].push_back(std::move(recorder));
+  return true;
+}
+
+void RecorderManager::TriggerAll() {
+  bool trigger = false;
+  std::lock_guard<std::mutex> lock(mtx_);
+  for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) {
+    for (auto &recorder : iter->second) {
+      recorder->Export();
+      trigger = true;
+    }
+  }
+  if (!trigger) {
+    MS_LOG(WARNING) << "There is no recorder to export.";
+  }
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/recorder_manager.h

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_RECORDER_MANAGER_H_
+#define MINDSPORE_RECORDER_MANAGER_H_
+#include <vector>
+#include <string>
+#include <sstream>
+#include <map>
+#include <unordered_map>
+#include <memory>
+#include <mutex>
+
+namespace mindspore {
+class BaseRecorder;
+using BaseRecorderPtr = std::shared_ptr<BaseRecorder>;
+using BaseRecorderPtrList = std::vector<BaseRecorderPtr>;
+class RecorderManager {
+ public:
+  static RecorderManager &Instance() {
+    static RecorderManager manager;
+    return manager;
+  }
+
+  bool RecordObject(const BaseRecorderPtr &recorder);
+  void TriggerAll();
+
+ private:
+  RecorderManager() {}
+  ~RecorderManager() {}
+
+  mutable std::mutex mtx_;
+  // module, BaserRecorderPtrList
+  std::unordered_map<std::string, BaseRecorderPtrList> recorder_container_;
+};
+}  // namespace mindspore
+#endif  // MINDSPORE_RECORDER_MANAGER_H

mindspore/ccsrc/debug/rdr/running_data_recorder.cc

@@ -0,0 +1,129 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/running_data_recorder.h"
+#include <utility>
+#include "debug/rdr/graph_recorder.h"
+#include "debug/rdr/somas_recorder.h"
+#include "debug/rdr/graph_exec_order_recorder.h"
+#include "debug/rdr/recorder_manager.h"
+#include "mindspore/core/ir/func_graph.h"
+#include "mindspore/core/ir/anf.h"
+
+namespace mindspore {
+namespace {
+static const char *GetSubModuleName(SubModuleId module_id) {
+  static const char *sub_module_names[NUM_SUBMODUES] = {
+    "UNKNOWN",    // SM_UNKNOWN
+    "CORE",       // SM_CORE
+    "ANALYZER",   // SM_ANALYZER
+    "COMMON",     // SM_COMMON
+    "DEBUG",      // SM_DEBUG
+    "DEVICE",     // SM_DEVICE
+    "GE_ADPT",    // SM_GE_ADPT
+    "IR",         // SM_IR
+    "KERNEL",     // SM_KERNEL
+    "MD",         // SM_MD
+    "ME",         // SM_ME
+    "EXPRESS",    // SM_EXPRESS
+    "OPTIMIZER",  // SM_OPTIMIZER
+    "PARALLEL",   // SM_PARALLEL
+    "PARSER",     // SM_PARSER
+    "PIPELINE",   // SM_PIPELINE
+    "PRE_ACT",    // SM_PRE_ACT
+    "PYNATIVE",   // SM_PYNATIVE
+    "SESSION",    // SM_SESSION
+    "UTILS",      // SM_UTILS
+    "VM",         // SM_VM
+    "PROFILER",   // SM_PROFILER
+    "PS",         // SM_PS
+    "LITE",       // SM_LITE
+    "HCCL_ADPT"   // SM_HCCL_ADPT
+  };
+
+  return sub_module_names[module_id % NUM_SUBMODUES];
+}
+}  // namespace
+namespace RDR {
+#ifdef __linux__
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+                    const std::string &file_type, int graph_id) {
+  std::string submodule_name = std::string(GetSubModuleName(module));
+  GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, tag, graph, file_type, graph_id);
+  bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder));
+  return ans;
+}
+
+bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
+                          const std::vector<CNodePtr> &&final_exec_order) {
+  std::string submodule_name = std::string(GetSubModuleName(module));
+  GraphExecOrderRecorderPtr graph_exec_order_recorder =
+    std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order);
+  bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder));
+  return ans;
+}
+
+bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
+  std::string submodule_name = std::string(GetSubModuleName(module));
+  SomasRecorderPtr somas_recorder = std::make_shared<SomasRecorder>(submodule_name, tag, somas_ptr, graph_id);
+  somas_recorder->GenString();
+  bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder));
+  return ans;
+}
+
+void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }
+#else
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+                    const std::string &file_type, int graph_id) {
+  static bool already_printed = false;
+  std::string submodule_name = std::string(GetSubModuleName(module));
+  if (already_printed) {
+    return false;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The RDR presently only support linux os " << submodule_name;
+  return false;
+}
+
+bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std::vector<CNodePtr> &&final_exec_order) {
+  static bool already_printed = false;
+  if (already_printed) {
+    return false;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The RDR presently only support linux os.";
+  return false;
+}
+
+bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) {
+  static bool already_printed = false;
+  if (already_printed) {
+    return false;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The RDR presently only support linux os.";
+  return false;
+}
+void TriggerAll() {
+  static bool already_printed = false;
+  if (already_printed) {
+    return;
+  }
+  already_printed = true;
+  MS_LOG(WARNING) << "The RDR presently only support linux os.";
+}
+#endif  // __linux__
+}  // namespace RDR
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/running_data_recorder.h

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RUNNING_DATA_RECORDER_H_
+#define RUNNING_DATA_RECORDER_H_
+
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "backend/optimizer/somas/somas.h"
+#include "mindspore/core/utils/log_adapter.h"
+namespace mindspore {
+class FuncGraph;
+class CNode;
+using FuncGraphPtr = std::shared_ptr<FuncGraph>;
+using CNodePtr = std::shared_ptr<CNode>;
+using SomasPtr = std::shared_ptr<somas::Somas>;
+namespace RDR {
+bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph,
+                    const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0);
+bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
+                          const std::vector<CNodePtr> &&final_exec_order);
+bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id);
+void TriggerAll();
+}  // namespace RDR
+}  // namespace mindspore
+#endif  // RUNNING_DATA_RECORDER_H_

mindspore/ccsrc/debug/rdr/somas_recorder.cc

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "debug/rdr/somas_recorder.h"
+#include "backend/optimizer/somas/somas.h"
+namespace mindspore {
+void SomasRecorder::Export() {
+  if (filename_.empty()) {
+    filename_ = directory_ + module_ + "_" + tag_;
+  }
+  std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
+  somas_reuse_util_ptr_->DumpSomasInfoIR(filename);
+  std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir";
+  somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename);
+}
+
+bool SomasRecorder::GenString() { return true; }
+}  // namespace mindspore

mindspore/ccsrc/debug/rdr/somas_recorder.h

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SOMAS_RECORDER_H_
+#define SOMAS_RECORDER_H_
+
+#include <string>
+#include <sstream>
+#include <memory>
+#include "debug/rdr/base_recorder.h"
+
+namespace mindspore {
+namespace somas {
+class Somas;
+using SomasPtr = std::shared_ptr<Somas>;
+}  // namespace somas
+class SomasRecorder : public BaseRecorder {
+ public:
+  SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {}
+  SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr,
+                int graph_id)
+      : BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {}
+  void SetModule(const std::string &module) { module_ = module; }
+  void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; }
+  bool GenString();
+  virtual void Export();
+
+ private:
+  somas::SomasPtr somas_reuse_util_ptr_;
+  int graph_id_{0};
+};
+using SomasRecorderPtr = std::shared_ptr<SomasRecorder>;
+}  // namespace mindspore
+#endif  // SOMAS_RECORDER_H