!12265 [MS][RDR] optimize saving FuncGraph

From: @louie5
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc

@@ -267,10 +267,6 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr<session::KernelGrap
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
-#ifdef ENABLE_DUMP_IR
-  std::string tag = "hwopt_before_graph";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, false, ".ir;.pb");
-#endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_ir_fusion_before_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";
     DumpIR(file_name, kernel_graph);
@@ -391,12 +387,9 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern
   (void)optimizer2->Optimize(kernel_graph);
   kernel_graph->SetExecOrderByDefault();
 #ifdef ENABLE_DUMP_IR
-  std::string tag = "hwopt_after_graph";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_OPTIMIZER, tag, kernel_graph, true, ".ir;.pb");
   const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order();
-  std::vector<CNodePtr> graph_exec_order(exec_order);
-  tag = "graph_exec_order";
-  mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, tag, std::move(graph_exec_order));
+  std::string exec_order_tag = "graph_exec_order";
+  mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, exec_order_tag, exec_order, kernel_graph->graph_id());
 #endif
   if (save_graphs) {
     std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";

mindspore/ccsrc/backend/session/ascend_session.cc

@@ -999,15 +999,16 @@ void AscendSession::DumpAllGraphs(const std::vector<KernelGraphPtr> &all_graphs)
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
-  if (!save_graphs) {
-    return;
-  }
   for (auto &graph : all_graphs) {
     MS_EXCEPTION_IF_NULL(graph);
-    std::string file_name = "graph_build_" + std::to_string(graph->graph_id()) + ".ir";
-    DumpIR(file_name, graph, true, kWholeStack);
-    DumpIRProto(graph, "vm_build_" + std::to_string(graph->graph_id()));
-    DumpIR("trace_code_graph", graph, true, kWholeStack);
+    std::string tag = "graph_build";
+    mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph, true, ".ir;.pb", graph->graph_id());
+    if (save_graphs) {
+      std::string file_name = "graph_build_" + std::to_string(graph->graph_id()) + ".ir";
+      DumpIR(file_name, graph, true, kWholeStack);
+      DumpIRProto(graph, "vm_build_" + std::to_string(graph->graph_id()));
+      DumpIR("trace_code_graph", graph, true, kWholeStack);
+    }
   }
 #endif
 }

mindspore/ccsrc/backend/session/executor.cc

@@ -20,9 +20,6 @@
 #include "runtime/device/kernel_runtime_manager.h"
 #include "utils/comm_manager.h"
 #include "utils/scoped_long_running.h"
-#ifdef ENABLE_DUMP_IR
-#include "debug/rdr/running_data_recorder.h"
-#endif
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/ps_cache/ps_cache_manager.h"
 #endif

mindspore/ccsrc/backend/session/session_basic.cc

@@ -39,9 +39,6 @@
 #include "utils/utils.h"
 #include "debug/anf_ir_dump.h"
 #include "utils/trace_base.h"
-#ifdef ENABLE_DUMP_IR
-#include "debug/rdr/running_data_recorder.h"
-#endif
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/ps_cache/ps_cache_manager.h"
 #include "ps/common.h"
@@ -1449,11 +1446,6 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
   auto node_list = TopoSort(func_graph->get_return());
   auto graph = NewKernelGraph();
   MS_EXCEPTION_IF_NULL(graph);
-#ifdef ENABLE_DUMP_IR
-  std::string tag = "constructed_kernel_graph";
-  std::string file_type = ".ir;.pb";
-  mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, file_type);
-#endif
   front_backend_graph_map_[func_graph] = graph;
   MS_LOG(INFO) << "Create graph: " << graph->graph_id();
   for (const auto &node : node_list) {

mindspore/ccsrc/debug/rdr/base_recorder.h

@@ -57,7 +57,7 @@ class BaseRecorder {
       timestamp_ = ss.str();
     }
   }
-  ~BaseRecorder() {}
+  virtual ~BaseRecorder() {}
 
   std::string GetModule() const { return module_; }
   std::string GetTag() const { return tag_; }

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc

@@ -50,7 +50,7 @@ void GraphExecOrderRecorder::Export() {
   if (!realpath.has_value()) {
     return;
   }
-  std::string real_file_path = realpath.value() + ".txt";
+  std::string real_file_path = realpath.value() + std::to_string(graph_id_);
   DumpGraphExeOrder(real_file_path, exec_order_);
 }
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h

@@ -28,14 +28,15 @@ class GraphExecOrderRecorder : public BaseRecorder {
  public:
   GraphExecOrderRecorder() : BaseRecorder() {}
   GraphExecOrderRecorder(const std::string &module, const std::string &tag,
-                         const std::vector<CNodePtr> &final_exec_order)
-      : BaseRecorder(module, tag), exec_order_(final_exec_order) {}
+                         const std::vector<CNodePtr> &final_exec_order, int graph_id)
+      : BaseRecorder(module, tag), exec_order_(final_exec_order), graph_id_(graph_id) {}
   void SetModule(const std::string &module) { module_ = module; }
   void SetExecOrder(const std::vector<CNodePtr> &final_exec_order) { exec_order_ = final_exec_order; }
   virtual void Export();
 
  private:
   std::vector<CNodePtr> exec_order_;
+  int graph_id_;
 };
 using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>;
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/graph_recorder.cc

@@ -63,11 +63,13 @@ void GraphRecorder::Export() {
     return;
   }
 
-  std::string realpath = tmp_realpath.value() + std::to_string(id_);
-
+  std::string realpath = tmp_realpath.value();
+  if (graph_id_ >= 0) {
+    realpath += "_" + std::to_string(graph_id_);
+  }
   if (graph_type_.find(".dat") != std::string::npos) {
     save_flag = true;
-    AnfExporter exporter(std::to_string(id_));
+    AnfExporter exporter("");
     std::string realpath_dat = realpath + ".dat";
     ChangeFileMode(realpath_dat, S_IRWXU);
     exporter.ExportFuncGraph(realpath_dat, func_graph_);

mindspore/ccsrc/debug/rdr/graph_recorder.h

@@ -29,21 +29,21 @@ class GraphRecorder : public BaseRecorder {
  public:
   GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {}
   GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph,
-                const std::string &file_type, const int graph_id)
-      : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), id_(graph_id) {}
+                const std::string &file_type, int graph_id)
+      : BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type), graph_id_(graph_id) {}
   ~GraphRecorder() {}
   void SetModule(const std::string &module) { module_ = module; }
   void SetGraphType(const std::string &file_type) { graph_type_ = file_type; }
   void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; }
   void SetDumpFlag(bool full_name) { full_name_ = full_name; }
-  void SetNodeId(int id) { id_ = id; }
+
   virtual void Export();
 
  private:
   FuncGraphPtr func_graph_;
   std::string graph_type_;
+  int graph_id_;
   bool full_name_{false};
-  int id_{0};
 };
 using GraphRecorderPtr = std::shared_ptr<GraphRecorder>;
 }  // namespace mindspore

mindspore/ccsrc/debug/rdr/running_data_recorder.cc

@@ -69,10 +69,10 @@ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const Func
 }
 
 bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
-                          const std::vector<CNodePtr> &&final_exec_order) {
+                          const std::vector<CNodePtr> &final_exec_order, int graph_id) {
   std::string submodule_name = std::string(GetSubModuleName(module));
   GraphExecOrderRecorderPtr graph_exec_order_recorder =
-    std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order);
+    std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order, graph_id);
   bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder));
   return ans;
 }
@@ -100,7 +100,8 @@ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const Func
   return false;
 }
 
-bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std::vector<CNodePtr> &&final_exec_order) {
+bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
+                          const std::vector<CNodePtr> &final_exec_order, int graph_id) {
   static bool already_printed = false;
   if (already_printed) {
     return false;

mindspore/ccsrc/debug/rdr/running_data_recorder.h

@@ -28,9 +28,9 @@ using FuncGraphPtr = std::shared_ptr<FuncGraph>;
 using CNodePtr = std::shared_ptr<CNode>;
 namespace RDR {
 bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
-                    const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0);
+                    const std::string &file_type = ".ir;.pb;.dat", int graph_id = -1);
 bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
-                          const std::vector<CNodePtr> &&final_exec_order);
+                          const std::vector<CNodePtr> &final_exec_order, int graph_id = 0);
 bool RecordString(SubModuleId module, const std::string &tag, const std::string &data,
                   const std::string &filename = "");
 void TriggerAll();