!12764 [Ascend][GPU] Add execution order dumping of final execution graphs

From: @islam_amin
Reviewed-by: @john_tzanakakis,@yelihua
Signed-off-by: @yelihua

mindspore/ccsrc/backend/session/ascend_session.cc

@@ -1003,16 +1003,21 @@ void AscendSession::DumpAllGraphs(const std::vector<KernelGraphPtr> &all_graphs)
     }
     std::string final_graph = "trace_code_graph_" + std::to_string(graph->graph_id());
     if (json_parser.e2e_dump_enabled()) {
-      std::string target_dir =
-        json_parser.path() + "/" + json_parser.net_name() + "/device_" + std::to_string(device_id) + "/graphs";
+      std::string root_dir = json_parser.path() + "/" + json_parser.net_name() + "/device_" + std::to_string(device_id);
+      std::string target_dir = root_dir + "/graphs";
       std::string ir_file_path = target_dir + "/" + "ms_output_" + final_graph + ".ir";
       DumpIRProtoWithSrcInfo(graph, final_graph, target_dir, kDebugWholeStack);
       DumpIR("trace_code_graph", graph, true, kWholeStack, ir_file_path);
+      DumpGraphExeOrder("ms_execution_order_" + std::to_string(graph->graph_id()) + ".csv", root_dir,
+                        graph->execution_order());
     } else if (json_parser.async_dump_enabled()) {
-      std::string target_dir = json_parser.path() + "/device_" + std::to_string(device_id) + "/graphs";
+      std::string root_dir = json_parser.path() + "/device_" + std::to_string(device_id);
+      std::string target_dir = root_dir + "/graphs";
       std::string ir_file_path = target_dir + "/" + "ms_output_" + final_graph + ".ir";
       DumpIRProtoWithSrcInfo(graph, final_graph, target_dir, kDebugWholeStack);
       DumpIR("trace_code_graph", graph, true, kWholeStack, ir_file_path);
+      DumpGraphExeOrder("ms_execution_order_" + std::to_string(graph->graph_id()) + ".csv", root_dir,
+                        graph->execution_order());
     }
   }
 #endif

mindspore/ccsrc/backend/session/gpu_session.cc

@@ -378,11 +378,13 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) {
     DumpIRProto(graph, "after_opt_" + std::to_string(graph->graph_id()));
   }
   if (json_parser.e2e_dump_enabled()) {
-    std::string target_dir =
-      json_parser.path() + "/" + json_parser.net_name() + "/device_" + std::to_string(device_id) + "/graphs";
+    std::string root_dir = json_parser.path() + "/" + json_parser.net_name() + "/device_" + std::to_string(device_id);
+    std::string target_dir = root_dir + "/graphs";
     std::string ir_file_path = target_dir + "/" + "after_opt_" + std::to_string(graph->graph_id()) + ".ir";
     DumpIRProtoWithSrcInfo(graph, "after_opt_" + std::to_string(graph->graph_id()), target_dir, kDebugWholeStack);
     DumpIR("trace_code_graph", graph, true, kWholeStack, ir_file_path);
+    DumpGraphExeOrder("ms_execution_order_" + std::to_string(graph->graph_id()) + ".csv", root_dir,
+                      graph->execution_order());
   }
   // Set graph manager.
   MS_EXCEPTION_IF_NULL(context_);

mindspore/ccsrc/backend/session/session_basic.cc

@@ -38,6 +38,7 @@
 #include "ir/func_graph_cloner.h"
 #include "utils/utils.h"
 #include "debug/anf_ir_dump.h"
+#include "debug/common.h"
 #include "utils/trace_base.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/ps_cache/ps_cache_manager.h"
@@ -2404,4 +2405,47 @@ void SessionBasic::InitPSParamAndOptim(const KernelGraphPtr &kernel_graph,
 }
 #endif
 }  // namespace session
+void DumpGraphExeOrder(const std::string &file_name, const std::string &target_dir,
+                       const std::vector<CNodePtr> &execution_order) {
+  bool status = Common::CreateNotExistDirs(target_dir + "/execution_order/");
+  std::string file_path = target_dir + "/execution_order/" + file_name;
+  if (!status) {
+    MS_LOG(ERROR) << "Failed at CreateNotExistDirs in DumpGraphExeOrder";
+    return;
+  }
+  if (file_path.size() > PATH_MAX) {
+    MS_LOG(ERROR) << "File path " << file_path << " is too long.";
+    return;
+  }
+  char real_path[PATH_MAX] = {0};
+  char *real_path_ret = nullptr;
+#if defined(_WIN32) || defined(_WIN64)
+  real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
+#else
+  real_path_ret = realpath(file_path.c_str(), real_path);
+#endif
+  if (nullptr == real_path_ret) {
+    MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
+  } else {
+    std::string path_string = real_path;
+    if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {
+      MS_LOG(ERROR) << "Modify file:" << real_path << " to rw fail.";
+      return;
+    }
+  }
+
+  // write to csv file
+  std::ofstream ofs(real_path);
+  if (!ofs.is_open()) {
+    MS_LOG(ERROR) << "Open file '" << real_path << "' failed!";
+    return;
+  }
+  ofs << "NodeExecutionOrder-FullNameWithScope\n";
+  for (const CNodePtr &node : execution_order) {
+    ofs << node->fullname_with_scope() << "\n";
+  }
+  ofs.close();
+  // set file mode to read only by user
+  ChangeFileMode(file_path, S_IRUSR);
+}
 }  // namespace mindspore

mindspore/ccsrc/backend/session/session_basic.h

@@ -250,5 +250,7 @@ class SessionBasic : public std::enable_shared_from_this<SessionBasic> {
 using SessionPtr = std::shared_ptr<session::SessionBasic>;
 using NamedSummaryOutputs = std::map<std::string, std::pair<AnfNodePtr, int>>;
 }  // namespace session
+void DumpGraphExeOrder(const std::string &file_name, const std::string &target_dir,
+                       const std::vector<CNodePtr> &execution_order);
 }  // namespace mindspore
 #endif  // MINDSPORE_CCSRC_BACKEND_SESSION_SESSION_BASIC_H