!4277 Profiling Support Multi Graph

Merge pull request !4277 from caifubi/profiling

mindspore/ccsrc/runtime/device/ascend/profiling/profiling_utils.cc

@@ -167,7 +167,8 @@ std::string ProfilingUtils::GetGraphLastTbeKernelName(const std::vector<CNodePtr
   std::string last_tbe_kernel_name;
   // find last tbe_kernel
   for (auto iter = cnode_exec_order.rbegin(); iter != cnode_exec_order.rend(); ++iter) {
-    if (AnfAlgo::GetKernelType(*iter) == TBE_KERNEL || AnfAlgo::GetKernelType(*iter) == AKG_KERNEL) {
+    if (AnfAlgo::GetKernelType(*iter) == TBE_KERNEL || AnfAlgo::GetKernelType(*iter) == AKG_KERNEL ||
+        AnfAlgo::IsCommunicationOp(*iter)) {
       last_tbe_kernel_name = (*iter)->fullname_with_scope();
       break;
     }

mindspore/ccsrc/runtime/device/kernel_adjust.cc

@@ -35,6 +35,9 @@
 #include "runtime/base.h"
 #include "runtime/device/ascend/ascend_stream_assign.h"
 
+namespace {
+constexpr auto kProfilingGraphId = "PROFILING_GRAPH_ID";
+}  // namespace
 namespace mindspore {
 namespace device {
 using device::ascend::ProfilingUtils;
@@ -608,6 +611,15 @@ void KernelAdjust::Profiling(NotNull<session::KernelGraph *> kernel_graph_ptr) {
     MS_LOG(INFO) << "No need to profiling";
     return;
   }
+  auto graph_id_env = std::getenv(kProfilingGraphId);
+  if (graph_id_env != nullptr) {
+    auto graph_id = std::stoul(graph_id_env);
+    if (graph_id != kernel_graph_ptr->graph_id()) {
+      MS_LOG(WARNING) << "Get PROFILING_GRAPH_ID " << graph_id
+                      << " Not Match Current Graph Id:" << kernel_graph_ptr->graph_id();
+      return;
+    }
+  }
   ProfilingTraceInfo profiling_trace_info = ProfilingUtils::GetProfilingTraceFromEnv(kernel_graph_ptr);
   if (!profiling_trace_info.IsValid()) {
     MS_LOG(WARNING) << "[profiling] no profiling node found!";