!11271 Fix the bug of step_trace cannot get the step_trace_point name in callback scene

From: @gzhcv
Reviewed-by: @ouwenchang,@lilongfei15
Signed-off-by: @lilongfei15

mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@ constexpr auto kInitDatasetQueueOpName = "InitDataSetQueue";
 
 bool ProfilingUtils::have_communication_op = false;
 ProfilingTraceInfo ProfilingUtils::profiling_trace = {"", "", ""};
+std::unordered_map<uint32_t, bool> ProfilingUtils::is_first_step_map_ = {};
 
 ProfilingTraceInfo ProfilingUtils::GetProfilingTraceFromEnv(NotNull<const session::KernelGraph *> graph_ptr) {
   MS_LOG(INFO) << "get current subgraph op name start.";
@@ -45,6 +46,7 @@ ProfilingTraceInfo ProfilingUtils::GetProfilingTraceFromEnv(NotNull<const sessio
   GetTraceHccl(cnode_exec_order);
 
   OutputStepTraceOpNameStatus();
+  is_first_step_map_[graph_ptr->graph_id()] = false;
   return profiling_trace;
 }
 
@@ -170,6 +172,14 @@ std::string ProfilingUtils::GetGraphSecondLastKernelName(const std::vector<CNode
 
   return second_last_kernel_name;
 }
+
+bool ProfilingUtils::IsFirstStep(const uint32_t graph_id) {
+  auto iter = is_first_step_map_.find(graph_id);
+  if (iter != is_first_step_map_.end()) {
+    is_first_step_map_[graph_id] = true;
+  }
+  return is_first_step_map_[graph_id];
+}
 }  // namespace gpu
 }  // namespace profiler
 }  // namespace mindspore

mindspore/ccsrc/profiler/device/gpu/gpu_profiling_utils.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -54,6 +54,7 @@ class ProfilingUtils {
   // export PROFILING_ITER_END='full name of last cnode in graph to execute'
   static ProfilingTraceInfo GetProfilingTraceFromEnv(NotNull<const session::KernelGraph *> graph_ptr);
   static void OutputStepTraceOpNameStatus();
+  static bool IsFirstStep(const uint32_t graph_id);
 
   static bool have_communication_op;
   static ProfilingTraceInfo profiling_trace;
@@ -64,6 +65,7 @@ class ProfilingUtils {
   static void SetTraceIterEnd(const std::vector<CNodePtr> &cnode_exec_order);
   static std::string GetGraphSecondLastKernelName(const std::vector<CNodePtr> &cnode_exec_order);
   static void GetTraceHccl(const std::vector<CNodePtr> &cnode_exec_order);
+  static std::unordered_map<uint32_t, bool> is_first_step_map_;
 };
 }  // namespace gpu
 }  // namespace profiler

mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -618,7 +618,7 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo
   auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance();
   MS_EXCEPTION_IF_NULL(profiler_inst);
 
-  if (profiler_inst->GetEnableFlag() && is_first_step_map_[graph->graph_id()]) {
+  if (profiler_inst->GetEnableFlag() && profiler::gpu::ProfilingUtils::IsFirstStep(graph->graph_id())) {
     profiler::gpu::ProfilingTraceInfo profiling_trace =
       profiler::gpu::ProfilingUtils::GetProfilingTraceFromEnv(NOT_NULL(graph));
     profiler_inst->SetStepTraceOpName(profiling_trace);