From 1cfcfccd71fcd099053f4a0fdef4449977511cc4 Mon Sep 17 00:00:00 2001 From: caifubi Date: Wed, 29 Apr 2020 15:18:30 +0800 Subject: [PATCH] Add debug log for task name and task id --- .../device/ascend/ascend_kernel_runtime.cc | 19 ++++++++++++++++++- .../device/ascend/ascend_kernel_runtime.h | 1 + .../device/ascend/profiling/profiling_utils.h | 2 ++ .../device/ascend/tasksink/task_generator.cc | 4 +--- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index 10517299cc..d9b3e6ebe4 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -343,6 +343,22 @@ bool AscendKernelRuntime::LoadTask(const session::KernelGraph *graph) { return true; } +void AscendKernelRuntime::DebugTaskIdName(GraphId graph_id) { + auto task_ids = ge::model_runner::ModelRunner::Instance().GetTaskIdList(graph_id); + auto graph_task_names = ProfilingUtils::graph_kernel_name(); + auto iter = graph_task_names.find(graph_id); + if (iter != graph_task_names.end()) { + const auto &task_names = iter->second; + if (task_ids.size() != task_names.size()) { + MS_LOG(WARNING) << "Task_ids and task_names size not match"; + return; + } + for (size_t i = 0; i < task_ids.size(); ++i) { + MS_LOG(INFO) << "Task_id:" << task_ids[i] << " task_name:" << task_names[i]; + } + } +} + bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); MS_LOG(INFO) << "RunTask start. GraphId:" << graph->graph_id(); @@ -363,7 +379,8 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { bool status = ge::model_runner::ModelRunner::Instance().RunModel(graph->graph_id(), input_tensors, output_tensors); if (!status) { - MS_LOG(INFO) << "run task failed"; + MS_LOG(ERROR) << "run task failed"; + DebugTaskIdName(graph->graph_id()); return false; } return true; diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h index 5d0f61d0a6..920e28cb87 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h @@ -57,6 +57,7 @@ class AscendKernelRuntime : public KernelRuntime { void ReleaseDeviceRes() override; bool GraphWithEmptyTaskList(const session::KernelGraph *graph) const; bool CheckGraphIdValid(GraphId graph_id) const; + static void DebugTaskIdName(GraphId graph_id) ; rtContext_t rt_context_{nullptr}; bool initialized_{false}; diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h index 59909c1f2f..6986eaab54 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h @@ -101,6 +101,8 @@ class ProfilingUtils { NotNull graph_ptr, NotNull *> kernel_list); + static std::unordered_map> graph_kernel_name() { return graph_kernel_name_; } + inline static constexpr char kProfiling[] = "Profiling"; inline static constexpr char kNotify[] = "notify"; inline static constexpr char kProfilerTraceId[] = "profiler_trace_id"; diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc index 7b2a7dad9f..bdcc178b54 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc @@ -147,9 +147,7 @@ bool TaskGenerator::LaunchAllKernel(const std::vector &anf_node_list, } current_op_index++; } - if (ProfilingManager::GetInstance().IsProfiling()) { - ProfilingUtils::SetGraphKernelName(graph_id, kernel_name_list); - } + ProfilingUtils::SetGraphKernelName(graph_id, kernel_name_list); return true; } } // namespace tasksink