diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index 7789a62a3c..16bde676b7 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -984,9 +984,6 @@ void AscendSession::Execute(const std::shared_ptr &kernel_graph, bo #endif MS_LOG(EXCEPTION) << "run task error!"; } -#ifdef ENABLE_DUMP_IR - mindspore::RDR::ClearAll(); -#endif MS_LOG(INFO) << "Finish!"; } diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 78cb7fe684..a813dca080 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -9,6 +9,9 @@ set(_DEBUG_SRC_LIST ) if(ENABLE_DUMP_IR) + if(ENABLE_D) + list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/rdr/task_debug_info_recorder.cc") + endif() list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/rdr/base_recorder.cc" "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" @@ -29,7 +32,6 @@ if(ENABLE_DEBUGGER) "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" ) endif() - if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/common.cc") list(APPEND _DEBUG_SRC_LIST "data_dump/dump_json_parser.cc") diff --git a/mindspore/ccsrc/debug/rdr/base_recorder.h b/mindspore/ccsrc/debug/rdr/base_recorder.h index 6f84c1ad19..300b116e8a 100644 --- a/mindspore/ccsrc/debug/rdr/base_recorder.h +++ b/mindspore/ccsrc/debug/rdr/base_recorder.h @@ -15,7 +15,6 @@ */ #ifndef MINDSPORE_CCSRC_DEBUG_RDR_BASE_RECORDER_H_ #define MINDSPORE_CCSRC_DEBUG_RDR_BASE_RECORDER_H_ - #include #include #include @@ -75,7 +74,6 @@ class BaseRecorder { std::string filename_; std::string timestamp_; // year,month,day,hour,minute,second }; - using BaseRecorderPtr = std::shared_ptr; } // namespace mindspore #endif // MINDSPORE_CCSRC_DEBUG_RDR_BASE_RECORDER_H_ diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.cc b/mindspore/ccsrc/debug/rdr/graph_recorder.cc index 109665d44e..a71d25c0ba 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.cc @@ -86,7 +86,6 @@ void GraphRecorder::Export() { } if (graph_type_.find(".pb") != std::string::npos) { save_flag = true; - protobuf::DumpIRProto(realpath + ".pb", func_graph_); // save *.pb file } if (!save_flag) { diff --git a/mindspore/ccsrc/debug/rdr/graph_recorder.h b/mindspore/ccsrc/debug/rdr/graph_recorder.h index e811142e8c..ba8a382cf2 100644 --- a/mindspore/ccsrc/debug/rdr/graph_recorder.h +++ b/mindspore/ccsrc/debug/rdr/graph_recorder.h @@ -15,7 +15,6 @@ */ #ifndef MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_RECORDER_H_ #define MINDSPORE_CCSRC_DEBUG_RDR_GRAPH_RECORDER_H_ - #include #include #include diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc index a88ffcba50..70a42ff03d 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.cc +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.cc @@ -22,7 +22,10 @@ #include "debug/rdr/stream_exec_order_recorder.h" #include "mindspore/core/ir/func_graph.h" #include "mindspore/core/ir/anf.h" - +#ifdef ENABLE_D +#include "runtime/device/ascend/tasksink/task_generator.h" +#include "debug/rdr/task_debug_info_recorder.h" +#endif // ENABLE_D namespace mindspore { namespace { static const char *GetSubModuleName(SubModuleId module_id) { @@ -59,6 +62,17 @@ static const char *GetSubModuleName(SubModuleId module_id) { } } // namespace namespace RDR { +#ifdef ENABLE_D +bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, + const std::vector &task_debug_info_list, int graph_id) { + std::string submodule_name = std::string(GetSubModuleName(module)); + TaskDebugInfoRecorderPtr task_debug_info_recorder = + std::make_shared(submodule_name, tag, task_debug_info_list, graph_id); + bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(task_debug_info_recorder)); + return ans; +} +#endif // ENABLE_D + #ifdef __linux__ bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type, int graph_id) { @@ -152,6 +166,15 @@ void TriggerAll() { already_printed = true; MS_LOG(WARNING) << "The RDR presently only support linux os."; } + +void ClearAll() { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The RDR presently only support linux os."; +} #endif // __linux__ } // namespace RDR } // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/running_data_recorder.h b/mindspore/ccsrc/debug/rdr/running_data_recorder.h index e65a6a6390..3233c271ce 100644 --- a/mindspore/ccsrc/debug/rdr/running_data_recorder.h +++ b/mindspore/ccsrc/debug/rdr/running_data_recorder.h @@ -15,7 +15,6 @@ */ #ifndef MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ #define MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ - #include #include #include @@ -26,6 +25,16 @@ class FuncGraph; class CNode; using FuncGraphPtr = std::shared_ptr; using CNodePtr = std::shared_ptr; +#ifdef ENABLE_D +namespace device { +namespace ascend { +namespace tasksink { +class TaskDebugInfo; +} // namespace tasksink +} // namespace ascend +} // namespace device +using TaskDebugInfoPtr = std::shared_ptr; +#endif // ENABLE_D namespace RDR { bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name, const std::string &file_type = ".ir;.pb;.dat", int graph_id = -1); @@ -35,6 +44,10 @@ bool RecordString(SubModuleId module, const std::string &tag, const std::string const std::string &filename = ""); bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id, const std::vector &exec_order); +#ifdef ENABLE_D +bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag, + const std::vector &task_debug_info_list, int graph_id = 0); +#endif // ENABLE_D void TriggerAll(); void ClearAll(); } // namespace RDR diff --git a/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.cc b/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.cc new file mode 100644 index 0000000000..84607ea33a --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/rdr/task_debug_info_recorder.h" +#include "runtime/device/ascend/tasksink/task_generator.h" + +namespace mindspore { +void TaskDebugInfoRecorder::Export() { + auto realpath = GetFileRealPath(std::to_string(graph_id_)); + if (!realpath.has_value()) { + return; + } + std::string file_path = realpath.value() + ".ir"; + device::ascend::tasksink::TaskGenerator::DumpTaskInfo(file_path, task_debug_info_); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.h b/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.h new file mode 100644 index 0000000000..8abe482565 --- /dev/null +++ b/mindspore/ccsrc/debug/rdr/task_debug_info_recorder.h @@ -0,0 +1,48 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_RDR_TASK_DEBUG_INFO_RECORDER_H_ +#define MINDSPORE_CCSRC_DEBUG_RDR_TASK_DEBUG_INFO_RECORDER_H_ +#include +#include +#include + +#include "debug/rdr/base_recorder.h" + +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +class TaskDebugInfo; +} // namespace tasksink +} // namespace ascend +} // namespace device + +using TaskDebugInfoPtr = std::shared_ptr; +class TaskDebugInfoRecorder : public BaseRecorder { + public: + TaskDebugInfoRecorder() {} + TaskDebugInfoRecorder(const std::string &module, const std::string &tag, + const std::vector &task_debug_info, int graph_id) + : BaseRecorder(module, tag), graph_id_(graph_id), task_debug_info_(task_debug_info) {} + virtual void Export(); + + private: + int graph_id_; + std::vector task_debug_info_; +}; +using TaskDebugInfoRecorderPtr = std::shared_ptr; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_RDR_TASK_DEBUG_INFO_RECORDER_H_ diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index d7b9cdeffe..63a1eeccc1 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -62,6 +62,9 @@ #include "transform/graph_ir/df_graph_manager.h" #include "transform/graph_ir/op_adapter_map.h" #endif +#ifdef ENABLE_DUMP_IR +#include "debug/rdr/running_data_recorder.h" +#endif namespace mindspore { // namespace to support intermediate representation definition @@ -346,6 +349,9 @@ void ExecutorPy::DelNetRes(const std::string &id) { void ExecutorPy::ClearRes() { MS_LOG(INFO) << "Clean executor resource!"; Resource::mem_cleaner().ClearPrimitivePyPythonObj(); +#ifdef ENABLE_DUMP_IR + mindspore::RDR::ClearAll(); +#endif executor_ = nullptr; } diff --git a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc index 86f7454eb2..c01a77b547 100644 --- a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc @@ -250,6 +250,56 @@ bool TaskGenerator::LaunchAllKernel(const std::vector &anf_node_list, } #ifdef ENABLE_DUMP_IR +void TaskGenerator::DumpTaskInfo(const string &real_filename, + const std::vector &task_debug_info_list) { + OrderedMap para_map; + ChangeFileMode(real_filename, S_IRWXU); + std::ofstream fout(real_filename); + + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open dump file '" << real_filename << "' failed!"; + return; + } + + size_t index = 0; + for (auto &task_debug_info : task_debug_info_list) { + fout << "op_name:" << task_debug_info->op_name_ << "\n" + << "task_index:" << index << "\t" + << "task_num:" << task_debug_info->task_num_ << "\t" + << "task0_stream_id:" << task_debug_info->stream_id_ << "\t" + << "task0_type:" << task_debug_info->type_ << "\t" + << "task0_dump_flag:" << task_debug_info->dump_flag_ << "\n"; + index++; + if (task_debug_info->input_addrs_.size()) { + fout << "input address:"; + for (auto &input : task_debug_info->input_addrs_) { + fout << input->addr << "(" << input->size << ")\t"; + } + fout << "\n"; + } + + if (task_debug_info->output_addrs_.size()) { + fout << "output address:"; + for (auto &output : task_debug_info->output_addrs_) { + fout << output->addr << "(" << output->size << ")\t"; + } + fout << "\n"; + } + + if (task_debug_info->workspace_addrs_.size()) { + fout << "workspace address:"; + for (auto &workspace : task_debug_info->workspace_addrs_) { + fout << workspace->addr << "(" << workspace->size << ")\t"; + } + fout << "\n"; + } + fout << "\n"; + } + + fout.close(); + // set file mode to read only by user + ChangeFileMode(real_filename, S_IRUSR); +} void TaskGenerator::DumpTaskInfo(const std::string &real_filename) { if (real_filename.size() > PATH_MAX) { MS_LOG(ERROR) << "File path " << real_filename << " is too long."; @@ -322,8 +372,18 @@ void TaskGenerator::DumpTaskInfo(const std::string &real_filename) { return; } already_printed = true; - MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " - << "please recompile source to enable it. See help of building script."; + MS_LOG(WARNING) << "The functionality of dumping task debug info is disabled, " + << "please enable ENABLE_DUMP_IR with '-D on' and recomiple source."; +} +void TaskGenerator::DumpTaskInfo(const string &real_filename, + const std::vector &task_debug_info_list) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping task debug info is disabled, " + << "please enable ENABLE_DUMP_IR with '-D on' and recomiple source."; } #endif } // namespace tasksink diff --git a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h index b058fbbebe..a77e5d0c06 100644 --- a/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h +++ b/mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.h @@ -58,6 +58,8 @@ class TaskGenerator { bool GenTasks(const std::vector &anf_node_list, std::vector *task_info_list, uint32_t graph_id); + std::vector GetTaskDebugInfo() const { return task_debug_info_list_; } + static void DumpTaskInfo(const string &real_filename, const std::vector &task_debug_info_list); private: std::vector task_debug_info_list_;