From a2ba47e18a58efa02205fdb8025bf4d9c08ef79c Mon Sep 17 00:00:00 2001 From: huanghui Date: Wed, 10 Mar 2021 15:06:44 +0800 Subject: [PATCH] 1. Add unique id for .dat and .dot file to avoid covering 2. Dump the end graph in gpu session and cu session --- .../ccsrc/backend/session/cpu_session.cc | 4 +++ .../ccsrc/backend/session/gpu_session.cc | 2 ++ .../ccsrc/backend/session/session_basic.cc | 14 +++++++++ .../ccsrc/backend/session/session_basic.h | 1 + mindspore/ccsrc/debug/anf_ir_dump.cc | 22 ++----------- mindspore/ccsrc/debug/anf_ir_utils.cc | 31 +++++++------------ mindspore/ccsrc/debug/common.cc | 19 ++++++++++++ mindspore/ccsrc/debug/common.h | 2 ++ mindspore/ccsrc/debug/draw.cc | 5 +-- 9 files changed, 59 insertions(+), 41 deletions(-) diff --git a/mindspore/ccsrc/backend/session/cpu_session.cc b/mindspore/ccsrc/backend/session/cpu_session.cc index 36c4697eef..d7a44dbdaf 100644 --- a/mindspore/ccsrc/backend/session/cpu_session.cc +++ b/mindspore/ccsrc/backend/session/cpu_session.cc @@ -28,6 +28,8 @@ #include "backend/optimizer/common/optimizer.h" #include "backend/optimizer/common/pass_manager.h" #include "backend/optimizer/pass/replace_node_by_proxy.h" +#include "debug/anf_ir_dump.h" +#include "debug/dump_proto.h" #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) #include "ps/util.h" #include "ps/ps_context.h" @@ -98,6 +100,8 @@ GraphId CPUSession::CompileGraphImpl(const AnfNodePtrList &lst, const AnfNodePtr MS_LOG(INFO) << "Assign kernel address"; runtime_.AssignKernelAddress(graph.get()); + + DumpGraph(graph); return graph_id; } diff --git a/mindspore/ccsrc/backend/session/gpu_session.cc b/mindspore/ccsrc/backend/session/gpu_session.cc index c134cc0569..8b070bd4cd 100644 --- a/mindspore/ccsrc/backend/session/gpu_session.cc +++ b/mindspore/ccsrc/backend/session/gpu_session.cc @@ -403,6 +403,8 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) { AllocateMemory(graph.get()); } + DumpGraph(graph); + #ifdef ENABLE_DEBUGGER if (debugger_ && debugger_->DebuggerBackendEnabled()) { debugger_->LoadGraphs(graph); diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 1f7c5e10df..43556deea3 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -38,6 +38,7 @@ #include "ir/func_graph_cloner.h" #include "utils/utils.h" #include "debug/anf_ir_dump.h" +#include "debug/dump_proto.h" #include "debug/common.h" #include "utils/trace_base.h" #include "frontend/parallel/context.h" @@ -2451,6 +2452,19 @@ void SessionBasic::ClearAllBucket(const GraphId &graph_id) { } } +void SessionBasic::DumpGraph(const std::shared_ptr &kernel_graph) { +#ifdef ENABLE_DUMP_IR + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->get_param(MS_CTX_SAVE_GRAPHS_FLAG); + if (save_graphs) { + DumpIR("graph_build_" + std::to_string(kernel_graph->graph_id()) + ".ir", kernel_graph, true, kWholeStack); + DumpIRProto(kernel_graph, "vm_build_" + std::to_string(kernel_graph->graph_id())); + DumpIR("trace_code_graph", kernel_graph, true, kWholeStack); + } +#endif +} + #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) void SessionBasic::InitPsWorker(const KernelGraphPtr &kernel_graph) { if (!ps::PSContext::instance()->is_worker()) { diff --git a/mindspore/ccsrc/backend/session/session_basic.h b/mindspore/ccsrc/backend/session/session_basic.h index dff8789f73..37c6d34ae9 100644 --- a/mindspore/ccsrc/backend/session/session_basic.h +++ b/mindspore/ccsrc/backend/session/session_basic.h @@ -230,6 +230,7 @@ class SessionBasic : public std::enable_shared_from_this { void ClearAllBucket(const GraphId &graph_id); std::vector GetAllReduceSplitIndex(); virtual std::string GetCommWorldGroup() { return std::string(); } + void DumpGraph(const std::shared_ptr &kernel_graph); #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU)) void CheckPSModeConsistence(const KernelGraphPtr &kernel_graph) const; void GetBatchElements(const AnfNodePtr &kernel_node) const; diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc index 97445e2a63..734cd25a14 100644 --- a/mindspore/ccsrc/debug/anf_ir_dump.cc +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -527,24 +527,6 @@ void DumpSubgraph(const OrderedMap } } -std::string AddGlobalId(const std::string &filename) { - static size_t g_id = 0; - std::ostringstream s; - auto i = filename.rfind(".ir"); - if (i >= filename.size()) { - s << filename; - s << "_" << std::setfill('0') << std::setw(4) << g_id; - } else { - s << filename.substr(0, i); - s << "_" << std::setfill('0') << std::setw(4) << g_id; - if (i + 1 < filename.size()) { - s << filename.substr(i); - } - } - ++g_id; - return s.str(); -} - void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) { static std::unordered_map dump_level_map = { {std::to_string(kOff), kOff}, {std::to_string(kTopStack), kTopStack}, {std::to_string(kWholeStack), kWholeStack}}; @@ -564,7 +546,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu if (graph == nullptr) { return; } - auto path = pipeline::GetSaveGraphsPathName(AddGlobalId(filename)); + auto path = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".ir")); if (!target_file.empty()) { path = target_file; } @@ -609,7 +591,7 @@ void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool d if (graph == nullptr) { return; } - auto path = AddGlobalId(filename); + auto path = Common::AddId(filename, ".ir"); auto realpath = Common::GetRealPath(path); if (!realpath.has_value()) { MS_LOG(ERROR) << "Get real path failed. path=" << path; diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index c60fd8718c..ee98741254 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -40,6 +40,7 @@ #include "utils/ms_context.h" #include "frontend/operator/ops.h" #include "pipeline/jit/base.h" +#include "debug/common.h" using mindspore::tensor::TensorPy; @@ -189,6 +190,14 @@ std::string AnfExporter::GetMultitypeFuncGraphText(const prim::MultitypeFuncGrap return oss.str(); } +inline bool Skip(const MetaFuncGraphPtr &meta_func_graph) { + return meta_func_graph->isa() || meta_func_graph->isa() || + meta_func_graph->isa() || meta_func_graph->isa() || + meta_func_graph->isa() || meta_func_graph->isa() || + meta_func_graph->isa() || meta_func_graph->isa() || + meta_func_graph->isa(); +} + /* inherit relation of MetaFuncGraph * * MetaGraph @@ -239,23 +248,7 @@ std::string AnfExporter::GetMetaFuncGraphText(const MetaFuncGraphPtr &meta_func_ prim::GradOperationPtr grad_op = meta_func_graph->cast(); oss << "{get_all=" << grad_op->get_all_ << ", get_by_list=" << grad_op->get_by_list_ << ", sens_param=" << grad_op->sens_param_ << "}"; - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { - // do nothing - } else if (meta_func_graph->isa()) { + } else if (Skip(meta_func_graph)) { // do nothing } else { MS_LOG(EXCEPTION) << "Unknown MetaFuncGraph type " << meta_func_graph->type_name(); @@ -711,7 +704,7 @@ void ExportIR(const std::string &filename, const std::string &id, const FuncGrap return; } - auto real_filename = pipeline::GetSaveGraphsPathName(filename); + auto real_filename = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dat")); AnfExporter exporter(id); ChangeFileMode(real_filename, S_IRWXU); exporter.ExportFuncGraph(real_filename, func_graph); @@ -720,7 +713,7 @@ void ExportIR(const std::string &filename, const std::string &id, const FuncGrap } void ExportIR(const std::string &filename, const std::vector &graphs) { - auto real_filename = pipeline::GetSaveGraphsPathName(filename); + auto real_filename = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dat")); AnfExporter exporter("", false); ChangeFileMode(real_filename, S_IRWXU); exporter.ExportFuncGraph(real_filename, graphs); diff --git a/mindspore/ccsrc/debug/common.cc b/mindspore/ccsrc/debug/common.cc index a3751c3f18..3d544b088f 100644 --- a/mindspore/ccsrc/debug/common.cc +++ b/mindspore/ccsrc/debug/common.cc @@ -17,6 +17,7 @@ #include "debug/common.h" #include +#include #include #include "utils/ms_context.h" #include "utils/system/env.h" @@ -271,4 +272,22 @@ bool Common::IsFilenameValid(const std::string &filename, const int &length_limi } return true; } + +std::string Common::AddId(const std::string &filename, const std::string &suffix) { + static size_t g_id = 0; + std::ostringstream s; + auto i = filename.rfind(suffix); + if (i >= filename.size()) { + s << filename; + s << "_" << std::setfill('0') << std::setw(4) << g_id; + } else { + s << filename.substr(0, i); + s << "_" << std::setfill('0') << std::setw(4) << g_id; + if (i + 1 < filename.size()) { + s << filename.substr(i); + } + } + g_id++; + return s.str(); +} } // namespace mindspore diff --git a/mindspore/ccsrc/debug/common.h b/mindspore/ccsrc/debug/common.h index fefd325d4c..c8a1ec1f1c 100644 --- a/mindspore/ccsrc/debug/common.h +++ b/mindspore/ccsrc/debug/common.h @@ -40,6 +40,8 @@ class Common { const std::string &error_message = ""); static bool CreateNotExistDirs(const std::string &path); + static std::string AddId(const std::string &filename, const std::string &suffix); + private: static bool IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message); }; diff --git a/mindspore/ccsrc/debug/draw.cc b/mindspore/ccsrc/debug/draw.cc index 89bf23a1ad..ff68fcfdee 100644 --- a/mindspore/ccsrc/debug/draw.cc +++ b/mindspore/ccsrc/debug/draw.cc @@ -30,6 +30,7 @@ #include "pipeline/jit/parse/resolve.h" #include "ir/tensor.h" #include "pipeline/jit/base.h" +#include "debug/common.h" namespace mindspore { @@ -190,11 +191,11 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) { const std::string dot_suffix = ".dot"; std::string filename_with_suffix = (filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename; - DrawByOpt(pipeline::GetSaveGraphsPathName(filename_with_suffix), func_graph, false); + DrawByOpt(pipeline::GetSaveGraphsPathName(Common::AddId(filename_with_suffix, dot_suffix)), func_graph, false); } void DrawUserFuncGraph(const std::string &filename, const FuncGraphPtr &func_graph) { - DrawByOpt(pipeline::GetSaveGraphsPathName(filename), func_graph, true); + DrawByOpt(pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dot")), func_graph, true); } #else void Draw(const std::string &, const FuncGraphPtr &) {