From 5b63f8f2bc9c355e078293ef69786d6bb7588c15 Mon Sep 17 00:00:00 2001 From: limingqi107 Date: Wed, 2 Jun 2021 18:12:08 +0800 Subject: [PATCH] fix the coredump of actor runtime singleton instance deconstruction --- .../ccsrc/backend/session/session_basic.cc | 4 +- .../ccsrc/backend/session/session_basic.h | 2 + mindspore/ccsrc/pipeline/jit/pipeline.cc | 12 ++-- .../pipeline/pynative/pynative_execute.cc | 1 + .../framework/actor/data_source_actor.cc | 1 - .../runtime/framework/actor/kernel_actor.cc | 14 ++-- .../runtime/framework/device_tensor_store.h | 2 + .../ccsrc/runtime/framework/graph_compiler.cc | 50 ++++++-------- .../ccsrc/runtime/framework/graph_compiler.h | 30 ++++----- .../runtime/framework/graph_scheduler.cc | 27 ++++++-- .../ccsrc/runtime/framework/graph_scheduler.h | 21 +++--- .../hardware/device_context_manager.cc | 2 - .../runtime/hardware/device_context_manager.h | 1 - .../hardware/gpu/gpu_device_context.cc | 12 ++-- mindspore/ccsrc/vm/backend.cc | 65 ++++++++++++------- mindspore/ccsrc/vm/backend.h | 2 + 16 files changed, 134 insertions(+), 112 deletions(-) diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index d5c4f6959f..22bc5aab77 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -51,9 +51,12 @@ #include "ps/ps_context.h" #include "abstract/abstract_value.h" #endif +#include "backend/session/session_factory.h" namespace mindspore { namespace session { +MS_REG_SESSION(kSessionBasic, SessionBasic); + namespace { const int kSummaryGetItem = 2; const size_t max_depth = 128; @@ -1164,7 +1167,6 @@ KernelGraphPtr SessionBasic::ConstructKernelGraph(const AnfNodePtrList &lst, con } // add a make_tuple at the end of graph as output graph->set_output(ConstructOutput(outputs, graph)); - MS_EXCEPTION_IF_NULL(context_); FuncGraphManagerPtr manager = MakeManager({graph}); if (manager) { manager->AddFuncGraph(graph); diff --git a/mindspore/ccsrc/backend/session/session_basic.h b/mindspore/ccsrc/backend/session/session_basic.h index 2ac1ecd758..2a3e22b143 100644 --- a/mindspore/ccsrc/backend/session/session_basic.h +++ b/mindspore/ccsrc/backend/session/session_basic.h @@ -47,6 +47,8 @@ class GraphCompiler; namespace mindspore { using GraphId = uint32_t; using GraphInfo = std::string; +const char kSessionBasic[] = "SessionBasic"; + namespace session { using CallBackFunc = uint32_t (*)(uint32_t graph_id, const std::map ¶ms_list); diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index 9460839198..a3989953b5 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -50,7 +50,7 @@ #include "utils/info.h" #include "load_mindir/load_model.h" #include "pipeline/jit/prim_bprop_optimizer.h" -#include "mindrt/src/actor/actormgr.h" +#include "runtime/hardware/device_context_manager.h" #if ((defined ENABLE_CPU) && (!defined _WIN32)) #include "ps/constants.h" @@ -1085,10 +1085,10 @@ bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batc if (compile::IsMindRTUsed()) { const auto &mindrt_backend = std::dynamic_pointer_cast(backend); MS_EXCEPTION_IF_NULL(mindrt_backend); - auto graph_id = mindrt_backend->CompileGraphs(func_graph); + auto &actor_info = mindrt_backend->CompileGraphs(func_graph); VectorRef args; if (need_run) { - (void)mindrt_backend->RunGraph(graph_id, args); + (void)mindrt_backend->RunGraph(actor_info, args); } ConfigManager::GetInstance().set_iter_num(size); return true; @@ -1254,6 +1254,11 @@ void ClearResAtexit() { #endif session::ExecutorManager::Instance().Clear(); device::KernelRuntimeManager::Instance().ClearRuntimeResource(); + + // Clear the resource of mindRT. + runtime::GraphScheduler::GetInstance().Clear(); + device::DeviceContextManager::GetInstance().ClearDeviceContexts(); + ReleaseGeTsd(); parse::python_adapter::ResetPythonScope(); #ifdef ENABLE_DEBUGGER @@ -1266,7 +1271,6 @@ void ClearResAtexit() { parse::Parser::CleanParserResource(); parse::CleanDataClassToClassMap(); trace::ClearTraceStack(); - ActorMgr::GetActorMgrRef()->TerminateAll(); } } // namespace pipeline } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc b/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc index 8cf7ada28f..4a43901279 100644 --- a/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc @@ -2724,6 +2724,7 @@ void PynativeExecutor::ClearRes() { ad::CleanRes(); pipeline::ReclaimOptimizer(); kSession = nullptr; + mind_rt_backend = nullptr; } void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) { diff --git a/mindspore/ccsrc/runtime/framework/actor/data_source_actor.cc b/mindspore/ccsrc/runtime/framework/actor/data_source_actor.cc index 4d63432eec..fef69cd04a 100644 --- a/mindspore/ccsrc/runtime/framework/actor/data_source_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/data_source_actor.cc @@ -54,7 +54,6 @@ void DataSourceActor::FetchData(OpContext *context) { } void DataSourceActor::SendOutput(OpContext *context) { - MS_LOG(INFO) << "Data source actor(" << GetAID().Name() << ") sends output data."; MS_EXCEPTION_IF_NULL(context); // No output. if ((output_data_arrows_.size() == 0) && (output_result_arrows_.size() == 0)) { diff --git a/mindspore/ccsrc/runtime/framework/actor/kernel_actor.cc b/mindspore/ccsrc/runtime/framework/actor/kernel_actor.cc index 2567335a7f..1c72fd2e08 100644 --- a/mindspore/ccsrc/runtime/framework/actor/kernel_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/kernel_actor.cc @@ -297,6 +297,13 @@ void KernelActor::SendOutput(OpContext *context) const { Async(output_data->op_id_, &OpActor::RunOpData, output_data, context); } + // Send graph output result. + for (const auto &result_arrow : output_result_arrows_) { + MS_EXCEPTION_IF_NULL(result_arrow); + Async(result_arrow->to_op_id_, &OutputActor::CollectOutput, kernel_, result_arrow->from_output_index_, + result_arrow->to_input_index_, context); + } + // Send output control. if (output_control_arrows_.size() > 0) { auto source_aid = const_cast(&GetAID()); @@ -305,13 +312,6 @@ void KernelActor::SendOutput(OpContext *context) const { } } - // Send graph output result. - for (const auto &result_arrow : output_result_arrows_) { - MS_EXCEPTION_IF_NULL(result_arrow); - Async(result_arrow->to_op_id_, &OutputActor::CollectOutput, kernel_, result_arrow->from_output_index_, - result_arrow->to_input_index_, context); - } - // Send recorder info. if (recorder_aid_ != nullptr) { Async(*recorder_aid_, &RecorderActor::RecordMemAddressInfo, kernel_.get(), &launch_info_, device_context_, context); diff --git a/mindspore/ccsrc/runtime/framework/device_tensor_store.h b/mindspore/ccsrc/runtime/framework/device_tensor_store.h index a0bc7f48d8..e00749ee23 100644 --- a/mindspore/ccsrc/runtime/framework/device_tensor_store.h +++ b/mindspore/ccsrc/runtime/framework/device_tensor_store.h @@ -90,6 +90,8 @@ class DeviceTensorStore { return nullptr; } + void Clear() { device_tensors_.clear(); } + private: DeviceTensorStore() = default; ~DeviceTensorStore() = default; diff --git a/mindspore/ccsrc/runtime/framework/graph_compiler.cc b/mindspore/ccsrc/runtime/framework/graph_compiler.cc index d3631bcc77..14533fd809 100644 --- a/mindspore/ccsrc/runtime/framework/graph_compiler.cc +++ b/mindspore/ccsrc/runtime/framework/graph_compiler.cc @@ -18,7 +18,6 @@ #include #include #include -#include "runtime/framework/graph_scheduler.h" #include "runtime/device/device_address.h" #include "common/trans.h" #include "utils/convert_utils.h" @@ -252,44 +251,33 @@ void SetSummaryNodesRefCount(const KernelGraph *graph) { } } // namespace -void GraphCompiler::set_device_context(DeviceContext *device_context) { - MS_EXCEPTION_IF_NULL(device_context); - device_context_ = device_context; - - // The member variable 'session_' will be removed after removing session module. - if (session_ == nullptr) { - session_ = std::make_shared(); - const device::DeviceContextKey &device_context_key = device_context->device_context_key(); - session_->InitExecutor(device_context_key.device_name_, device_context_key.device_id_); - } -} - -GraphId GraphCompiler::CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs) { +GraphId GraphCompiler::CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs, + const DeviceContext *device_context) { MS_EXCEPTION_IF_NULL(session_); // Generate kernel graph. KernelGraphPtr graph = session_->ConstructKernelGraph(nodes, outputs); MS_EXCEPTION_IF_NULL(graph); - return CompileGraphImpl(graph); + return CompileGraphImpl(graph, device_context); } -GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph) const { +GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph, const DeviceContext *device_context) const { MS_EXCEPTION_IF_NULL(graph); - MS_EXCEPTION_IF_NULL(device_context_); + MS_EXCEPTION_IF_NULL(device_context); // Execute optimization pass. - device_context_->OptimizeGraph(graph); + device_context->OptimizeGraph(graph); // Generate 'KernelMod' for all kernels and set 'KernelMod' into kernel, // 'KernelMod' is real executive object of kernel. - device_context_->CreateKernel(graph->execution_order()); + device_context->CreateKernel(graph->execution_order()); // Create device address for all anf nodes of graph. - CreateDeviceAddress(graph); + CreateDeviceAddress(graph, device_context); graph->set_is_all_nop_node(opt::IsAllNopNode(graph.get())); MS_EXCEPTION_IF_NULL(session_); - session_->InitAllBucket(graph, device_context_); + session_->InitAllBucket(graph, device_context); session_->SetSummaryNodes(graph.get()); SetSummaryNodesRefCount(graph.get()); @@ -299,7 +287,7 @@ GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph) const { GraphId GraphCompiler::CompileGraph(const session::OpRunInfo &op_run_info, const GraphInfo &graph_info, const std::vector *tensors_mask, std::vector *input_tensors, - bool *single_op_cache_hit) { + bool *single_op_cache_hit, const DeviceContext *device_context) { // Check if the graph cache exists. auto iter = run_op_graphs_.find(graph_info); if (iter != run_op_graphs_.end()) { @@ -314,18 +302,18 @@ GraphId GraphCompiler::CompileGraph(const session::OpRunInfo &op_run_info, const KernelGraphPtr graph = session_->ConstructSingleOpGraph(op_run_info, *input_tensors, *tensors_mask); MS_EXCEPTION_IF_NULL(graph); - MS_EXCEPTION_IF_NULL(device_context_); - device_context_->OptimizeSingleOpGraph(graph); + MS_EXCEPTION_IF_NULL(device_context); + device_context->OptimizeSingleOpGraph(graph); MS_EXCEPTION_IF_NULL(session_); session_->RunOpHideNopNode(graph); session_->RunOpRemoveNopNode(graph); // Generate 'KernelMod' for kernel in graph. - device_context_->CreateKernel(graph->execution_order()); + device_context->CreateKernel(graph->execution_order()); // Create device address for all anf nodes of graph. - CreateDeviceAddress(graph); + CreateDeviceAddress(graph, device_context); graph->set_is_all_nop_node(opt::IsAllNopNode(graph.get())); run_op_graphs_[graph_info] = graph; @@ -346,11 +334,11 @@ KernelGraphPtr GraphCompiler::Fetch(const GraphInfo &graph_info) const { return iter->second; } -void GraphCompiler::CreateDeviceAddress(const KernelGraphPtr &graph) const { - CreateParameterDeviceAddress(device_context_, graph); - CreateValueNodeDeviceAddress(device_context_, graph); - CreateKernelOutputDeviceAddress(device_context_, graph); - CreateKernelWorkspaceDeviceAddress(device_context_, graph); +void GraphCompiler::CreateDeviceAddress(const KernelGraphPtr &graph, const DeviceContext *device_context) const { + CreateParameterDeviceAddress(device_context, graph); + CreateValueNodeDeviceAddress(device_context, graph); + CreateKernelOutputDeviceAddress(device_context, graph); + CreateKernelWorkspaceDeviceAddress(device_context, graph); UpdateDeviceAddressForInplaceNode(graph); } diff --git a/mindspore/ccsrc/runtime/framework/graph_compiler.h b/mindspore/ccsrc/runtime/framework/graph_compiler.h index 579cbd09ef..6a4be9a6a1 100644 --- a/mindspore/ccsrc/runtime/framework/graph_compiler.h +++ b/mindspore/ccsrc/runtime/framework/graph_compiler.h @@ -24,36 +24,32 @@ #include #include "runtime/hardware/device_context.h" #include "backend/session/session_basic.h" +#include "backend/session/session_factory.h" #include "ir/tensor.h" namespace mindspore { using device::DeviceContext; -using mindspore::tensor::TensorPtr; using session::CallBackFunc; using session::InputTensorInfo; +using session::KernelGraph; using session::KernelWithIndex; using session::OpRunInfo; +using tensor::TensorPtr; namespace runtime { class GraphCompiler { public: - static GraphCompiler &GetInstance() { - static GraphCompiler instance; - return instance; - } - - // Set device context which is initialized, the function must be called - // before using GraphCompiler and after changing device type or device id. - void set_device_context(DeviceContext *device_context); + GraphCompiler() { session_ = session::SessionFactory::Get().Create(kSessionBasic); } + ~GraphCompiler() = default; // Construct kernel graph from anf nodes list and compile kernel graph in Graph mode, // the detailed implementation of compiling graph is in 'CompileGraphImpl'. - GraphId CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs); + GraphId CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs, const DeviceContext *device_context); // Construct single op kernel graph and compile the kernel graph in PyNative mode. GraphId CompileGraph(const session::OpRunInfo &op_run_info, const GraphInfo &graph_info, const std::vector *tensors_mask, std::vector *input_tensors, - bool *single_op_cache_hit); + bool *single_op_cache_hit, const DeviceContext *device_context); // Get graph by graph id, if not exist return nullptr, used in Graph mode. KernelGraphPtr Fetch(GraphId graph_id) const; @@ -101,25 +97,23 @@ class GraphCompiler { void Summary(const std::vector &graphs) const; private: - GraphCompiler() = default; - ~GraphCompiler() = default; DISABLE_COPY_AND_ASSIGN(GraphCompiler); // The implementation of compiling graph in Graph Mode, including optimizing graph, // setting operator info, creating kernel and transforming kernel graph to ActorSet. - GraphId CompileGraphImpl(const KernelGraphPtr &graph) const; + GraphId CompileGraphImpl(const KernelGraphPtr &graph, const DeviceContext *device_context) const; // Create device address for all anf nodes of graph. - void CreateDeviceAddress(const KernelGraphPtr &graph) const; - - DeviceContext *device_context_{nullptr}; + void CreateDeviceAddress(const KernelGraphPtr &graph, const DeviceContext *device_context) const; // Single op kernel graph cache for PyNative mode. std::unordered_map run_op_graphs_; // The member variable 'session_' will be removed after removing session module. - session::SessionPtr session_{nullptr}; + // Now all the GraphCompiler share the same 'session_'. + session::SessionPtr session_; }; + } // namespace runtime } // namespace mindspore #endif // MINDSPORE_CCSRC_RUNTIME_FRAMEWORK_GRAPH_COMPILER_H_ diff --git a/mindspore/ccsrc/runtime/framework/graph_scheduler.cc b/mindspore/ccsrc/runtime/framework/graph_scheduler.cc index c24b53da98..e0421121ba 100644 --- a/mindspore/ccsrc/runtime/framework/graph_scheduler.cc +++ b/mindspore/ccsrc/runtime/framework/graph_scheduler.cc @@ -322,22 +322,35 @@ void PrepareDataForHostDataSourceActor(const std::unordered_mapFinalize(); + + // Clear the member of DeviceTensorStore. + DeviceTensorStore::GetInstance().Clear(); - // Local maps clear. + // Clear global maps. + actors_.clear(); actor_name_to_actor_.clear(); + actor_to_host_queue_.clear(); + device_tensor_to_actor_.clear(); + + // Clear local maps and vectors. graph_output_to_actor_.clear(); + front_node_to_actor_.clear(); + copy_actors_.clear(); + + // Delete the thread pool. delete thread_pool_; thread_pool_ = nullptr; } void GraphScheduler::Initialize() { - // Local maps and vcetors clear. + // Local maps and vectors clear. graph_output_to_actor_.clear(); + front_node_to_actor_.clear(); copy_actors_.clear(); if (init_) { diff --git a/mindspore/ccsrc/runtime/framework/graph_scheduler.h b/mindspore/ccsrc/runtime/framework/graph_scheduler.h index 2051ed8394..0c5a95ac29 100644 --- a/mindspore/ccsrc/runtime/framework/graph_scheduler.h +++ b/mindspore/ccsrc/runtime/framework/graph_scheduler.h @@ -133,9 +133,12 @@ class GraphScheduler { } // 1. Thread pool creating. - // 2. The memory manager creating and scheduling. + // 2. The global actors creating and scheduling. void Initialize(); + // Clear the members. + void Clear(); + // Transform graph to actor DAG, contains build and link. ActorSet *Transform(const GraphCompilerInfo &graph_compiler_info, GraphExecutionStrategy strategy = GraphExecutionStrategy::kPipeline); @@ -160,7 +163,7 @@ class GraphScheduler { private: GraphScheduler() = default; - ~GraphScheduler(); + ~GraphScheduler() = default; DISABLE_COPY_AND_ASSIGN(GraphScheduler); // Transform the nodes of graph to actors. @@ -265,18 +268,18 @@ class GraphScheduler { // The global maps, only be cleared in the deconstruction. std::unordered_map actors_; std::unordered_map *> actor_name_to_actor_; - // Since the control node does not have a backend node, it can only be connected through the relationship between - // the front node, so the mapping relationship between the front node and the actor needs to be recorded. - std::unordered_map front_node_to_actor_; std::unordered_map actor_to_host_queue_; // The second element of pair represents the output index of op actor corresponding to the device tensor. std::unordered_map device_tensor_to_actor_; - // The local maps and vectors, will be cleared at the beginning of each graph transform. - // The second element of pair represents the output index of op actor corresponding to the graph output front node. + // The local maps and vectors, will be cleared at the beginning of each graph transform: + // 1.The second element of pair represents the output index of op actor corresponding to the graph output front node. std::map graph_output_to_actor_; - // Beaceuse the copy actors are built in the link, so need record the all copy actors in the link process to push into - // the actor set after link. + // 2.Since the control node does not have a backend node, it can only be connected through the relationship between + // the front node, so the mapping relationship between the front node and the actor needs to be recorded. + std::unordered_map front_node_to_actor_; + // 3.Beaceuse the copy actors are built in the link, so need record the all copy actors in the link process to push + // into the actor set after link. std::vector copy_actors_; // The id of global actor. diff --git a/mindspore/ccsrc/runtime/hardware/device_context_manager.cc b/mindspore/ccsrc/runtime/hardware/device_context_manager.cc index 216142b985..13b54207e1 100644 --- a/mindspore/ccsrc/runtime/hardware/device_context_manager.cc +++ b/mindspore/ccsrc/runtime/hardware/device_context_manager.cc @@ -25,7 +25,6 @@ void DeviceContextManager::Register(const std::string &device_name, DeviceContex } void DeviceContextManager::ClearDeviceContexts() { - std::lock_guard guard(lock_); for (auto &iter : device_contexts_) { MS_LOG(INFO) << "Release device " << iter.first; MS_EXCEPTION_IF_NULL(iter.second); @@ -36,7 +35,6 @@ void DeviceContextManager::ClearDeviceContexts() { DeviceContext *DeviceContextManager::GetOrCreateDeviceContext(const DeviceContextKey &device_context_key) { std::string device_context_key_str = device_context_key.ToString(); - std::lock_guard guard(lock_); auto device_context_iter = device_contexts_.find(device_context_key_str); if (device_context_iter != device_contexts_.end()) { diff --git a/mindspore/ccsrc/runtime/hardware/device_context_manager.h b/mindspore/ccsrc/runtime/hardware/device_context_manager.h index 5d78bd3042..6c9579f993 100644 --- a/mindspore/ccsrc/runtime/hardware/device_context_manager.h +++ b/mindspore/ccsrc/runtime/hardware/device_context_manager.h @@ -48,7 +48,6 @@ class DeviceContextManager { std::map device_contexts_; // The name of device -> DeviceContextCreator. std::map device_context_creators_; - std::mutex lock_; }; class DeviceContextRegister { diff --git a/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc b/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc index 86675ca245..0ebad32ade 100644 --- a/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc +++ b/mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc @@ -187,6 +187,12 @@ void GPUDeviceContext::OptimizeGraph(const KernelGraphPtr &graph) const { // Optimization pass which is relevant to device type or format. OptimizeGraphWithDeviceInfo(graph); + // Graph kernel fusion optimization + if (context::GraphKernelFlags::GetInstance().IsEnableGraphKernel()) { + opt::GraphKernelOptimize(graph); + graph->SetExecOrderByDefault(); + } + // Assign the stream and insert the send/recv node for all reduce kernel, so must be the last in the optimizer. device::gpu::AssignGpuStream(graph); } @@ -244,12 +250,6 @@ void GPUDeviceContext::FuseOperators(const KernelGraphPtr &graph) const { optimizer->AddPassManager(pm); (void)optimizer->Optimize(graph); graph->SetExecOrderByDefault(); - - // Graph kernel fusion optimization - if (context::GraphKernelFlags::GetInstance().IsEnableGraphKernel()) { - opt::GraphKernelOptimize(graph); - graph->SetExecOrderByDefault(); - } } void GPUDeviceContext::UpdateGraphDynamicShapeAttr(const NotNull &graph) const { diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 00b86a9bab..e66897ec43 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -252,14 +252,16 @@ MindRTBackend::MindRTBackend(const std::string &backend_name, const std::string : Backend(backend_name), device_name_(device_name), device_id_(device_id) { auto cut_list = compile::GetMsNonlinearOps(); graph_partition_ = std::make_shared(cut_list, backend_name); + graph_compiler_ = std::make_shared(); } const ActorInfo &MindRTBackend::CompileGraphs(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(graph_compiler_); MS_EXCEPTION_IF_NULL(func_graph); FuncGraphPtr root_graph = WrapPrimitives(func_graph); MS_EXCEPTION_IF_NULL(root_graph); // Register a summary callback function, which is called in the final stages of summary. - runtime::GraphCompiler::GetInstance().RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback); + graph_compiler_->RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback); // Compile root graph. graph_id_to_device_context_.clear(); @@ -293,6 +295,7 @@ const ActorInfo &MindRTBackend::CompileGraphs(const FuncGraphPtr &func_graph) { void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) { MS_EXCEPTION_IF_NULL(func_graph); MS_EXCEPTION_IF_NULL(graph_partition_); + MS_EXCEPTION_IF_NULL(graph_compiler_); // Split graph to segments. const auto &segments = graph_partition_->Partition(func_graph); @@ -308,12 +311,11 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) { } MS_LOG(INFO) << "Compile normal segment, the first node: " << segment->nodes_[0]->fullname_with_scope(); - // Get and set the device context. + // Get the device context. const auto &cur_device_name = GetCNodeTarget(segment->nodes_[0]); const auto &device_context = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({cur_device_name, device_id_}); device_context->Initialize(); - runtime::GraphCompiler::GetInstance().set_device_context(device_context); // Transform nodes to inputs and outputs. FuncGraphPtr fg; @@ -322,7 +324,7 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) { std::tie(fg, inputs, outputs) = TransformSegmentToAnfGraph(segment->nodes_); // Compile graph. - auto graph_id = runtime::GraphCompiler::GetInstance().CompileGraph(segment->nodes_, outputs); + auto graph_id = graph_compiler_->CompileGraph(segment->nodes_, outputs, device_context); graph_id_to_device_context_[graph_id] = device_context; } else { // Compile the cut node. @@ -337,16 +339,15 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) { const ActorInfo &MindRTBackend::CompileGraph(const OpRunInfo &op_run_info, const GraphInfo &graph_info, const std::vector *tensors_mask, std::vector *input_tensors) { - static DeviceContext *device_context = nullptr; - if (!device_context) { - device_context = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({device_name_, device_id_}); - device_context->Initialize(); - runtime::GraphCompiler::GetInstance().set_device_context(device_context); - } + MS_EXCEPTION_IF_NULL(graph_compiler_); + // Get the device context. + const auto &device_context = + device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({device_name_, device_id_}); + device_context->Initialize(); bool single_op_cache_hit; - (void)runtime::GraphCompiler::GetInstance().CompileGraph(op_run_info, graph_info, tensors_mask, input_tensors, - &single_op_cache_hit); + (void)graph_compiler_->CompileGraph(op_run_info, graph_info, tensors_mask, input_tensors, &single_op_cache_hit, + device_context); if (single_op_cache_hit) { return graph_info; } @@ -366,28 +367,28 @@ const ActorInfo &MindRTBackend::CompileGraph(const OpRunInfo &op_run_info, const void MindRTBackend::RunGraphBySingleOp(const std::vector &graphs, const std::vector> &inputs, VectorRef *outputs) { + MS_EXCEPTION_IF_NULL(graph_compiler_); for (size_t graph_index = 0; graph_index < graphs.size(); ++graph_index) { const auto &graph = graphs[graph_index]; std::map parameter_index; std::map>> output_indexes; std::map op_output_map; - runtime::GraphCompiler::GetInstance().GetParamAndOutputIndex(graph, inputs[graph_index], outputs, ¶meter_index, - &output_indexes); + graph_compiler_->GetParamAndOutputIndex(graph, inputs[graph_index], outputs, ¶meter_index, &output_indexes); // Clear bucket resources every step if (graph->is_bprop()) { - runtime::GraphCompiler::GetInstance().ClearAllBucket(graph->graph_id()); + graph_compiler_->ClearAllBucket(graph->graph_id()); } for (const auto &kernel : graph->execution_order()) { OpRunInfo op_run_info; GraphInfo graph_info; InputTensorInfo input_tensor_info; - runtime::GraphCompiler::GetInstance().GetSingleOpInputTensors(kernel, op_output_map, parameter_index, - inputs[graph_index], &input_tensor_info); - runtime::GraphCompiler::GetInstance().GetSingleOpRunInfoAndGraphInfo(kernel, input_tensor_info.input_tensors, - &op_run_info, &graph_info); + graph_compiler_->GetSingleOpInputTensors(kernel, op_output_map, parameter_index, inputs[graph_index], + &input_tensor_info); + graph_compiler_->GetSingleOpRunInfoAndGraphInfo(kernel, input_tensor_info.input_tensors, &op_run_info, + &graph_info); const ActorInfo &actor_info = CompileGraph(op_run_info, graph_info, &input_tensor_info.input_tensors_mask, &input_tensor_info.input_tensors); @@ -395,12 +396,12 @@ void MindRTBackend::RunGraphBySingleOp(const std::vector &graphs RunGraph(actor_info, &op_run_info, &input_tensor_info.input_tensors_mask, &input_tensor_info.input_tensors); std::vector new_output_tensors; - runtime::GraphCompiler::GetInstance().RecoverGraphOutput(kernel, op_outputs, output_indexes, &op_output_map, - outputs, &new_output_tensors); + graph_compiler_->RecoverGraphOutput(kernel, op_outputs, output_indexes, &op_output_map, outputs, + &new_output_tensors); // Save grad node to Bucket if (graph->is_bprop()) { - runtime::GraphCompiler::GetInstance().AddGradAddrToBucket(graph->graph_id(), new_output_tensors); + graph_compiler_->AddGradAddrToBucket(graph->graph_id(), new_output_tensors); } } } @@ -490,18 +491,19 @@ VectorRef MindRTBackend::RunGraph(const ActorInfo &actor_info, const VectorRef & } MS_LOG(INFO) << "Run actor end, actor name: " << actor_info; - runtime::GraphCompiler::GetInstance().Summary(graph_compiler_info.graphs_); + graph_compiler_->Summary(graph_compiler_info.graphs_); return outputs; } std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo(const FuncGraphPtr &root_graph) { MS_EXCEPTION_IF_NULL(root_graph); + MS_EXCEPTION_IF_NULL(graph_compiler_); std::vector graphs; std::vector device_contexts; std::string name = "kernel_graph"; for (const auto &graph_id_to_context : graph_id_to_device_context_) { - graphs.emplace_back(runtime::GraphCompiler::GetInstance().Fetch(graph_id_to_context.first)); + graphs.emplace_back(graph_compiler_->Fetch(graph_id_to_context.first)); device_contexts.emplace_back(graph_id_to_context.second); name.append("_").append(std::to_string(graph_id_to_context.first)); } @@ -519,6 +521,10 @@ std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo(con for (const auto &output : outputs) { const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(output, 0, false); MS_EXCEPTION_IF_NULL(output_with_index.first); + // The InitDataSetQueue kernel has no output. + if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) { + continue; + } outputs_order.emplace(output_with_index, position++); } } else if (branch_output->isa()) { @@ -526,6 +532,10 @@ std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo(con for (size_t i = 0; i < outputs_num; i++) { const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(branch_output, i, false); MS_EXCEPTION_IF_NULL(output_with_index.first); + // The InitDataSetQueue kernel has no output. + if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) { + continue; + } outputs_order.emplace(output_with_index, position++); } } @@ -546,6 +556,7 @@ std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo(con std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo( const std::vector *tensors_mask, const std::vector *input_tensors) { + MS_EXCEPTION_IF_NULL(graph_compiler_); std::vector graphs; std::vector device_contexts; runtime::KernelMapPosition outputs_order; @@ -553,7 +564,7 @@ std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo( std::string name; for (const auto &graph_info_to_context : graph_info_to_device_context_) { - const auto &graph = runtime::GraphCompiler::GetInstance().Fetch(graph_info_to_context.first); + const auto &graph = graph_compiler_->Fetch(graph_info_to_context.first); graphs.emplace_back(graph); device_contexts.emplace_back(graph_info_to_context.second); name.append(graph_info_to_context.first); @@ -562,6 +573,10 @@ std::unique_ptr MindRTBackend::ConstructGraphCompilerInfo( for (const auto &output : outputs) { const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(output, 0, false); MS_EXCEPTION_IF_NULL(output_with_index.first); + // The InitDataSetQueue kernel has no output. + if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) { + continue; + } outputs_order.emplace(output_with_index, position++); } } diff --git a/mindspore/ccsrc/vm/backend.h b/mindspore/ccsrc/vm/backend.h index ebc209708a..b984133ca2 100644 --- a/mindspore/ccsrc/vm/backend.h +++ b/mindspore/ccsrc/vm/backend.h @@ -37,6 +37,7 @@ namespace compile { using OpRunInfo = session::OpRunInfo; using DeviceContext = device::DeviceContext; using ActorInfo = runtime::ActorInfo; +using GraphCompiler = runtime::GraphCompiler; using GraphCompilerInfo = runtime::GraphCompilerInfo; using ControlNodeParser = runtime::ControlNodeParser; using FrontToBackendNodeWithContext = runtime::FrontToBackendNodeWithContext; @@ -143,6 +144,7 @@ class MindRTBackend : public Backend { std::unordered_map> actor_to_graph_compiler_info_; GraphPartitionPtr graph_partition_; + std::shared_ptr graph_compiler_; std::string device_name_; uint32_t device_id_; };