!17642 fix the coredump of actor runtime singleton instance deconstruction

From: @limingqi107
Reviewed-by: @cristoval,@wilfchen
Signed-off-by: @wilfchen

mindspore/ccsrc/backend/session/session_basic.cc

@@ -51,9 +51,12 @@
 #include "ps/ps_context.h"
 #include "abstract/abstract_value.h"
 #endif
+#include "backend/session/session_factory.h"
 
 namespace mindspore {
 namespace session {
+MS_REG_SESSION(kSessionBasic, SessionBasic);
+
 namespace {
 const int kSummaryGetItem = 2;
 const size_t max_depth = 128;
@@ -1164,7 +1167,6 @@ KernelGraphPtr SessionBasic::ConstructKernelGraph(const AnfNodePtrList &lst, con
   }
   // add a make_tuple at the end of graph as output
   graph->set_output(ConstructOutput(outputs, graph));
-  MS_EXCEPTION_IF_NULL(context_);
   FuncGraphManagerPtr manager = MakeManager({graph});
   if (manager) {
     manager->AddFuncGraph(graph);

mindspore/ccsrc/backend/session/session_basic.h

@@ -47,6 +47,8 @@ class GraphCompiler;
 namespace mindspore {
 using GraphId = uint32_t;
 using GraphInfo = std::string;
+const char kSessionBasic[] = "SessionBasic";
+
 namespace session {
 using CallBackFunc = uint32_t (*)(uint32_t graph_id,
                                   const std::map<std::string, mindspore::tensor::TensorPtr> &params_list);

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -50,7 +50,7 @@
 #include "utils/info.h"
 #include "load_mindir/load_model.h"
 #include "pipeline/jit/prim_bprop_optimizer.h"
-#include "mindrt/src/actor/actormgr.h"
+#include "runtime/hardware/device_context_manager.h"
 
 #if ((defined ENABLE_CPU) && (!defined _WIN32))
 #include "ps/constants.h"
@@ -1085,10 +1085,10 @@ bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batc
   if (compile::IsMindRTUsed()) {
     const auto &mindrt_backend = std::dynamic_pointer_cast<compile::MindRTBackend>(backend);
     MS_EXCEPTION_IF_NULL(mindrt_backend);
-    auto graph_id = mindrt_backend->CompileGraphs(func_graph);
+    auto &actor_info = mindrt_backend->CompileGraphs(func_graph);
     VectorRef args;
     if (need_run) {
-      (void)mindrt_backend->RunGraph(graph_id, args);
+      (void)mindrt_backend->RunGraph(actor_info, args);
     }
     ConfigManager::GetInstance().set_iter_num(size);
     return true;
@@ -1254,6 +1254,11 @@ void ClearResAtexit() {
 #endif
   session::ExecutorManager::Instance().Clear();
   device::KernelRuntimeManager::Instance().ClearRuntimeResource();
+
+  // Clear the resource of mindRT.
+  runtime::GraphScheduler::GetInstance().Clear();
+  device::DeviceContextManager::GetInstance().ClearDeviceContexts();
+
   ReleaseGeTsd();
   parse::python_adapter::ResetPythonScope();
 #ifdef ENABLE_DEBUGGER
@@ -1266,7 +1271,6 @@ void ClearResAtexit() {
   parse::Parser::CleanParserResource();
   parse::CleanDataClassToClassMap();
   trace::ClearTraceStack();
-  ActorMgr::GetActorMgrRef()->TerminateAll();
 }
 }  // namespace pipeline
 }  // namespace mindspore

mindspore/ccsrc/pipeline/pynative/pynative_execute.cc

@@ -2724,6 +2724,7 @@ void PynativeExecutor::ClearRes() {
   ad::CleanRes();
   pipeline::ReclaimOptimizer();
   kSession = nullptr;
+  mind_rt_backend = nullptr;
 }
 
 void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) {

mindspore/ccsrc/runtime/framework/actor/data_source_actor.cc

@@ -54,7 +54,6 @@ void DataSourceActor::FetchData(OpContext<DeviceTensor> *context) {
 }
 
 void DataSourceActor::SendOutput(OpContext<DeviceTensor> *context) {
-  MS_LOG(INFO) << "Data source actor(" << GetAID().Name() << ") sends output data.";
   MS_EXCEPTION_IF_NULL(context);
   // No output.
   if ((output_data_arrows_.size() == 0) && (output_result_arrows_.size() == 0)) {

mindspore/ccsrc/runtime/framework/actor/kernel_actor.cc

@@ -297,6 +297,13 @@ void KernelActor::SendOutput(OpContext<DeviceTensor> *context) const {
     Async(output_data->op_id_, &OpActor::RunOpData, output_data, context);
   }
 
+  // Send graph output result.
+  for (const auto &result_arrow : output_result_arrows_) {
+    MS_EXCEPTION_IF_NULL(result_arrow);
+    Async(result_arrow->to_op_id_, &OutputActor::CollectOutput, kernel_, result_arrow->from_output_index_,
+          result_arrow->to_input_index_, context);
+  }
+
   // Send output control.
   if (output_control_arrows_.size() > 0) {
     auto source_aid = const_cast<AID *>(&GetAID());
@@ -305,13 +312,6 @@ void KernelActor::SendOutput(OpContext<DeviceTensor> *context) const {
     }
   }
 
-  // Send graph output result.
-  for (const auto &result_arrow : output_result_arrows_) {
-    MS_EXCEPTION_IF_NULL(result_arrow);
-    Async(result_arrow->to_op_id_, &OutputActor::CollectOutput, kernel_, result_arrow->from_output_index_,
-          result_arrow->to_input_index_, context);
-  }
-
   // Send recorder info.
   if (recorder_aid_ != nullptr) {
     Async(*recorder_aid_, &RecorderActor::RecordMemAddressInfo, kernel_.get(), &launch_info_, device_context_, context);

mindspore/ccsrc/runtime/framework/device_tensor_store.h

@@ -90,6 +90,8 @@ class DeviceTensorStore {
     return nullptr;
   }
 
+  void Clear() { device_tensors_.clear(); }
+
  private:
   DeviceTensorStore() = default;
   ~DeviceTensorStore() = default;

mindspore/ccsrc/runtime/framework/graph_compiler.cc

@@ -18,7 +18,6 @@
 #include <numeric>
 #include <map>
 #include <utility>
-#include "runtime/framework/graph_scheduler.h"
 #include "runtime/device/device_address.h"
 #include "common/trans.h"
 #include "utils/convert_utils.h"
@@ -253,44 +252,33 @@ void SetSummaryNodesRefCount(const KernelGraph *graph) {
 }
 }  // namespace
 
-void GraphCompiler::set_device_context(DeviceContext *device_context) {
-  MS_EXCEPTION_IF_NULL(device_context);
-  device_context_ = device_context;
-
-  // The member variable 'session_' will be removed after removing session module.
-  if (session_ == nullptr) {
-    session_ = std::make_shared<session::SessionBasic>();
-    const device::DeviceContextKey &device_context_key = device_context->device_context_key();
-    session_->InitExecutor(device_context_key.device_name_, device_context_key.device_id_);
-  }
-}
-
-GraphId GraphCompiler::CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs) {
+GraphId GraphCompiler::CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs,
+                                    const DeviceContext *device_context) {
   MS_EXCEPTION_IF_NULL(session_);
   // Generate kernel graph.
   KernelGraphPtr graph = session_->ConstructKernelGraph(nodes, outputs);
   MS_EXCEPTION_IF_NULL(graph);
-  return CompileGraphImpl(graph);
+  return CompileGraphImpl(graph, device_context);
 }
 
-GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph) const {
+GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph, const DeviceContext *device_context) const {
   MS_EXCEPTION_IF_NULL(graph);
-  MS_EXCEPTION_IF_NULL(device_context_);
+  MS_EXCEPTION_IF_NULL(device_context);
 
   // Execute optimization pass.
-  device_context_->OptimizeGraph(graph);
+  device_context->OptimizeGraph(graph);
 
   // Generate 'KernelMod' for all kernels and set 'KernelMod' into kernel,
   // 'KernelMod' is real executive object of kernel.
-  device_context_->CreateKernel(graph->execution_order());
+  device_context->CreateKernel(graph->execution_order());
 
   // Create device address for all anf nodes of graph.
-  CreateDeviceAddress(graph);
+  CreateDeviceAddress(graph, device_context);
 
   graph->set_is_all_nop_node(opt::IsAllNopNode(graph.get()));
 
   MS_EXCEPTION_IF_NULL(session_);
-  session_->InitAllBucket(graph, device_context_);
+  session_->InitAllBucket(graph, device_context);
 
   session_->SetSummaryNodes(graph.get());
   SetSummaryNodesRefCount(graph.get());
@@ -300,7 +288,7 @@ GraphId GraphCompiler::CompileGraphImpl(const KernelGraphPtr &graph) const {
 
 GraphId GraphCompiler::CompileGraph(const session::OpRunInfo &op_run_info, const GraphInfo &graph_info,
                                     const std::vector<int64_t> *tensors_mask, std::vector<TensorPtr> *input_tensors,
-                                    bool *single_op_cache_hit) {
+                                    bool *single_op_cache_hit, const DeviceContext *device_context) {
   // Check if the graph cache exists.
   auto iter = run_op_graphs_.find(graph_info);
   if (iter != run_op_graphs_.end()) {
@@ -315,18 +303,18 @@ GraphId GraphCompiler::CompileGraph(const session::OpRunInfo &op_run_info, const
   KernelGraphPtr graph = session_->ConstructSingleOpGraph(op_run_info, *input_tensors, *tensors_mask);
   MS_EXCEPTION_IF_NULL(graph);
 
-  MS_EXCEPTION_IF_NULL(device_context_);
-  device_context_->OptimizeSingleOpGraph(graph);
+  MS_EXCEPTION_IF_NULL(device_context);
+  device_context->OptimizeSingleOpGraph(graph);
 
   MS_EXCEPTION_IF_NULL(session_);
   session_->RunOpHideNopNode(graph);
   session_->RunOpRemoveNopNode(graph);
 
   // Generate 'KernelMod' for kernel in graph.
-  device_context_->CreateKernel(graph->execution_order());
+  device_context->CreateKernel(graph->execution_order());
 
   // Create device address for all anf nodes of graph.
-  CreateDeviceAddress(graph);
+  CreateDeviceAddress(graph, device_context);
 
   graph->set_is_all_nop_node(opt::IsAllNopNode(graph.get()));
   run_op_graphs_[graph_info] = graph;
@@ -347,11 +335,11 @@ KernelGraphPtr GraphCompiler::Fetch(const GraphInfo &graph_info) const {
   return iter->second;
 }
 
-void GraphCompiler::CreateDeviceAddress(const KernelGraphPtr &graph) const {
-  CreateParameterDeviceAddress(device_context_, graph);
-  CreateValueNodeDeviceAddress(device_context_, graph);
-  CreateKernelOutputDeviceAddress(device_context_, graph);
-  CreateKernelWorkspaceDeviceAddress(device_context_, graph);
+void GraphCompiler::CreateDeviceAddress(const KernelGraphPtr &graph, const DeviceContext *device_context) const {
+  CreateParameterDeviceAddress(device_context, graph);
+  CreateValueNodeDeviceAddress(device_context, graph);
+  CreateKernelOutputDeviceAddress(device_context, graph);
+  CreateKernelWorkspaceDeviceAddress(device_context, graph);
   UpdateDeviceAddressForInplaceNode(graph);
 }
 

mindspore/ccsrc/runtime/framework/graph_compiler.h

@@ -24,36 +24,32 @@
 #include <map>
 #include "runtime/hardware/device_context.h"
 #include "backend/session/session_basic.h"
+#include "backend/session/session_factory.h"
 #include "ir/tensor.h"
 
 namespace mindspore {
 using device::DeviceContext;
-using mindspore::tensor::TensorPtr;
 using session::CallBackFunc;
 using session::InputTensorInfo;
+using session::KernelGraph;
 using session::KernelWithIndex;
 using session::OpRunInfo;
+using tensor::TensorPtr;
 
 namespace runtime {
 class GraphCompiler {
  public:
-  static GraphCompiler &GetInstance() {
-    static GraphCompiler instance;
-    return instance;
-  }
-
-  // Set device context which is initialized, the function must be called
-  // before using GraphCompiler and after changing device type or device id.
-  void set_device_context(DeviceContext *device_context);
+  GraphCompiler() { session_ = session::SessionFactory::Get().Create(kSessionBasic); }
+  ~GraphCompiler() = default;
 
   // Construct kernel graph from anf nodes list and compile kernel graph in Graph mode,
   // the detailed implementation of compiling graph is in 'CompileGraphImpl'.
-  GraphId CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs);
+  GraphId CompileGraph(const AnfNodePtrList &nodes, const AnfNodePtrList &outputs, const DeviceContext *device_context);
 
   // Construct single op kernel graph and compile the kernel graph in PyNative mode.
   GraphId CompileGraph(const session::OpRunInfo &op_run_info, const GraphInfo &graph_info,
                        const std::vector<int64_t> *tensors_mask, std::vector<TensorPtr> *input_tensors,
-                       bool *single_op_cache_hit);
+                       bool *single_op_cache_hit, const DeviceContext *device_context);
 
   // Get graph by graph id, if not exist return nullptr, used in Graph mode.
   KernelGraphPtr Fetch(GraphId graph_id) const;
@@ -101,25 +97,23 @@ class GraphCompiler {
   void Summary(const std::vector<KernelGraphPtr> &graphs) const;
 
  private:
-  GraphCompiler() = default;
-  ~GraphCompiler() = default;
   DISABLE_COPY_AND_ASSIGN(GraphCompiler);
 
   // The implementation of compiling graph in Graph Mode, including optimizing graph,
   // setting operator info, creating kernel and transforming kernel graph to ActorSet.
-  GraphId CompileGraphImpl(const KernelGraphPtr &graph) const;
+  GraphId CompileGraphImpl(const KernelGraphPtr &graph, const DeviceContext *device_context) const;
 
   // Create device address for all anf nodes of graph.
-  void CreateDeviceAddress(const KernelGraphPtr &graph) const;
-
-  DeviceContext *device_context_{nullptr};
+  void CreateDeviceAddress(const KernelGraphPtr &graph, const DeviceContext *device_context) const;
 
   // Single op kernel graph cache for PyNative mode.
   std::unordered_map<GraphInfo, KernelGraphPtr> run_op_graphs_;
 
   // The member variable 'session_' will be removed after removing session module.
-  session::SessionPtr session_{nullptr};
+  // Now all the GraphCompiler share the same 'session_'.
+  session::SessionPtr session_;
 };
+
 }  // namespace runtime
 }  // namespace mindspore
 #endif  // MINDSPORE_CCSRC_RUNTIME_FRAMEWORK_GRAPH_COMPILER_H_

mindspore/ccsrc/runtime/framework/graph_scheduler.cc

@@ -322,22 +322,35 @@ void PrepareDataForHostDataSourceActor(const std::unordered_map<AnfNodePtr, size
 }
 }  // namespace
 
-GraphScheduler::~GraphScheduler() {
-  // Global maps clear.
-  device_tensor_to_actor_.clear();
-  actor_to_host_queue_.clear();
-  actors_.clear();
+void GraphScheduler::Clear() {
+  // Terminate all actors.
+  auto actorMgr = ActorMgr::GetActorMgrRef();
+  MS_EXCEPTION_IF_NULL(actorMgr);
+  actorMgr->Finalize();
+
+  // Clear the member of DeviceTensorStore.
+  DeviceTensorStore::GetInstance().Clear();
 
-  // Local maps clear.
+  // Clear global maps.
+  actors_.clear();
   actor_name_to_actor_.clear();
+  actor_to_host_queue_.clear();
+  device_tensor_to_actor_.clear();
+
+  // Clear local maps and vectors.
   graph_output_to_actor_.clear();
+  front_node_to_actor_.clear();
+  copy_actors_.clear();
+
+  // Delete the thread pool.
   delete thread_pool_;
   thread_pool_ = nullptr;
 }
 
 void GraphScheduler::Initialize() {
-  // Local maps and vcetors clear.
+  // Local maps and vectors clear.
   graph_output_to_actor_.clear();
+  front_node_to_actor_.clear();
   copy_actors_.clear();
 
   if (init_) {

mindspore/ccsrc/runtime/framework/graph_scheduler.h

@@ -133,9 +133,12 @@ class GraphScheduler {
   }
 
   // 1. Thread pool creating.
-  // 2. The memory manager creating and scheduling.
+  // 2. The global actors creating and scheduling.
   void Initialize();
 
+  // Clear the members.
+  void Clear();
+
   // Transform graph to actor DAG, contains build and link.
   ActorSet *Transform(const GraphCompilerInfo &graph_compiler_info,
                       GraphExecutionStrategy strategy = GraphExecutionStrategy::kPipeline);
@@ -160,7 +163,7 @@ class GraphScheduler {
 
  private:
   GraphScheduler() = default;
-  ~GraphScheduler();
+  ~GraphScheduler() = default;
   DISABLE_COPY_AND_ASSIGN(GraphScheduler);
 
   // Transform the nodes of graph to actors.
@@ -265,18 +268,18 @@ class GraphScheduler {
   // The global maps, only be cleared in the deconstruction.
   std::unordered_map<ActorInfo, ActorSetPtr> actors_;
   std::unordered_map<std::string, OpActor<DeviceTensor> *> actor_name_to_actor_;
-  // Since the control node does not have a backend node, it can only be connected through the relationship between
-  // the front node, so the mapping relationship between the front node and the actor needs to be recorded.
-  std::unordered_map<AnfNodePtr, KernelActorPtr> front_node_to_actor_;
   std::unordered_map<ActorInfo, HostTensorQueuePtr> actor_to_host_queue_;
   // The second element of pair represents the output index of op actor corresponding to the device tensor.
   std::unordered_map<DeviceTensorPtr, GraphOutputPair> device_tensor_to_actor_;
 
-  // The local maps and vectors, will be cleared at the beginning of each graph transform.
-  // The second element of pair represents the output index of op actor corresponding to the graph output front node.
+  // The local maps and vectors, will be cleared at the beginning of each graph transform:
+  // 1.The second element of pair represents the output index of op actor corresponding to the graph output front node.
   std::map<KernelWithIndex, GraphOutputPair, session::KernelWithIndexCmp> graph_output_to_actor_;
-  // Beaceuse the copy actors are built in the link, so need record the all copy actors in the link process to push into
-  // the actor set after link.
+  // 2.Since the control node does not have a backend node, it can only be connected through the relationship between
+  // the front node, so the mapping relationship between the front node and the actor needs to be recorded.
+  std::unordered_map<AnfNodePtr, KernelActorPtr> front_node_to_actor_;
+  // 3.Beaceuse the copy actors are built in the link, so need record the all copy actors in the link process to push
+  // into the actor set after link.
   std::vector<CopyActorPtr> copy_actors_;
 
   // The id of global actor.

mindspore/ccsrc/runtime/hardware/device_context_manager.cc

@@ -25,7 +25,6 @@ void DeviceContextManager::Register(const std::string &device_name, DeviceContex
 }
 
 void DeviceContextManager::ClearDeviceContexts() {
-  std::lock_guard<std::mutex> guard(lock_);
   for (auto &iter : device_contexts_) {
     MS_LOG(INFO) << "Release device " << iter.first;
     MS_EXCEPTION_IF_NULL(iter.second);
@@ -36,7 +35,6 @@ void DeviceContextManager::ClearDeviceContexts() {
 
 DeviceContext *DeviceContextManager::GetOrCreateDeviceContext(const DeviceContextKey &device_context_key) {
   std::string device_context_key_str = device_context_key.ToString();
-  std::lock_guard<std::mutex> guard(lock_);
 
   auto device_context_iter = device_contexts_.find(device_context_key_str);
   if (device_context_iter != device_contexts_.end()) {

mindspore/ccsrc/runtime/hardware/device_context_manager.h

@@ -48,7 +48,6 @@ class DeviceContextManager {
   std::map<std::string, DeviceContextPtr> device_contexts_;
   // The name of device -> DeviceContextCreator.
   std::map<std::string, DeviceContextCreator> device_context_creators_;
-  std::mutex lock_;
 };
 
 class DeviceContextRegister {

mindspore/ccsrc/runtime/hardware/gpu/gpu_device_context.cc

@@ -187,6 +187,12 @@ void GPUDeviceContext::OptimizeGraph(const KernelGraphPtr &graph) const {
   // Optimization pass which is relevant to device type or format.
   OptimizeGraphWithDeviceInfo(graph);
 
+  // Graph kernel fusion optimization
+  if (context::GraphKernelFlags::GetInstance().IsEnableGraphKernel()) {
+    opt::GraphKernelOptimize(graph);
+    graph->SetExecOrderByDefault();
+  }
+
   // Assign the stream and insert the send/recv node for all reduce kernel, so must be the last in the optimizer.
   device::gpu::AssignGpuStream(graph);
 }
@@ -248,12 +254,6 @@ void GPUDeviceContext::FuseOperators(const KernelGraphPtr &graph) const {
   optimizer->AddPassManager(pm);
   (void)optimizer->Optimize(graph);
   graph->SetExecOrderByDefault();
-
-  // Graph kernel fusion optimization
-  if (context::GraphKernelFlags::GetInstance().IsEnableGraphKernel()) {
-    opt::GraphKernelOptimize(graph);
-    graph->SetExecOrderByDefault();
-  }
 }
 
 void GPUDeviceContext::UpdateGraphDynamicShapeAttr(const NotNull<KernelGraphPtr> &graph) const {

mindspore/ccsrc/vm/backend.cc

@@ -252,14 +252,16 @@ MindRTBackend::MindRTBackend(const std::string &backend_name, const std::string
     : Backend(backend_name), device_name_(device_name), device_id_(device_id) {
   auto cut_list = compile::GetMsNonlinearOps();
   graph_partition_ = std::make_shared<GraphPartition>(cut_list, backend_name);
+  graph_compiler_ = std::make_shared<GraphCompiler>();
 }
 
 const ActorInfo &MindRTBackend::CompileGraphs(const FuncGraphPtr &func_graph) {
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
   MS_EXCEPTION_IF_NULL(func_graph);
   FuncGraphPtr root_graph = WrapPrimitives(func_graph);
   MS_EXCEPTION_IF_NULL(root_graph);
   // Register a summary callback function, which is called in the final stages of summary.
-  runtime::GraphCompiler::GetInstance().RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback);
+  graph_compiler_->RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback);
 
   // Compile root graph.
   graph_id_to_device_context_.clear();
@@ -293,6 +295,7 @@ const ActorInfo &MindRTBackend::CompileGraphs(const FuncGraphPtr &func_graph) {
 void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) {
   MS_EXCEPTION_IF_NULL(func_graph);
   MS_EXCEPTION_IF_NULL(graph_partition_);
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
 
   // Split graph to segments.
   const auto &segments = graph_partition_->Partition(func_graph);
@@ -308,12 +311,11 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) {
       }
       MS_LOG(INFO) << "Compile normal segment, the first node: " << segment->nodes_[0]->fullname_with_scope();
 
-      // Get and set the device context.
+      // Get the device context.
       const auto &cur_device_name = GetCNodeTarget(segment->nodes_[0]);
       const auto &device_context =
         device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({cur_device_name, device_id_});
       device_context->Initialize();
-      runtime::GraphCompiler::GetInstance().set_device_context(device_context);
 
       // Transform nodes to inputs and outputs.
       FuncGraphPtr fg;
@@ -322,7 +324,7 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) {
       std::tie(fg, inputs, outputs) = TransformSegmentToAnfGraph(segment->nodes_);
 
       // Compile graph.
-      auto graph_id = runtime::GraphCompiler::GetInstance().CompileGraph(segment->nodes_, outputs);
+      auto graph_id = graph_compiler_->CompileGraph(segment->nodes_, outputs, device_context);
       graph_id_to_device_context_[graph_id] = device_context;
     } else {
       // Compile the cut node.
@@ -337,16 +339,15 @@ void MindRTBackend::CompileGraph(const FuncGraphPtr &func_graph) {
 const ActorInfo &MindRTBackend::CompileGraph(const OpRunInfo &op_run_info, const GraphInfo &graph_info,
                                              const std::vector<int64_t> *tensors_mask,
                                              std::vector<tensor::TensorPtr> *input_tensors) {
-  static DeviceContext *device_context = nullptr;
-  if (!device_context) {
-    device_context = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({device_name_, device_id_});
-    device_context->Initialize();
-    runtime::GraphCompiler::GetInstance().set_device_context(device_context);
-  }
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
+  // Get the device context.
+  const auto &device_context =
+    device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext({device_name_, device_id_});
+  device_context->Initialize();
 
   bool single_op_cache_hit;
-  (void)runtime::GraphCompiler::GetInstance().CompileGraph(op_run_info, graph_info, tensors_mask, input_tensors,
-                                                           &single_op_cache_hit);
+  (void)graph_compiler_->CompileGraph(op_run_info, graph_info, tensors_mask, input_tensors, &single_op_cache_hit,
+                                      device_context);
   if (single_op_cache_hit) {
     return graph_info;
   }
@@ -366,28 +367,28 @@ const ActorInfo &MindRTBackend::CompileGraph(const OpRunInfo &op_run_info, const
 
 void MindRTBackend::RunGraphBySingleOp(const std::vector<KernelGraphPtr> &graphs,
                                        const std::vector<std::vector<tensor::TensorPtr>> &inputs, VectorRef *outputs) {
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
   for (size_t graph_index = 0; graph_index < graphs.size(); ++graph_index) {
     const auto &graph = graphs[graph_index];
     std::map<AnfNodePtr, size_t> parameter_index;
     std::map<KernelWithIndex, std::vector<std::vector<size_t>>> output_indexes;
     std::map<KernelWithIndex, tensor::TensorPtr> op_output_map;
 
-    runtime::GraphCompiler::GetInstance().GetParamAndOutputIndex(graph, inputs[graph_index], outputs, &parameter_index,
-                                                                 &output_indexes);
+    graph_compiler_->GetParamAndOutputIndex(graph, inputs[graph_index], outputs, &parameter_index, &output_indexes);
 
     // Clear bucket resources every step
     if (graph->is_bprop()) {
-      runtime::GraphCompiler::GetInstance().ClearAllBucket(graph->graph_id());
+      graph_compiler_->ClearAllBucket(graph->graph_id());
     }
 
     for (const auto &kernel : graph->execution_order()) {
       OpRunInfo op_run_info;
       GraphInfo graph_info;
       InputTensorInfo input_tensor_info;
-      runtime::GraphCompiler::GetInstance().GetSingleOpInputTensors(kernel, op_output_map, parameter_index,
-                                                                    inputs[graph_index], &input_tensor_info);
-      runtime::GraphCompiler::GetInstance().GetSingleOpRunInfoAndGraphInfo(kernel, input_tensor_info.input_tensors,
-                                                                           &op_run_info, &graph_info);
+      graph_compiler_->GetSingleOpInputTensors(kernel, op_output_map, parameter_index, inputs[graph_index],
+                                               &input_tensor_info);
+      graph_compiler_->GetSingleOpRunInfoAndGraphInfo(kernel, input_tensor_info.input_tensors, &op_run_info,
+                                                      &graph_info);
 
       const ActorInfo &actor_info =
         CompileGraph(op_run_info, graph_info, &input_tensor_info.input_tensors_mask, &input_tensor_info.input_tensors);
@@ -395,12 +396,12 @@ void MindRTBackend::RunGraphBySingleOp(const std::vector<KernelGraphPtr> &graphs
         RunGraph(actor_info, &op_run_info, &input_tensor_info.input_tensors_mask, &input_tensor_info.input_tensors);
 
       std::vector<tensor::TensorPtr> new_output_tensors;
-      runtime::GraphCompiler::GetInstance().RecoverGraphOutput(kernel, op_outputs, output_indexes, &op_output_map,
-                                                               outputs, &new_output_tensors);
+      graph_compiler_->RecoverGraphOutput(kernel, op_outputs, output_indexes, &op_output_map, outputs,
+                                          &new_output_tensors);
 
       // Save grad node to Bucket
       if (graph->is_bprop()) {
-        runtime::GraphCompiler::GetInstance().AddGradAddrToBucket(graph->graph_id(), new_output_tensors);
+        graph_compiler_->AddGradAddrToBucket(graph->graph_id(), new_output_tensors);
       }
     }
   }
@@ -490,18 +491,19 @@ VectorRef MindRTBackend::RunGraph(const ActorInfo &actor_info, const VectorRef &
   }
   MS_LOG(INFO) << "Run actor end, actor name: " << actor_info;
 
-  runtime::GraphCompiler::GetInstance().Summary(graph_compiler_info.graphs_);
+  graph_compiler_->Summary(graph_compiler_info.graphs_);
   return outputs;
 }
 
 std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(const FuncGraphPtr &root_graph) {
   MS_EXCEPTION_IF_NULL(root_graph);
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
 
   std::vector<KernelGraphPtr> graphs;
   std::vector<DeviceContext *> device_contexts;
   std::string name = "kernel_graph";
   for (const auto &graph_id_to_context : graph_id_to_device_context_) {
-    graphs.emplace_back(runtime::GraphCompiler::GetInstance().Fetch(graph_id_to_context.first));
+    graphs.emplace_back(graph_compiler_->Fetch(graph_id_to_context.first));
     device_contexts.emplace_back(graph_id_to_context.second);
     name.append("_").append(std::to_string(graph_id_to_context.first));
   }
@@ -519,6 +521,10 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(con
       for (const auto &output : outputs) {
         const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(output, 0, false);
         MS_EXCEPTION_IF_NULL(output_with_index.first);
+        // The InitDataSetQueue kernel has no output.
+        if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) {
+          continue;
+        }
         outputs_order.emplace(output_with_index, position++);
       }
     } else if (branch_output->isa<CNode>()) {
@@ -526,6 +532,10 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(con
       for (size_t i = 0; i < outputs_num; i++) {
         const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(branch_output, i, false);
         MS_EXCEPTION_IF_NULL(output_with_index.first);
+        // The InitDataSetQueue kernel has no output.
+        if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) {
+          continue;
+        }
         outputs_order.emplace(output_with_index, position++);
       }
     }
@@ -546,6 +556,7 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(con
 
 std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(
   const std::vector<int64_t> *tensors_mask, const std::vector<tensor::TensorPtr> *input_tensors) {
+  MS_EXCEPTION_IF_NULL(graph_compiler_);
   std::vector<KernelGraphPtr> graphs;
   std::vector<DeviceContext *> device_contexts;
   runtime::KernelMapPosition outputs_order;
@@ -553,7 +564,7 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(
   std::string name;
 
   for (const auto &graph_info_to_context : graph_info_to_device_context_) {
-    const auto &graph = runtime::GraphCompiler::GetInstance().Fetch(graph_info_to_context.first);
+    const auto &graph = graph_compiler_->Fetch(graph_info_to_context.first);
     graphs.emplace_back(graph);
     device_contexts.emplace_back(graph_info_to_context.second);
     name.append(graph_info_to_context.first);
@@ -562,6 +573,10 @@ std::unique_ptr<GraphCompilerInfo> MindRTBackend::ConstructGraphCompilerInfo(
     for (const auto &output : outputs) {
       const auto &output_with_index = AnfAlgo::VisitKernelWithReturnType(output, 0, false);
       MS_EXCEPTION_IF_NULL(output_with_index.first);
+      // The InitDataSetQueue kernel has no output.
+      if (AnfAlgo::GetCNodeName(output_with_index.first) == kInitDatasetQueueOpName) {
+        continue;
+      }
       outputs_order.emplace(output_with_index, position++);
     }
   }

mindspore/ccsrc/vm/backend.h

@@ -37,6 +37,7 @@ namespace compile {
 using OpRunInfo = session::OpRunInfo;
 using DeviceContext = device::DeviceContext;
 using ActorInfo = runtime::ActorInfo;
+using GraphCompiler = runtime::GraphCompiler;
 using GraphCompilerInfo = runtime::GraphCompilerInfo;
 using ControlNodeParser = runtime::ControlNodeParser;
 using FrontToBackendNodeWithContext = runtime::FrontToBackendNodeWithContext;
@@ -143,6 +144,7 @@ class MindRTBackend : public Backend {
   std::unordered_map<ActorInfo, std::unique_ptr<GraphCompilerInfo>> actor_to_graph_compiler_info_;
 
   GraphPartitionPtr graph_partition_;
+  std::shared_ptr<GraphCompiler> graph_compiler_;
   std::string device_name_;
   uint32_t device_id_;
 };