!15590 fix addr is null

From: @jjfeing
Reviewed-by: @zhoufeng54,@kisnwang
Signed-off-by: @kisnwang

mindspore/ccsrc/backend/session/session_basic.cc

@@ -52,8 +52,6 @@
 
 namespace mindspore {
 namespace session {
-static std::shared_ptr<std::map<ParamInfoPtr, ParameterPtr>> python_paras;
-void ClearPythonParasMap() { python_paras = nullptr; }
 namespace {
 const int kSummaryGetItem = 2;
 const size_t max_depth = 128;
@@ -681,19 +679,18 @@ ParameterPtr SessionBasic::CreateNewParameterFromParameter(const AnfNodePtr &anf
   MS_EXCEPTION_IF_NULL(graph_inputs);
   ParameterPtr new_parameter = nullptr;
   // if parameter's python parameter has been exist a backend parameter, reuse the exist parameter
-  if (python_paras == nullptr) {
-    python_paras = std::make_shared<std::map<ParamInfoPtr, ParameterPtr>>();
-  }
-  auto iter = python_paras->find(param_value);
-  if (iter != python_paras->end()) {
-    new_parameter = iter->second;
+  if (param_value != nullptr) {
+    new_parameter = param_value->parameter();
+    if (new_parameter == nullptr) {
+      TraceGuard trace_guard(std::make_shared<TraceCopy>(anf->debug_info()));
+      new_parameter = graph->NewParameter(anf->cast<ParameterPtr>());
+      param_value->set_parameter(new_parameter);
+    }
   } else {
     TraceGuard trace_guard(std::make_shared<TraceCopy>(anf->debug_info()));
     new_parameter = graph->NewParameter(anf->cast<ParameterPtr>());
-    if (param_value != nullptr) {
-      (*python_paras)[param_value] = new_parameter;
-    }
   }
+
   new_parameter->IncreaseUsedGraphCount();
   graph_inputs->push_back(new_parameter);
   valid_inputs->push_back(true);
@@ -1126,10 +1123,10 @@ ValueNodePtr SessionBasic::CreateValueNodeKernelGraph(const AnfNodePtr &anf, Ker
   MS_EXCEPTION_IF_NULL(value_node);
   auto sub_func_graph = AnfAlgo::GetValueNodeFuncGraph(anf);
   MS_EXCEPTION_IF_NULL(sub_func_graph);
-  if (front_backend_graph_map_.find(sub_func_graph) == front_backend_graph_map_.end()) {
+  if (front_backend_graph_map_.find(sub_func_graph.get()) == front_backend_graph_map_.end()) {
     MS_LOG(EXCEPTION) << "FuncGraph: " << sub_func_graph->ToString() << " has not been transformed to KernelGraph.";
   }
-  auto sub_kernel_graph = front_backend_graph_map_[sub_func_graph];
+  auto sub_kernel_graph = front_backend_graph_map_[sub_func_graph.get()];
 
   ValueNodePtr new_value_node = std::make_shared<ValueNode>(sub_kernel_graph);
   new_value_node->set_abstract(value_node->abstract());
@@ -1155,19 +1152,19 @@ ParameterPtr SessionBasic::CreateNewParameter(const AnfNodePtr &anf, KernelGraph
 
   auto param_value = GetParamDefaultValue(anf);
   ParameterPtr new_parameter = nullptr;
-  if (python_paras == nullptr) {
-    python_paras = std::make_shared<std::map<ParamInfoPtr, ParameterPtr>>();
-  }
-  auto iter = python_paras->find(param_value);
-  if (iter != python_paras->end()) {
-    new_parameter = iter->second;
+  // if parameter's python parameter has been exist a backend parameter, reuse the exist parameter
+  if (param_value != nullptr) {
+    new_parameter = param_value->parameter();
+    if (new_parameter == nullptr) {
+      TraceGuard trace_guard(std::make_shared<TraceCopy>(anf->debug_info()));
+      new_parameter = graph->NewParameter(anf->cast<ParameterPtr>());
+      param_value->set_parameter(new_parameter);
+    }
   } else {
     TraceGuard trace_guard(std::make_shared<TraceCopy>(anf->debug_info()));
     new_parameter = graph->NewParameter(anf->cast<ParameterPtr>());
-    if (param_value != nullptr) {
-      (*python_paras)[param_value] = new_parameter;
-    }
   }
+
   new_parameter->IncreaseUsedGraphCount();
 
   return new_parameter;
@@ -1423,7 +1420,7 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
   auto node_list = TopoSort(func_graph->get_return());
   auto graph = NewKernelGraph();
   MS_EXCEPTION_IF_NULL(graph);
-  front_backend_graph_map_[func_graph] = graph;
+  front_backend_graph_map_[func_graph.get()] = graph;
   MS_LOG(INFO) << "Create graph: " << graph->graph_id();
   for (const auto &node : node_list) {
     MS_EXCEPTION_IF_NULL(node);
@@ -1446,15 +1443,15 @@ std::shared_ptr<KernelGraph> SessionBasic::ConstructKernelGraph(const FuncGraphP
       }
       // Create child kernel graph according ValueNode<FuncGraph>
       FuncGraphPtr child_graph = AnfAlgo::GetValueNodeFuncGraph(node);
-      if (front_backend_graph_map_.find(child_graph) == front_backend_graph_map_.end()) {
+      if (front_backend_graph_map_.find(child_graph.get()) == front_backend_graph_map_.end()) {
         (void)ConstructKernelGraph(child_graph, all_out_graph);
       }
       (void)CreateValueNodeKernelGraph(node, graph.get());
-      auto &parent_graph = parent_graphs_[front_backend_graph_map_[child_graph]->graph_id()];
+      auto &parent_graph = parent_graphs_[front_backend_graph_map_[child_graph.get()]->graph_id()];
       auto parent_graph_it =
-        std::find(parent_graph.begin(), parent_graph.end(), front_backend_graph_map_[func_graph]->graph_id());
+        std::find(parent_graph.begin(), parent_graph.end(), front_backend_graph_map_[func_graph.get()]->graph_id());
       if (parent_graph_it == parent_graph.end()) {
-        parent_graph.push_back(front_backend_graph_map_[func_graph]->graph_id());
+        parent_graph.push_back(front_backend_graph_map_[func_graph.get()]->graph_id());
       }
       continue;
     }

mindspore/ccsrc/backend/session/session_basic.h

@@ -41,7 +41,6 @@ namespace mindspore {
 using GraphId = uint32_t;
 using GraphInfo = std::string;
 namespace session {
-void ClearPythonParasMap();
 using CallBackFunc = uint32_t (*)(uint32_t graph_id,
                                   const std::map<std::string, mindspore::tensor::TensorPtr> &params_list);
 using AnyList = std::vector<Any>;
@@ -254,7 +253,7 @@ class SessionBasic : public std::enable_shared_from_this<SessionBasic> {
   std::map<uint32_t, uint32_t> free_bucket_id_map_;
   std::unordered_map<GraphId, std::shared_ptr<KernelGraph>> graphs_;
   std::unordered_map<GraphInfo, std::shared_ptr<KernelGraph>> run_op_graphs_;
-  std::unordered_map<FuncGraphPtr, KernelGraphPtr> front_backend_graph_map_;
+  std::unordered_map<FuncGraph *, KernelGraphPtr> front_backend_graph_map_;
   std::unordered_map<GraphId, std::vector<GraphId>> parent_graphs_;
   std::shared_ptr<Context> context_;
   CallBackFunc summary_callback_;

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -1137,7 +1137,6 @@ void FinalizeBackend() {
 void ClearResAtexit() {
   MS_LOG(DEBUG) << "Pipeline clear all resource";
   pynative::ClearPyNativeSession();
-  session::ClearPythonParasMap();
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
   if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) {
     if (ps::PsDataPrefetch::GetInstance().cache_enable()) {

mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc

@@ -41,8 +41,9 @@ void AscendMemoryManager::MallocDeviceMemory() {
       auto context_ptr = MsContext::GetInstance();
       MS_EXCEPTION_IF_NULL(context_ptr);
       unsigned int device_id = context_ptr->get_param<uint32_t>(MS_CTX_DEVICE_ID);
-      MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, malloc device memory failed, size["
-                        << device_mem_size_ << "], ret[" << ret << "]";
+      MS_LOG(EXCEPTION) << "Malloc device memory failed, size[" << device_mem_size_ << "], ret[" << ret << "]"
+                        << "Device " << device_id
+                        << " may be other processes occupying this card, check as: ps -ef|grep python";
     } else {
       MS_EXCEPTION(DeviceProcessError) << "rtMalloc mem size[" << device_mem_size_ << "] fail, ret[" << ret << "]";
     }

mindspore/ccsrc/runtime/device/kernel_runtime.cc

@@ -513,7 +513,7 @@ void KernelRuntime::AssignCommunicationNodeInputMem(MemType type, const AnfNodeP
   std::vector<std::pair<DeviceAddressPtr, size_t>> addr_size;
   size_t input_num = AnfAlgo::GetInputTensorNum(node);
   for (size_t i = 0; i < input_num; ++i) {
-    auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, i);
+    auto input_node_with_index = AnfAlgo::GetPrevNodeOutput(node, i, true);
     auto input_node = input_node_with_index.first;
     if (AnfAlgo::OutputAddrExist(input_node, input_node_with_index.second)) {
       MS_LOG(INFO) << "Communication op " << input_node->fullname_with_scope() << " has input device address";

mindspore/core/ir/param_info.h

@@ -86,6 +86,8 @@ class ParamInfo {
 
   std::vector<int64_t> cache_shape() const { return cache_shape_; }
   void set_cache_shape(const std::vector<int64_t> &cache_shape) { cache_shape_ = cache_shape; }
+  ParameterPtr parameter() { return parameter_; }
+  void set_parameter(const ParameterPtr &parameter) { parameter_ = parameter; }
 
  private:
   std::string name_{"Parameter"};
@@ -100,6 +102,7 @@ class ParamInfo {
   bool parallel_optimizer_{true};
   bool cache_enable_{false};
   std::vector<int64_t> cache_shape_;
+  ParameterPtr parameter_{nullptr};
 };
 }  // namespace mindspore
 #endif  // MINDSPORE_CORE_IR_PARAM_INFO_H_

mindspore/ops/op_info_register.py

@@ -451,7 +451,7 @@ class TBERegOp(RegOp):
         Whether the operator need calop_select_format api.
 
         Args:
-            is_dynamic_format (bool): Value of is_dynamic_format_. Default: false.
+            is_dynamic_format (bool): Value of is_dynamic_format. Default: false.
         """
         self._is_bool(is_dynamic_format)
         self.is_dynamic_format_ = is_dynamic_format