!5866 clean idle mem at proper time

Merge pull request !5866 from liangzelang/fix_global_step_error

mindspore/ccsrc/backend/session/ascend_session.cc

@@ -1014,6 +1014,7 @@ void AscendSession::AssignStaticMemory(NotNull<KernelGraphPtr> graph,
   // assign static memory for parameters
   auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_);
   MS_EXCEPTION_IF_NULL(runtime_instance);
+  runtime_instance->ClearGlobalIdleMem();
   runtime_instance->AssignStaticMemoryInput(graph.get().get());
   runtime_instance->AssignStaticMemoryValueNode(graph.get().get());
   for (auto &child_graph : graph->child_graph_order()) {

mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc

@@ -155,6 +155,8 @@ void AscendKernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id, const std
   }
 }
 
+void AscendKernelRuntime::ClearGlobalIdleMem() { mem_manager_->ClearGlobalIdleMem(); }
+
 bool AscendKernelRuntime::NeedDestroyHccl() {
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);

mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h

@@ -49,6 +49,7 @@ class AscendKernelRuntime : public KernelRuntime {
   void ClearGraphRuntimeResource(uint32_t graph_id, const std::vector<AnfNodePtr> &inputs,
                                  const std::unordered_set<ValueNodePtr> &value_nodes,
                                  const std::vector<CNodePtr> &execution_order) override;
+  void ClearGlobalIdleMem() override;
   bool SyncStream() override;
 
  protected:

mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc

@@ -77,6 +77,8 @@ void AscendMemoryManager::ResetDynamicMemory() {
   AscendMemoryPool::GetInstance().set_graph_dynamic_mem_offset(dynamic_mem_offset_);
 }
 
+void AscendMemoryManager::ClearGlobalIdleMem() { AscendMemoryPool::GetInstance().ResetIdleMemBuf(); }
+
 void *AscendMemoryManager::MallocMemFromMemPool(size_t size) {
   auto align_size = GetCommonAlignSize(size);
   return AscendMemoryPool::GetInstance().AllocTensorMem(align_size);

mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.h

@@ -28,6 +28,7 @@ class AscendMemoryManager : public MemoryManager {
   void MallocDeviceMemory() override;
   void FreeDeviceMemory() override;
   void ResetDynamicMemory() override;
+  void ClearGlobalIdleMem() override;
   void *MallocMemFromMemPool(size_t size) override;
 
  protected:

mindspore/ccsrc/runtime/device/kernel_runtime.h

@@ -74,6 +74,7 @@ class KernelRuntime {
                                   const std::unordered_set<ValueNodePtr> &value_nodes,
                                   const std::vector<CNodePtr> &execution_order);
   virtual bool SyncStream() = 0;
+  virtual void ClearGlobalIdleMem() {}
 
 #ifdef ENABLE_DUMP_E2E
   DumpConfPtr GetDumpConf();

mindspore/ccsrc/runtime/device/memory_manager.h

@@ -39,6 +39,7 @@ class MemoryManager {
     total_dynamic_size_ = 0;
     dynamic_mem_offset_ = 0;
   }
+  virtual void ClearGlobalIdleMem() {}
 
   void MallocReusedDynamicMem(const session::KernelGraph *graph);
   uint8_t *MallocOutputMem(const AnfNodePtr &node, size_t index, MemType type, size_t size,