diff --git a/ge/graph/load/new_model_manager/data_dumper.cc b/ge/graph/load/new_model_manager/data_dumper.cc index b33a062d..f7f23dc1 100644 --- a/ge/graph/load/new_model_manager/data_dumper.cc +++ b/ge/graph/load/new_model_manager/data_dumper.cc @@ -830,6 +830,13 @@ Status DataDumper::UnloadDumpInfo() { return SUCCESS; } +void DataDumper::DumpShrink() { + compute_graph_.reset(); + input_map_.clear(); + ref_info_.clear(); + op_list_.clear(); +} + void DataDumper::PrintCheckLog(string &dump_list_key) { std::set model_list = dump_properties_.GetAllDumpModel(); if (model_list.empty()) { diff --git a/ge/graph/load/new_model_manager/data_dumper.h b/ge/graph/load/new_model_manager/data_dumper.h index 46ead310..8e612688 100755 --- a/ge/graph/load/new_model_manager/data_dumper.h +++ b/ge/graph/load/new_model_manager/data_dumper.h @@ -83,6 +83,8 @@ class DataDumper { Status UnloadDumpInfo(); + void DumpShrink(); + void SetDumpProperties(const DumpProperties &dump_properties) { dump_properties_ = dump_properties; } const DumpProperties &GetDumpProperties() const { return dump_properties_; } bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const; @@ -112,18 +114,18 @@ class DataDumper { struct InnerInputMapping; std::vector op_desc_info_; - std::vector op_list_; + std::vector op_list_; // release after DavinciModel::Init uint32_t end_graph_task_id_ = 0; uint32_t end_graph_stream_id_ = 0; bool is_end_graph_ = false; - std::multimap input_map_; + std::multimap input_map_; // release after DavinciModel::Init bool load_flag_; uint32_t device_id_; uintptr_t global_step_; uintptr_t loop_per_iter_; uintptr_t loop_cond_; - ComputeGraphPtr compute_graph_; - std::map ref_info_; + ComputeGraphPtr compute_graph_; // release after DavinciModel::Init + std::map ref_info_; // release after DavinciModel::Init void *l1_fusion_addr_ = nullptr; diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index 232bcde7..eb12351f 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -148,14 +148,7 @@ DavinciModel::~DavinciModel() { GELOGW("UnloadDumpInfo failed, ret: %u.", ret); } - for (const auto &op_and_addr : saved_task_addrs_) { - auto addr = op_and_addr.second; - if (addr != nullptr) { - GE_CHK_RT(rtFree(addr)); - } - addr = nullptr; - } - saved_task_addrs_.clear(); + ClearTaskAddrs(); GE_CHK_STATUS(ModelRunStop()); @@ -220,6 +213,17 @@ DavinciModel::~DavinciModel() { } } +void DavinciModel::ClearTaskAddrs() { + for (const auto &op_and_addr : saved_task_addrs_) { + auto addr = op_and_addr.second; + if (addr != nullptr) { + GE_CHK_RT(rtFree(addr)); + } + addr = nullptr; + } + saved_task_addrs_.clear(); +} + void DavinciModel::UnbindHcomStream() { if (!all_hccl_stream_list_.empty()) { for (size_t i = 0; i < all_hccl_stream_list_.size(); i++) { @@ -261,7 +265,10 @@ Status DavinciModel::Assign(const GeModelPtr &ge_model) { /// @return: void /// void DavinciModel::Shrink() { + //DumperShrink(); ge_model_.reset(); // delete object. + //op_list_.clear(); + //ClearTaskAddrs(); } Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weight_size) { @@ -591,6 +598,7 @@ void DavinciModel::OpDebugUnRegister() { // initialize op sequence and call initialization function of each op respectively Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) { // validating params + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(priority_ < 0 || priority_ > 7, return PARAM_INVALID, "Priority must between 0-7, now is %d", priority_); GE_CHK_BOOL_RET_STATUS(ge_model_ != nullptr, PARAM_INVALID, "GeModel is null."); @@ -735,7 +743,6 @@ Status DavinciModel::ReportProfilingData() { } ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); - op_list_.clear(); return SUCCESS; } @@ -934,11 +941,15 @@ Status DavinciModel::InitInputOutputForDynamic(const ComputeGraphPtr &compute_gr } if (IsDataOp(op_desc->GetType())) { GELOGD("init data op %s", op_desc->GetName().c_str()); - data_op_list_.push_back(op_desc); + auto data_op = AttrUtils::CopyOpDesc(op_desc); + GE_CHECK_NOTNULL(data_op); + data_op_list_.push_back(data_op); } if (op_desc->GetType() == NETOUTPUT) { GELOGD("init netouput op %s", op_desc->GetName().c_str()); - output_op_list_.push_back(op_desc); + auto output_op = AttrUtils::CopyOpDesc(op_desc); + GE_CHECK_NOTNULL(output_op); + output_op_list_.push_back(output_op); } } return SUCCESS; @@ -976,7 +987,9 @@ Status DavinciModel::InitDataOp(const NodePtr &node, uint32_t &data_op_index, ma return SUCCESS; } - data_op_list_.push_back(op_desc); + auto data_op = AttrUtils::CopyOpDesc(op_desc); + GE_CHECK_NOTNULL(data_op); + data_op_list_.push_back(data_op); // Make information for copy input data. const vector output_size_list = ModelUtils::GetOutputSize(op_desc); @@ -1031,7 +1044,9 @@ void DavinciModel::AdjustDataOpList(const map &data_by_inde data_op_list_.clear(); for (auto &item : data_by_index) { - data_op_list_.emplace_back(item.second); + auto data_op = AttrUtils::CopyOpDesc(item.second); + GE_CHECK_NOTNULL_EXEC(data_op, return); + data_op_list_.emplace_back(data_op); } } @@ -1063,7 +1078,9 @@ Status DavinciModel::InitNetOutput(const NodePtr &node) { return SUCCESS; } - output_op_list_.push_back(op_desc); + auto output_op = AttrUtils::CopyOpDesc(op_desc); + GE_CHECK_NOTNULL(output_op); + output_op_list_.push_back(output_op); // Make information for copy output data. const vector input_size_list = ModelUtils::GetInputSize(op_desc); const vector virtual_addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc); diff --git a/ge/graph/load/new_model_manager/davinci_model.h b/ge/graph/load/new_model_manager/davinci_model.h index be8efd90..d5b04d54 100755 --- a/ge/graph/load/new_model_manager/davinci_model.h +++ b/ge/graph/load/new_model_manager/davinci_model.h @@ -458,6 +458,10 @@ class DavinciModel { data_dumper_.SaveDumpTask(task_id, stream_id, op_desc, args); } + void DumperShrink() { + data_dumper_.DumpShrink(); + } + void SetEndGraphId(uint32_t task_id, uint32_t stream_id); DavinciModel &operator=(const DavinciModel &model) = delete; @@ -620,6 +624,8 @@ class DavinciModel { void ReleaseTask(); + void ClearTaskAddrs(); + void UnbindTaskSinkStream(); bool IsAicpuKernelConnectSpecifiedLayer(); @@ -842,12 +848,12 @@ class DavinciModel { string om_name_; uint32_t version_; - GeModelPtr ge_model_; + GeModelPtr ge_model_; // release after DavinciModel::Init bool need_destroy_aicpu_kernel_{false}; vector out_node_name_; - map op_list_; + map op_list_; // release after DavinciModel::Init // data op_desc vector data_op_list_; @@ -944,7 +950,7 @@ class DavinciModel { DataDumper data_dumper_; uint64_t iterator_count_; bool is_l1_fusion_enable_; - std::map saved_task_addrs_; + std::map saved_task_addrs_; // release after DavinciModel::Init void *l1_fusion_addr_ = nullptr; bool known_node_ = false; diff --git a/metadef b/metadef index c14d2be3..03ad8a28 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit c14d2be38171eed63416e71178774103faf1f5cd +Subproject commit 03ad8a2899d7ac4f06e3ce5e21f533a48aa77895 diff --git a/tests/depends/error_manager/src/error_manager_stub.cc b/tests/depends/error_manager/src/error_manager_stub.cc index edf5a487..678e1252 100644 --- a/tests/depends/error_manager/src/error_manager_stub.cc +++ b/tests/depends/error_manager/src/error_manager_stub.cc @@ -58,7 +58,7 @@ /// @param [in] value: vector parameter value /// void ErrorManager::ATCReportErrMessage(std::string error_code, const std::vector &key, - const std::vector &value) { + const std::vector &value) { } /// @@ -66,13 +66,8 @@ /// @param [in] msg: failed message map, key is error code, value is op_name /// @return int 0(success) -1(fail) /// - int ErrorManager::ReportMstuneCompileFailedMsg(const std::map &msg) { return 0; } - - /// - /// @brief save graph compile failed message from thread local map to global map - /// @param [in] graph_name: graph name - /// - void ErrorManager::SaveMstuneCompileFailedMsg(const std::string &graph_name) {} + int ErrorManager::ReportMstuneCompileFailedMsg(const std::string &root_graph_name, + const std::map &msg) { return 0; } /// /// @brief get graph compile failed message in mstune case