diff --git a/ge/graph/load/new_model_manager/davinci_model.cc b/ge/graph/load/new_model_manager/davinci_model.cc index 4cc4e692..d179d48e 100755 --- a/ge/graph/load/new_model_manager/davinci_model.cc +++ b/ge/graph/load/new_model_manager/davinci_model.cc @@ -1761,7 +1761,7 @@ Status DavinciModel::InitAIPPInfo(uint32_t index, const OpDescPtr &op_desc) { "Data node do not contain param aipp!"); GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, &aipp_params), "get aipp params failed"); GELOGI("GetAIPPInfo: node data: %s, type: %s, current index: %u, current node related input rank: %u", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, aipp_params->related_input_rank()); + op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, aipp_params.related_input_rank()); AippConfigInfo aipp_info; GE_CHK_STATUS_RET(AippUtils::ConvertAippParams2AippInfo(&aipp_params, aipp_info), diff --git a/ge/graph/load/new_model_manager/davinci_model.cc.bak b/ge/graph/load/new_model_manager/davinci_model.cc.bak deleted file mode 100644 index 7d719e7b..00000000 --- a/ge/graph/load/new_model_manager/davinci_model.cc.bak +++ /dev/null @@ -1,4054 +0,0 @@ -/** - * Copyright 2019-2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "graph/load/new_model_manager/davinci_model.h" - -#include -#include -#include -#include - -#include "common/debug/log.h" -#include "common/formats/formats.h" -#include "common/formats/utils/formats_trans_utils.h" -#include "common/math/math_util.h" -#include "common/op/ge_op_utils.h" -#include "common/profiling/profiling_manager.h" -#include "common/properties_manager.h" -#include "common/scope_guard.h" -#include "common/thread_pool.h" -#include "framework/common/debug/ge_log.h" -#include "graph/common/ge_call_wrapper.h" -#include "graph/compute_graph.h" -#include "graph/debug/ge_attr_define.h" -#include "graph/ge_context.h" -#include "graph/graph.h" -#include "graph/load/new_model_manager/cpu_queue_schedule.h" -#include "graph/load/new_model_manager/model_manager.h" -#include "graph/load/new_model_manager/tbe_handle_store.h" -#include "graph/manager/graph_mem_allocator.h" -#include "graph/manager/graph_var_manager.h" -#include "graph/manager/trans_var_data_utils.h" -#include "graph/manager/util/debug.h" -#include "graph/model_serialize.h" -#include "graph/node.h" -#include "graph/utils/graph_utils.h" -#include "graph/utils/type_utils.h" -#include "init/gelib.h" -#include "mmpa/mmpa_api.h" -#include "omm/csa_interact.h" -#include "runtime/base.h" -#include "runtime/dev.h" -#include "runtime/event.h" -#include "runtime/mem.h" -#include "runtime/rt_model.h" -#include "runtime/stream.h" -#include "securec.h" -#include "graph/common/local_context.h" -#include "common/formats/utils/formats_trans_utils.h" - -// create std::thread, catch exceptions using try/catch -#define CREATE_STD_THREAD(thread_id, func, args) \ - do { \ - try { \ - thread_id = std::thread(func, args); \ - } catch (const std::system_error &e) { \ - GELOGE(FAILED, "Caught system_error with code:%d, meaning:%s", e.code().value(), e.what()); \ - GELOGE(FAILED, "Thread creat FAIL, Please check the left resource!"); \ - return FAILED; \ - } \ - } while (0) - -namespace ge { -namespace { -const uint32_t kDataIndex = 0; -const uint32_t kOutputNum = 1; -const uint32_t kTrueBranchStreamNum = 1; -const uint32_t kGetDynamicDimsCount = 1; -const uint32_t kThreadNum = 16; -const uint32_t kAddrLen = sizeof(void *); -const int kDecimal = 10; -const int kBytes = 8; -const uint32_t kDataMemAlignSizeCompare = 64; -const uint32_t kDumpL1FusionOpMByteSize = 2097152; // 2 * 1024 * 1024 -const uint32_t kDumpFlagOfL1Fusion = 0; -const char *const kDefaultBatchLable = "Batch_default"; -const char *const kGetDynamicDimsName = "ascend_mbatch_get_dynamic_dims_node"; -const int32_t kInvalidStream = -1; -const uint32_t kEndOfSequence = 0x0704000a; -const uint32_t kEndOfSequenceNew = 507005; -const int32_t kModelAbortNormal = 0x0704000e; -const int32_t kModelAbortNormalNew = 507024; - -inline bool IsDataOp(const std::string &node_type) { - return node_type == DATA_TYPE || node_type == AIPP_DATA_TYPE || node_type == ANN_DATA_TYPE; -} -inline bool IsNoTaskAndDumpNeeded(const OpDescPtr &op_desc) { - bool save_dump_info = false; - (void)ge::AttrUtils::GetBool(op_desc, ATTR_NO_TASK_AND_DUMP_NEEDED, save_dump_info); - return save_dump_info; -} -} // namespace - -std::mutex DavinciModel::tvm_bin_mutex_; - -DavinciModel::DavinciModel(int32_t priority, const std::shared_ptr &listener) - : weights_mem_base_(nullptr), - var_mem_base_(nullptr), - fixed_mem_base_(0), - mem_base_(nullptr), - is_inner_mem_base_(false), - is_inner_weight_base_(false), - is_inner_p2p_mem_base_(false), - data_inputer_(nullptr), - load_begin_time_(0), - load_end_time_(0), - time_info_(), - dataInputTid(0), - is_weight_mem_has_inited_(false), - is_feature_map_mem_has_inited_(false), - model_id_(0), - runtime_model_id_(0), - version_(0), - ge_model_(nullptr), - thread_id_(), - listener_(listener), - run_flg_(false), - priority_(priority), - rt_model_handle_(nullptr), - rt_model_stream_(nullptr), - is_inner_model_stream_(false), - is_async_mode_(false), - last_execute_mode_(INITIALIZATION), - session_id_(0), - device_id_(0), - maxDumpOpNum_(0), data_dumper_(runtime_param_), - iterator_count_(0), - is_l1_fusion_enable_(false), - is_first_execute_(true) { - op_list_.clear(); - skt_info_ = {0, 0, 0, 0, nullptr, nullptr, {}, {}, {}, {}, {}, RT_KERNEL_DEFAULT, -1, 0, nullptr}; -} - -DavinciModel::~DavinciModel() { - try { - Status ret = data_dumper_.UnloadDumpInfo(); - if (ret != SUCCESS) { - GELOGW("UnloadDumpInfo failed, ret: %u.", ret); - } - - for (const auto &op_and_addr : saved_task_addrs_) { - auto addr = op_and_addr.second; - if (addr != nullptr) { - GE_CHK_RT(rtFree(addr)); - } - addr = nullptr; - } - saved_task_addrs_.clear(); - - GE_CHK_STATUS(ModelRunStop()); - - op_list_.clear(); - data_op_list_.clear(); - tensor_name_to_fixed_addr_size_.clear(); - tensor_name_to_peer_output_index_.clear(); - GE_DELETE_NEW_SINGLE(data_inputer_); - // check rt ctx is exist. rt api call will cause error log when ctx not exist - rtContext_t ctx = nullptr; - rtError_t rt_ret = rtCtxGetCurrent(&ctx); - if (rt_ret == RT_ERROR_NONE) { - UnbindTaskSinkStream(); - for (size_t i = 0; i < label_list_.size(); ++i) { - if (label_list_[i] != nullptr) { - GE_LOGW_IF(rtLabelDestroy(label_list_[i]) != RT_ERROR_NONE, "Destroy label failed, index: %zu", i); - } - } - - for (size_t i = 0; i < stream_list_.size(); ++i) { - GE_LOGW_IF(rtStreamDestroy(stream_list_[i]) != RT_ERROR_NONE, "Destroy stream failed, index: %zu", i); - } - - for (size_t i = 0; i < event_list_.size(); ++i) { - GE_LOGW_IF(rtEventDestroy(event_list_[i]) != RT_ERROR_NONE, "Destroy event failed, index: %zu", i); - } - - FreeWeightsMem(); - - FreeFeatureMapMem(); - - FreeP2PMem(); - - if (l1_fusion_addr_ != nullptr) { - GE_CHK_RT(rtFree(l1_fusion_addr_)); - } - - if (rt_model_handle_ != nullptr) { - GE_CHK_RT(rtModelDestroy(rt_model_handle_)); - rt_model_handle_ = nullptr; - } - } - - OpDebugUnRegister(); - - ReleaseTask(); - CleanTbeHandle(); - - var_mem_base_ = nullptr; - if (known_node_) { - if (args_ != nullptr) { - GE_CHK_RT(rtFree(args_)); - } - total_io_addrs_.clear(); - if (fixed_addrs_ != nullptr) { - GE_CHK_RT(rtFree(fixed_addrs_)); - } - } - } catch (...) { - GELOGW("DavinciModel::~DavinciModel: clear op_list catch exception."); - } -} - -void DavinciModel::UnbindHcomStream() { - if (!all_hccl_stream_list_.empty()) { - for (size_t i = 0; i < all_hccl_stream_list_.size(); i++) { - GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, all_hccl_stream_list_[i]) != RT_ERROR_NONE, - "Unbind hccl stream from model failed! Index: %zu", i); - GE_LOGW_IF(rtStreamDestroy(all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Destroy hccl stream for rt_model failed!") - } - } - return; -} - -void DavinciModel::ReleaseTask() { - for (const auto &task : cpu_task_list_) { - if (task != nullptr) { - GE_CHK_STATUS(task->Release(), "Release task failed."); - } - } - cpu_task_list_.clear(); - - for (const auto &task : task_list_) { - if (task != nullptr) { - GE_CHK_STATUS(task->Release(), "Release task failed."); - } - } -} - -Status DavinciModel::Assign(const GeModelPtr &ge_model) { - if (ge_model == nullptr) { - GELOGI("can't assign null ge_model"); - return FAILED; - } - ge_model_ = ge_model; - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Reduce memory usage after task sink. -/// @return: void -/// -void DavinciModel::Shrink() { - skt_info_ = {0, 0, 0, 0, nullptr, nullptr, {}, {}, {}, {}, {}, RT_KERNEL_DEFAULT, -1, 0, nullptr}; - ge_model_.reset(); // delete object. -} - -Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weight_size) { - if (is_weight_mem_has_inited_) { - GELOGE(FAILED, "call InitWeightMem more than once."); - return FAILED; - } - is_weight_mem_has_inited_ = true; - - const Buffer &weights = ge_model_->GetWeight(); - std::size_t weights_size = weights.GetSize(); - GE_CHECK_LE(weights_size, ALLOC_MEMORY_MAX_SIZE); - - if ((weight_ptr != nullptr) && (weight_size < weights_size)) { - GELOGE(FAILED, "Invalid mem param: weight_size=%zu totalsize=%zu.", weight_size, weights_size); - return FAILED; - } - - weights_mem_base_ = static_cast(dev_ptr); - is_inner_weight_base_ = false; - - if (weights_size != 0) { - weights_mem_base_ = static_cast(weight_ptr); - is_inner_weight_base_ = false; - if (weight_ptr == nullptr) { - weights_mem_base_ = MallocWeightsMem(weights_size); - if (weights_mem_base_ == nullptr) { - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc weight memory failed. size: %zu", weights_size); - return ACL_ERROR_GE_MEMORY_ALLOCATION; - } - is_inner_weight_base_ = true; - } - GELOGI("[IMAS]InitWeightMem graph_%u MallocMemory type[W] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id, - weights_mem_base_, weights_size); - GE_CHK_RT_RET(rtMemcpy(weights_mem_base_, weights_size, weights.GetData(), weights_size, RT_MEMCPY_HOST_TO_DEVICE)); - GELOGI("copy weights data to device"); - } - - runtime_param_.weight_base = weights_mem_base_; - return SUCCESS; -} - - -Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { - if (is_feature_map_mem_has_inited_) { - GELOGE(PARAM_INVALID, "call InitFeatureMapMem more than once."); - return PARAM_INVALID; - } - is_feature_map_mem_has_inited_ = true; - - std::size_t data_size = TotalMemSize(); - std::size_t p2p_data_size = P2PMemInfos().at(RT_MEMORY_P2P_DDR).memory_size; - - if ((dev_ptr != nullptr) && (mem_size < TotalMemSize())) { - GELOGE(PARAM_INVALID, "Invalid mem param: mem_size=%zu totalsize=%zu.", mem_size, TotalMemSize()); - return PARAM_INVALID; - } - - mem_base_ = static_cast(dev_ptr); - p2p_mem_base_ = static_cast(dev_ptr); - is_inner_mem_base_ = false; - - if (TotalMemSize() && mem_base_ == nullptr) { - mem_base_ = MallocFeatureMapMem(data_size); - if (mem_base_ == nullptr) { - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc feature map memory failed. size: %zu", data_size); - return ACL_ERROR_GE_MEMORY_ALLOCATION; - } - GEEVENT("[IMAS]InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", - runtime_param_.graph_id, mem_base_, data_size); - - if (!is_inner_weight_base_) { - weights_mem_base_ = mem_base_; - is_inner_weight_base_ = true; - } - is_inner_mem_base_ = true; - } - - if (p2p_data_size != 0) { - p2p_mem_base_ = MallocP2PMem(p2p_data_size); - if (p2p_mem_base_ == nullptr) { - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc p2p memory failed,size: %zu", p2p_data_size); - return ACL_ERROR_GE_MEMORY_ALLOCATION; - } - GELOGI("InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id, - p2p_mem_base_, p2p_data_size); - is_inner_p2p_mem_base_ = true; - } - - GE_CHK_STATUS_RET(InitVariableMem(), "Init variable memory failed."); - runtime_param_.mem_base = mem_base_; - runtime_param_.weight_base = weights_mem_base_; - runtime_param_.memory_infos[RT_MEMORY_P2P_DDR].memory_base = p2p_mem_base_; - return SUCCESS; -} - -Status DavinciModel::InitVariableMem() { - // malloc variable memory base - var_mem_base_ = VarManager::Instance(session_id_)->GetVarMemoryBase(RT_MEMORY_HBM); - if (TotalVarMemSize() && var_mem_base_ == nullptr) { - Status ret = VarManager::Instance(session_id_)->MallocVarMemory(TotalVarMemSize()); - if (ret != SUCCESS) { - GELOGE(ret, "Malloc variable memory failed."); - return ret; - } - var_mem_base_ = VarManager::Instance(session_id_)->GetVarMemoryBase(RT_MEMORY_HBM); - GEEVENT("[IMAS]InitVariableMem graph_%u MallocMemory type[V] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id, - var_mem_base_, TotalVarMemSize()); - } - runtime_param_.var_base = var_mem_base_; - return SUCCESS; -} - -void DavinciModel::InitRuntimeParams() { - int64_t value = 0; - bool ret; - MemInfo p2p_mem_info; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_MEMORY_SIZE, value); - runtime_param_.mem_size = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_WEIGHT_SIZE, value); - runtime_param_.weight_size = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_STREAM_NUM, value); - runtime_param_.stream_num = ret ? (uint32_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_EVENT_NUM, value); - runtime_param_.event_num = ret ? (uint32_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_LABEL_NUM, value); - runtime_param_.label_num = ret ? (uint32_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_BATCH_NUM, value); - runtime_param_.batch_num = ret ? (uint32_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, MODEL_ATTR_TASK_GEN_BASE_ADDR, value); - runtime_param_.logic_mem_base = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, value); - runtime_param_.logic_weight_base = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ge::MODEL_ATTR_SESSION_ID, value); - runtime_param_.session_id = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_TASK_GEN_VAR_ADDR, value); - runtime_param_.logic_var_base = ret ? (uint64_t)value : 0; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_VAR_SIZE, value); - runtime_param_.var_size = ret ? (uint64_t)value : 0; - session_id_ = runtime_param_.session_id; - ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_P2P_MEMORY_SIZE, value); - p2p_mem_info.memory_size = ret ? (uint64_t)value : 0; - runtime_param_.memory_infos[RT_MEMORY_P2P_DDR] = std::move(p2p_mem_info); - - GELOGI( - "InitRuntimeParams(), session_id:%lu, stream_num:%u, event_num:%u, label_num:%u, " - "logic_mem_base:0x%lx, logic_weight_base:0x%lx, logic_var_base:0x%lx, " - "memory_size:%lu, weight_size:%lu, var_size:%lu", - runtime_param_.session_id, runtime_param_.stream_num, runtime_param_.event_num, runtime_param_.label_num, - runtime_param_.logic_mem_base, runtime_param_.logic_weight_base, runtime_param_.logic_var_base, - runtime_param_.mem_size, runtime_param_.weight_size, runtime_param_.var_size); -} - -void DavinciModel::CheckHasHcomOp() { - Graph graph = ge_model_->GetGraph(); - auto compute_graph = GraphUtils::GetComputeGraph(graph); - if (compute_graph == nullptr) { - return; - } - for (const auto &node : compute_graph->GetAllNodes()) { - OpDescPtr op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGW("Node OpDesc is nullptr"); continue); - GE_IF_BOOL_EXEC(((op_desc->GetType() == HCOMBROADCAST) || (op_desc->GetType() == HCOMALLGATHER) || - (op_desc->GetType() == HCOMALLREDUCE) || (op_desc->GetType() == HCOMSEND) || - (op_desc->GetType() == HCOMRECEIVE) || (op_desc->GetType() == HCOMREDUCESCATTER) || - (op_desc->GetType() == HVDCALLBACKALLREDUCE) || (op_desc->GetType() == HVDCALLBACKALLGATHER) || - (op_desc->GetType() == HVDCALLBACKBROADCAST) || (op_desc->GetType() == HVDWAIT) || - (op_desc->GetType() == HCOMREDUCE)), - uint32_t stream_id = static_cast(op_desc->GetStreamId()); - (void)hcom_streams_.emplace(stream_id); GELOGD("hcom stream: %u.", stream_id); continue); - } -} - -/// -/// @ingroup ge -/// @brief Make active stream list and bind to model. -/// @return: 0 for success / others for fail -/// -Status DavinciModel::BindModelStream() { - // Stream not in active_stream_indication_ is active stream. - is_stream_list_bind_ = false; - if ((!input_queue_ids_.empty() || !output_queue_ids_.empty()) || (deploy_type_ == AICPU_DEPLOY_CROSS_THREAD)) { - for (size_t i = 0; i < stream_list_.size(); ++i) { - if (active_stream_indication_.count(i) == 0) { - active_stream_list_.push_back(stream_list_[i]); - active_stream_indication_.insert(i); // deactive all model stream. - } - } - } - - for (size_t i = 0; i < stream_list_.size(); ++i) { - if (active_stream_indication_.count(i) > 0) { - GELOGI("rtModelBindStream[%zu]", i); - GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, stream_list_[i], RT_INVALID_FLAG)); - } else { - // bind rt_model_handel to all streams that relates to op - GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, stream_list_[i], RT_HEAD_STREAM)); - } - } - is_stream_list_bind_ = true; - return SUCCESS; -} - -Status DavinciModel::DoTaskSink() { - // task sink is supported as model_task_def is set - const auto &model_task_def = ge_model_->GetModelTaskDefPtr(); - if (model_task_def == nullptr) { - return SUCCESS; - } - - GE_CHK_RT_RET(rtGetAicpuDeploy(&deploy_type_)); - GELOGI("do task_sink. AiCpu deploy type is: %x.", deploy_type_); - - GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed."); - - if (known_node_) { - GE_CHK_STATUS_RET(MallocKnownArgs(), "Mallloc known node args failed."); - } - - GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "InitTaskInfo failed."); - - GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed."); - - GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), "Check aicpu op type failed."); - - GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed."); - - GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed."); - - GE_CHK_RT_RET(rtModelLoadComplete(rt_model_handle_)); - - SetCopyOnlyOutput(); - return SUCCESS; -} - -// set device use aicore(0) or vectorcore(1) -Status DavinciModel::SetTSDevice() { - int64_t value = 0; - bool ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_CORE_TYPE, value); - uint32_t core_type = ret ? static_cast(value) : 0; - GELOGD("SetTSDevice: %u", core_type); - rtError_t rt_ret = rtSetTSDevice(core_type); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "SetTSDevice failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - return SUCCESS; -} - -Status DavinciModel::OpDebugRegister() { - bool is_op_debug = false; - (void)ge::AttrUtils::GetBool(ge_model_, ATTR_OP_DEBUG_FLAG, is_op_debug); - GELOGD("The value of op_debug in ge_model_ is %d.", is_op_debug); - if (is_op_debug) { - debug_reg_mutex_.lock(); - rtError_t rt_ret = rtMalloc(&op_debug_addr_, kOpDebugMemorySize, RT_MEMORY_DDR); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - - uint64_t debug_addrs_tmp = static_cast(reinterpret_cast(op_debug_addr_)); - - // For data dump, aicpu needs the pointer to pointer that save the real debug address. - rt_ret = rtMalloc(&p2p_debug_addr_, kDebugP2pSize, RT_MEMORY_HBM); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - rt_ret = rtMemcpy(p2p_debug_addr_, sizeof(uint64_t), &debug_addrs_tmp, sizeof(uint64_t), RT_MEMCPY_HOST_TO_DEVICE); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "rtMemcpy to p2p_addr error: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - - uint32_t op_debug_mode = 0; - (void)ge::AttrUtils::GetInt(ge_model_, ATTR_OP_DEBUG_MODE, op_debug_mode); - GELOGD("The value of op_debug_mode in ge_model_ is %u.", op_debug_mode); - uint32_t debug_task_id = 0; - uint32_t debug_stream_id = 0; - rt_ret = rtDebugRegister(rt_model_handle_, op_debug_mode, op_debug_addr_, &debug_stream_id, &debug_task_id); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "rtDebugRegister error, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - GELOGI("debug_task_id:%d, debug_stream_id:%u", debug_task_id, debug_stream_id); - is_op_debug_reg_ = true; - - data_dumper_.SaveOpDebugId(debug_task_id, debug_stream_id, p2p_debug_addr_, is_op_debug); - } - - return SUCCESS; -} - -void DavinciModel::OpDebugUnRegister() { - if (is_op_debug_reg_) { - debug_reg_mutex_.unlock(); - rtError_t rt_ret = RT_ERROR_NONE; - if (rt_model_handle_ != nullptr) { - GELOGD("start call debug_unregister."); - rt_ret = rtDebugUnRegister(rt_model_handle_); - if (rt_ret != RT_ERROR_NONE) { - GELOGW("rtDebugUnRegister failed, ret: 0x%X", rt_ret); - } - } - - if (op_debug_addr_ != nullptr) { - rt_ret = rtFree(op_debug_addr_); - if (rt_ret != RT_ERROR_NONE) { - GELOGW("rtFree failed, ret: 0x%X", rt_ret); - } - op_debug_addr_ = nullptr; - } - - if (p2p_debug_addr_ != nullptr) { - rt_ret = rtFree(p2p_debug_addr_); - if (rt_ret != RT_ERROR_NONE) { - GELOGW("rtFree failed, ret: 0x%X", rt_ret); - } - p2p_debug_addr_ = nullptr; - } - is_op_debug_reg_ = false; - } - return; -} - -// initialize op sequence and call initialization function of each op respectively -Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) { - // validating params - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(priority_ < 0 || priority_ > 7, return PARAM_INVALID, - "Priority must between 0-7, now is %d", priority_); - GE_CHK_BOOL_RET_STATUS(ge_model_ != nullptr, PARAM_INVALID, "GeModel is null."); - Graph graph = ge_model_->GetGraph(); - ComputeGraphPtr compute_graph = GraphUtils::GetComputeGraph(graph); - GE_CHK_BOOL_RET_STATUS(compute_graph != nullptr, INTERNAL_ERROR, "Get compute graph is nullptr."); - - // Initializing runtime_param_ - InitRuntimeParams(); - - // RTS set aicore or vectorcore - GE_CHK_STATUS_RET(SetTSDevice(), "SetTSDevice failed"); - - version_ = ge_model_->GetVersion(); - name_ = ge_model_->GetName(); - (void)ge::AttrUtils::GetBool(ge_model_, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_); - GELOGD("The value of ge.l1Fusion in ge_model_ is %d.", is_l1_fusion_enable_); - CheckHasHcomOp(); - - vector huge_stream_list; - (void)ge::AttrUtils::GetListInt(ge_model_, ATTR_MODEL_HUGE_STREAM_LIST, huge_stream_list); - std::set huge_streams(huge_stream_list.begin(), huge_stream_list.end()); - - for (uint32_t i = 0; i < StreamNum(); i++) { - rtStream_t stream = nullptr; - GE_MAKE_GUARD_RTSTREAM(stream); - - uint32_t stream_flags = RT_STREAM_PERSISTENT; - if (huge_streams.find(i) != huge_streams.end()) { - GELOGI("Stream %u is huge stream.", i); - stream_flags |= RT_STREAM_HUGE; - } - - if (hcom_streams_.find(i) != hcom_streams_.end()) { - GE_CHK_RT_RET(rtStreamCreateWithFlags(&stream, priority_, stream_flags | RT_STREAM_FORCE_COPY)); - } else { - GE_CHK_RT_RET(rtStreamCreateWithFlags(&stream, priority_, stream_flags)); - } - - GE_DISMISS_GUARD(stream); - stream_list_.push_back(stream); - int32_t rt_stream_id = kInvalidStream; - (void)rtGetStreamId(stream, &rt_stream_id); - GELOGI("Logical stream index:%u, stream:%p, rtstream: %d.", i, stream, rt_stream_id); - } - - for (uint32_t i = 0; i < EventNum(); i++) { - rtEvent_t rt_event; - GE_CHK_RT_RET(rtEventCreate(&rt_event)); - event_list_.push_back(rt_event); - } - - label_list_.resize(LabelNum(), nullptr); - - // create model_handle to load model - GE_CHK_RT_RET(rtModelCreate(&rt_model_handle_, 0)); - GE_CHK_RT_RET(rtModelGetId(rt_model_handle_, &runtime_model_id_)); - - // inference will use default graph_id 0; - runtime_param_.graph_id = compute_graph->GetGraphID(); - - // op debug register - GE_CHK_STATUS_RET(OpDebugRegister(), "OpDebugRegister failed"); - - GE_TIMESTAMP_START(TransAllVarData); - GE_CHK_STATUS_RET(TransAllVarData(compute_graph, runtime_param_.graph_id), "TransAllVarData failed."); - GE_TIMESTAMP_END(TransAllVarData, "GraphLoader::TransAllVarData"); - GE_CHK_STATUS_RET(TransVarDataUtils::CopyVarData(compute_graph, session_id_, device_id_), "copy var data failed."); - - GE_TIMESTAMP_START(InitModelMem); - GELOGD("Known node is %d", known_node_); - GE_CHK_STATUS_RET_NOLOG(InitWeightMem(dev_ptr, weight_ptr, weight_size)); - if (!known_node_) { - GE_CHK_STATUS_RET_NOLOG(InitFeatureMapAndP2PMem(dev_ptr, mem_size)); - data_inputer_ = new (std::nothrow) DataInputer(); - GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, MEMALLOC_FAILED, "data_inputer_ is nullptr."); - } - fixed_mem_base_ = reinterpret_cast(mem_base_); - GE_TIMESTAMP_END(InitModelMem, "GraphLoader::InitModelMem"); - - for (const ge::NodePtr &node : compute_graph->GetDirectNode()) { - auto op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, continue); - GE_IF_BOOL_EXEC(op_desc->GetType() != VARIABLE, continue); - GE_IF_BOOL_EXEC(IsBroadCastOpData(node), - (void)ge::AttrUtils::SetStr(op_desc, VAR_ATTR_VAR_IS_BROADCAST, "var_is_restore");); - } - - GE_CHK_STATUS_RET(InitNodes(compute_graph), "Init nodes failed"); - - SetDataDumperArgs(compute_graph); - GE_TIMESTAMP_START(DoTaskSink); - GE_CHK_STATUS_RET(DoTaskSink(), "Task sink failed"); - GE_TIMESTAMP_END(DoTaskSink, "GraphLoader::DoTaskSink"); - - auto all_dump_model = GetDumpProperties().GetAllDumpModel(); - bool findByOmName = all_dump_model.find(om_name_) != all_dump_model.end(); - bool findByModelName = all_dump_model.find(name_) != all_dump_model.end(); - bool dump_l1fusion_op = (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end()) || - findByOmName || findByModelName; - if (dump_l1fusion_op) { - // malloc 2M for dump l1fusion op - GE_CHK_RT_RET(rtMalloc(&l1_fusion_addr_, kDumpL1FusionOpMByteSize, RT_MEMORY_DDR)); - - // send l1fusion dump addr to rts - GE_CHK_RT_RET(rtDumpAddrSet(rt_model_handle_, l1_fusion_addr_, kDumpL1FusionOpMByteSize, kDumpFlagOfL1Fusion)); - } - - /// In zero copy model, if a aicpu operator is connected to the first or last layer, before model execution, - /// the aicpu opertor needs to destroy history record, and update operator memory address. - /// The model with specified aicpu operators is only marked here, and destruction is in ModelManager::ExecuteModel(). - need_destroy_aicpu_kernel_ = IsAicpuKernelConnectSpecifiedLayer(); - (void)ge::AttrUtils::GetListStr(ge_model_, ATTR_MODEL_OUT_NODES_NAME, out_node_name_); - - string fp_ceiling_mode; - if (ge::AttrUtils::GetStr(ge_model_, ATTR_FP_CEILING_MODE, fp_ceiling_mode)) { - GELOGI("Get attr ATTR_FP_CEILING_MODE from model, value is %s.", fp_ceiling_mode.c_str()); - // mode 0: Do not perform saturation processing. By default, IEEE754 is used. - GE_CHK_RT_RET(rtSetCtxINFMode((fp_ceiling_mode != "0"))); - } - - // collect profiling for ge - GE_CHK_STATUS_RET(InitModelProfile(), "Init model profile failed"); - auto &profiling_manager = ProfilingManager::Instance(); - if (profiling_manager.ProfilingModelLoadOn()) { - Status p_ret = ReportProfilingData(); - if (p_ret != SUCCESS) { - GELOGE(p_ret, "Report profiling data failed."); - return p_ret; - } - } - - Shrink(); - return SUCCESS; -} - -Status DavinciModel::ReportProfilingData() { - std::vector compute_graph_desc_info; - Status ret = GetComputeGraphInfo(compute_graph_desc_info); - if (ret != SUCCESS) { - GELOGE(ret, "GetComputeGraphInfo failed."); - return ret; - } - ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); - GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); - op_list_.clear(); - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Travel all nodes and determine if destruction is required. -/// @return bool -/// -bool DavinciModel::IsAicpuKernelConnectSpecifiedLayer() { - Graph graph = ge_model_->GetGraph(); - ComputeGraphPtr compute_graph = GraphUtils::GetComputeGraph(graph); - auto all_nodes = compute_graph->GetAllNodes(); - for (auto &node : all_nodes) { - GE_IF_BOOL_EXEC(node == nullptr, continue); - OpDescPtr op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, continue); - - int64_t imply_type = -1; - (void)ge::AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, imply_type); - if (imply_type != static_cast(domi::ImplyType::AI_CPU)) { - continue; - } - GELOGD("Current operator imply type is %ld, name is %s.", imply_type, op_desc->GetName().c_str()); - - for (auto &in_data_anchor : node->GetAllInDataAnchors()) { - GE_IF_BOOL_EXEC(in_data_anchor == nullptr, continue); - auto peer_out_data_anchor = in_data_anchor->GetPeerOutAnchor(); - GE_IF_BOOL_EXEC(peer_out_data_anchor == nullptr, continue); - auto peer_node = peer_out_data_anchor->GetOwnerNode(); - GE_IF_BOOL_EXEC(peer_node == nullptr, continue); - auto peer_op_desc = peer_node->GetOpDesc(); - GE_IF_BOOL_EXEC(peer_op_desc == nullptr, continue); - if (IsDataOp(peer_op_desc->GetType())) { - GELOGI("Mark specified aicpu operator connected to data."); - return true; - } - } - for (auto &out_data_anchor : node->GetAllOutDataAnchors()) { - GE_IF_BOOL_EXEC(out_data_anchor == nullptr, continue); - auto peer_in_data_anchors = out_data_anchor->GetPeerInDataAnchors(); - for (auto &peer_in_data_anchor : peer_in_data_anchors) { - GE_IF_BOOL_EXEC(peer_in_data_anchor == nullptr, continue); - auto peer_node = peer_in_data_anchor->GetOwnerNode(); - GE_IF_BOOL_EXEC(peer_node == nullptr, continue); - auto peer_op_desc = peer_node->GetOpDesc(); - GE_IF_BOOL_EXEC(peer_op_desc == nullptr, continue); - if (peer_op_desc->GetType() == NETOUTPUT) { - GELOGI("Mark specified aicpu operator connected to netoutput."); - return true; - } - } - } - } - - return false; -} - -Status DavinciModel::UpdateSessionId(uint64_t session_id) { - GE_CHECK_NOTNULL(ge_model_); - if (!AttrUtils::SetInt(ge_model_, MODEL_ATTR_SESSION_ID, static_cast(session_id))) { - GELOGW("Set attr[%s] failed in updating session_id.", MODEL_ATTR_SESSION_ID.c_str()); - } - - GELOGD("Update session id: %lu.", session_id); - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Travel all nodes and do some init. -/// @param [in] compute_graph: ComputeGraph to load. -/// @return Status -/// -Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { - uint32_t data_op_index = 0; - GE_TIMESTAMP_CALLNUM_START(LoadTBEKernelBinToOpDesc); - GE_TIMESTAMP_CALLNUM_START(InitTbeHandle); - - typedef Status (DavinciModel::*OpDescCall)(const OpDescPtr &); - static std::map op_desc_handle = { - {VARIABLE, &DavinciModel::InitVariable}, - {CONSTANTOP, &DavinciModel::InitConstant}, - {STREAMACTIVE, &DavinciModel::InitStreamActive}, - {STREAMSWITCH, &DavinciModel::InitStreamSwitch}, - {STREAMSWITCHN, &DavinciModel::InitStreamSwitchN}, - {LABELSET, &DavinciModel::InitLabelSet}, - {CASE, &DavinciModel::InitCase}, - }; - - vector output_op_list; - map data_by_index; - auto nodes = compute_graph->GetAllNodes(); - const CustAICPUKernelStore &aicpu_kernel_store = ge_model_->GetCustAICPUKernelStore(); - for (size_t i = 0; i < nodes.size(); ++i) { - auto node = nodes.at(i); - auto op_desc = node->GetOpDesc(); - if (op_desc == nullptr) { - GELOGE(PARAM_INVALID, "op_desc is null."); - return PARAM_INVALID; - } - - op_list_[op_desc->GetId()] = op_desc; - - GE_TIMESTAMP_RESTART(LoadTBEKernelBinToOpDesc); - aicpu_kernel_store.LoadCustAICPUKernelBinToOpDesc(op_desc); - GE_TIMESTAMP_ADD(LoadTBEKernelBinToOpDesc); - - if (IsDataOp(op_desc->GetType())) { - if (InitDataOp(compute_graph, node, data_op_index, data_by_index) != SUCCESS) { - GELOGE(PARAM_INVALID, "Data init failed, Name: %s", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - data_dumper_.SaveDumpInput(node); - continue; - } - - if (op_desc->GetType() == NETOUTPUT) { - if (InitNetOutput(compute_graph, node, output_op_list) != SUCCESS) { - GELOGE(PARAM_INVALID, "NetOutput init failed, Name: %s", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - continue; - } - - auto it = op_desc_handle.find(op_desc->GetType()); - if (it != op_desc_handle.end()) { - if ((this->*it->second)(op_desc) != SUCCESS) { - GELOGE(PARAM_INVALID, "NetOutput init failed, Name: %s", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - continue; - } - // for dynamic shape with control flow - SetLabelForDynamic(node); - if (IsNoTaskAndDumpNeeded(op_desc)) { - GELOGD("node[%s] without task, and save op_desc and addr for dump", op_desc->GetName().c_str()); - const RuntimeParam &rts_param = GetRuntimeParam(); - const vector input_data_addrs = ModelUtils::GetInputDataAddrs(rts_param, op_desc); - const vector output_data_addrs = ModelUtils::GetOutputDataAddrs(rts_param, op_desc); - const vector workspace_data_addrs = ModelUtils::GetWorkspaceDataAddrs(rts_param, op_desc); - vector tensor_device_addrs; - tensor_device_addrs.insert(tensor_device_addrs.end(), input_data_addrs.begin(), input_data_addrs.end()); - tensor_device_addrs.insert(tensor_device_addrs.end(), output_data_addrs.begin(), output_data_addrs.end()); - tensor_device_addrs.insert(tensor_device_addrs.end(), workspace_data_addrs.begin(), workspace_data_addrs.end()); - void *addr = nullptr; - auto size = kAddrLen * tensor_device_addrs.size(); - GE_CHK_RT_RET(rtMalloc(&addr, size, RT_MEMORY_HBM)); - - rtError_t rt_ret = rtMemcpy(addr, size, tensor_device_addrs.data(), size, RT_MEMCPY_HOST_TO_DEVICE); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "rtMemcpy error, ret: 0x%X", rt_ret); - GE_CHK_RT(rtFree(addr)); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - saved_task_addrs_.emplace(op_desc, addr); - } - - GE_TIMESTAMP_RESTART(InitTbeHandle); - uint32_t run_mode = static_cast(domi::ImplyType::INVALID); - if (AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, run_mode) && - run_mode == static_cast(domi::ImplyType::TVM)) { - // Skip no_task operator, such as concat and split. - bool attr_notask = false; - bool get_attr_notask_flag = ge::AttrUtils::GetBool(op_desc, ATTR_NAME_NOTASK, attr_notask); - GE_IF_BOOL_EXEC(get_attr_notask_flag && attr_notask, - GELOGI("Node[name:%s, type:%s] does not generate task, skip initialization.", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - continue;); - - Status status = InitTbeHandle(op_desc); - if (status != SUCCESS) { - GELOGE(status, "TBE init failed. %s", op_desc->GetName().c_str()); - return status; - } - } - GE_TIMESTAMP_ADD(InitTbeHandle); - } - - GE_TIMESTAMP_CALLNUM_END(LoadTBEKernelBinToOpDesc, "GraphLoader::LoadTBEKernelBinToOpDesc."); - GE_TIMESTAMP_CALLNUM_END(InitTbeHandle, "GraphLoader::InitTbeHandle."); - return OptInputOutputInfo(data_by_index, output_op_list); -} - -void DavinciModel::SetLabelForDynamic(const NodePtr &node) { - if (known_node_ && node->GetOpDesc()->GetType() == LABELSWITCHBYINDEX) { - for (auto &in_data_anchor : node->GetAllInDataAnchors()) { - auto peer_out_data_anchor = in_data_anchor->GetPeerOutAnchor(); - if (peer_out_data_anchor != nullptr) { - string tensor_name = node->GetName(); - auto peer_node = peer_out_data_anchor->GetOwnerNode(); - (void)AttrUtils::SetStr(peer_node->GetOpDesc(), ATTR_DYNAMIC_SHAPE_FIXED_ADDR, tensor_name); - (void)AttrUtils::SetInt(peer_node->GetOpDesc(), ATTR_DYNAMIC_SHAPE_FIXED_ADDR_INDEX, 0); - tensor_name_to_peer_output_index_[tensor_name] = 0; - } - } - } -} - -/// -/// @ingroup ge -/// @brief Data Op Initialize. -/// @param [in] ComputeGraphPtr: root graph of the model. -/// @param [in] NodePtr: Data Op. -/// @param [in/out] data_op_index: index of courrent count. -/// @param [in/out] data_by_index: Data ordered by index. -/// @return Status -/// -Status DavinciModel::InitDataOp(const ComputeGraphPtr &graph, const NodePtr &node, uint32_t &data_op_index, - map &data_by_index) { - // op_desc Checked by Init: Data, valid. - auto op_desc = node->GetOpDesc(); - if (node->GetOwnerComputeGraph() != graph) { - GELOGI("Skip subgraph Data node: %s.", op_desc->GetName().c_str()); - return SUCCESS; - } - - GELOGI("Init Data node: %s.", op_desc->GetName().c_str()); - auto data_index = data_op_index++; - if (AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, data_index)) { - GELOGD("Get new index %u, old %u", data_index, data_op_index - 1); - } - - data_by_index[data_index] = op_desc; - data_op_list_.push_back(op_desc); - if (known_node_) { - return SUCCESS; - } - - // Make information for copy input data. - const vector output_size_list = ModelUtils::GetOutputSize(op_desc); - const vector virtual_addr_list = ModelUtils::GetOutputDataAddrs(runtime_param_, op_desc); - const vector output_offset_list = op_desc->GetOutputOffset(); - if (output_size_list.empty() || virtual_addr_list.empty() || (output_size_list.size() != virtual_addr_list.size()) || - (output_offset_list.size() != virtual_addr_list.size())) { - GELOGE(PARAM_INVALID, "Data[%s] init failed: output size is %zu, virtual_addr size is %zu, offset size is %zu.", - op_desc->GetName().c_str(), output_size_list.size(), virtual_addr_list.size(), output_offset_list.size()); - return PARAM_INVALID; - } - - bool fusion_flag = false; - ZeroCopyOffset zero_copy_offset; - int64_t data_size = output_size_list[kDataIndex]; - void *virtual_addr = virtual_addr_list[kDataIndex]; - Status ret = zero_copy_offset.InitInputDataInfo(data_size, virtual_addr, op_desc, fusion_flag); - if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - new_input_data_info_[data_index] = zero_copy_offset; - - for (size_t index = 0; index < virtual_addr_list.size(); ++index) { - void *addr = virtual_addr_list.at(index); - if (new_input_outside_addrs_.find(addr) != new_input_outside_addrs_.end()) { - continue; - } - zero_copy_offset.SetInputOutsideAddrs(output_offset_list, addr, index, fusion_flag, real_virtual_addrs_); - new_input_outside_addrs_[addr] = zero_copy_offset; - } - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Sort Data op list by index. -/// @param [in] data_by_index: map of Data Op. -/// @param [in] output_op_list: list of NetOutput op. -/// @return Status -/// -Status DavinciModel::OptInputOutputInfo(const map &data_by_index, - const vector &output_op_list) { - GELOGD("Data node size: %zu, NetOutput node size: %zu", data_op_list_.size(), output_op_list.size()); - if (data_by_index.size() != data_op_list_.size()) { - GELOGE(INTERNAL_ERROR, "Data map size: %zu, Data list size: %zu.", data_by_index.size(), data_op_list_.size()); - return INTERNAL_ERROR; - } - - data_op_list_.clear(); - for (auto &item : data_by_index) { - data_op_list_.emplace_back(item.second); - auto output_addrs = ModelUtils::GetOutputDataAddrs(runtime_param_, item.second); - GELOGD("Data node: %s, output addr size: %zu", item.second->GetName().c_str(), output_addrs.size()); - input_addrs_list_.emplace_back(output_addrs); - - if (item.second->GetType() == AIPP_DATA_TYPE) { - GELOGI("This is dynamic aipp model, Node: %s", item.second->GetName().c_str()); - is_dynamic_aipp_ = true; - } - } - - for (const auto &op_desc : output_op_list) { - auto input_addrs = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc); - GELOGD("NetOutput node: %s, input addr size: %zu", op_desc->GetName().c_str(), input_addrs.size()); - output_addrs_list_.emplace_back(input_addrs); - - bool getnext_sink_dynamic = false; - if (AttrUtils::GetBool(op_desc, ATTR_GETNEXT_SINK_DYNMAIC, getnext_sink_dynamic) && getnext_sink_dynamic) { - GELOGI("ATTR_GETNEXT_SINK_DYNMAIC has been set and is true, node: %s", op_desc->GetName().c_str()); - is_getnext_sink_dynamic_ = true; - } - - vector shape_info; - if (AttrUtils::GetListStr(op_desc, ATTR_NAME_DYNAMIC_OUTPUT_DIMS, shape_info)) { - dynamic_output_shape_info_.insert(dynamic_output_shape_info_.end(), shape_info.begin(), shape_info.end()); - } - - if (InitOutputTensorInfo(op_desc) != SUCCESS) { - return INTERNAL_ERROR; - } - } - - InitInputDescInfo(data_op_list_); - return InitOutputDescInfo(output_op_list); -} - -bool DavinciModel::IsGetNextSinkDynamic(const OpDescPtr &op_desc) { - bool getnext_sink_dynamic = false; - if (ge::AttrUtils::GetBool(op_desc, ATTR_GETNEXT_SINK_DYNMAIC, getnext_sink_dynamic) && getnext_sink_dynamic) { - GELOGI("ATTR_GETNEXT_SINK_DYNMAIC has been set and is true."); - return true; - } - return false; -} - -/// @ingroup ge -/// @brief NetOutput Op Initialize. -/// @param [in] ComputeGraphPtr: root graph of the model. -/// @param [in] NodePtr: NetOutput Op. -/// @param [in/out] vector: All NetOutput node in model. -/// @return Status -Status DavinciModel::InitNetOutput(const ComputeGraphPtr &graph, const NodePtr &node, - vector &output_op_list) { - // node->GetOpDesc Checked by Init: NetOutput, valid. - auto op_desc = node->GetOpDesc(); - // excludes the function op sub graph, e.g. case,if - if (node->GetOwnerComputeGraph() != graph) { - GELOGI("Skip subgraph NetOutput node: %s.", op_desc->GetName().c_str()); - op_list_.erase(op_desc->GetId()); - return SUCCESS; - } - - GELOGI("Init NetOutput node: %s.", op_desc->GetName().c_str()); - output_op_list.push_back(op_desc); - if (known_node_) { - return SUCCESS; - } - - // Make information for copy output data. - const vector input_size_list = ModelUtils::GetInputSize(op_desc); - const vector virtual_addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc); - const vector input_offset_list = op_desc->GetInputOffset(); - GE_IF_BOOL_EXEC(input_offset_list.size() != virtual_addr_list.size(), - GELOGE(PARAM_INVALID, "virtual_addr size should be equal to offset size."); return PARAM_INVALID;); - if (input_size_list.empty() && virtual_addr_list.empty()) { - GELOGI("NetOutput[%s] is empty.", op_desc->GetName().c_str()); - return SUCCESS; - } - if (input_size_list.empty() || input_size_list.size() != virtual_addr_list.size()) { - GELOGE(PARAM_INVALID, "NetOutput[%s] init failed: Input size is %zu, Input addr is %zu", op_desc->GetName().c_str(), - input_size_list.size(), virtual_addr_list.size()); - return PARAM_INVALID; - } - - size_t num = new_output_data_info_.size(); - bool fusion_flag = false; - - size_t input_count = input_size_list.size(); - is_getnext_sink_dynamic_ = false; - if (IsGetNextSinkDynamic(op_desc)) { - input_count = input_size_list.size() - kGetDynamicDimsCount; - is_getnext_sink_dynamic_ = true; - } - for (size_t idx = 0; idx < input_count; ++idx) { - ZeroCopyOffset zero_copy_offset; - Status ret = zero_copy_offset.InitOutputDataInfo(input_size_list, virtual_addr_list, op_desc, idx, fusion_flag); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", - op_desc->GetName().c_str()); return PARAM_INVALID;); - new_output_data_info_[num + idx] = zero_copy_offset; - void *addr = virtual_addr_list.at(idx); - int64_t input_offset = input_offset_list.at(idx); - vector tensor_addrs; - zero_copy_offset.SetOutputOutsideAddrs(input_offset, fusion_flag, addr, tensor_addrs); - auto rslt = new_output_outside_addrs_.insert(std::pair(addr, zero_copy_offset)); - if (!rslt.second) { - GELOGI("same output_tensor_addr %p to different input_tensor of %s", addr, op_desc->GetName().c_str()); - DisableZeroCopy(addr); - } - - for (size_t i = 0; i < tensor_addrs.size(); ++i) { - void *real_addr = tensor_addrs.at(i); - DisableZeroCopy(real_addr); - real_virtual_addrs_.insert(real_addr); - } - } - - GetAllGearsInfo(node); - if (is_getnext_sink_dynamic_) { - GE_IF_BOOL_EXEC(GetGetDynamicDimsNodeInfo(node) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get info of getdynamicdims node."); return PARAM_INVALID;); - } - if (is_online_infer_dynamic_) { - GE_IF_BOOL_EXEC(GetGearAndRealOutSizeInfo(input_count, node) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get gear and real out size info."); return PARAM_INVALID;); - GE_IF_BOOL_EXEC(GetGearAndRealOutShapeInfo(input_count, op_desc) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get gear and real out shape info."); return PARAM_INVALID;); - } - - return SUCCESS; -} - -void DavinciModel::GetAllGearsInfo(const NodePtr &node) { - is_online_infer_dynamic_ = false; - all_gears_info_.clear(); - std::string shapes; - (void) AttrUtils::GetStr(node->GetOpDesc(), ATTR_ALL_GEARS_INFO, shapes); - if (!shapes.empty()) { - is_online_infer_dynamic_ = true; - std::vector shape_strs = ge::StringUtils::Split(shapes, ';'); - for (const auto &shape_str : shape_strs) { - if (shape_str.empty()) { - continue; - } - std::vector gear_info; - std::vector dims = ge::StringUtils::Split(shape_str, ','); - for (const auto &dim : dims) { - if (dim.empty()) { - continue; - } - gear_info.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal)); - } - if (!gear_info.empty()) { - all_gears_info_.emplace_back(gear_info); - GELOGD("Init all gears info from %s, gaer info is %s.", node->GetName().c_str(), - formats::JoinToString(gear_info).c_str()); - } - } - } -} -Status DavinciModel::GetGetDynamicDimsNodeInfo(const NodePtr &node) { - GE_CHECK_NOTNULL(node->GetOpDesc()); - size_t input_count = node->GetAllInDataAnchors().size(); - GELOGI("input_anchor count of %s is %zu.", node->GetName().c_str(), input_count); - size_t get_dynamic_dims_index = input_count - kGetDynamicDimsCount; - auto in_anchor = node->GetAllInDataAnchors().at(get_dynamic_dims_index); - auto peer_out_anchor = in_anchor->GetPeerOutAnchor(); - if (peer_out_anchor == nullptr) { - GELOGE(PARAM_INVALID, "Out anchor of getdynmaicdims node should not be nullptr."); - return PARAM_INVALID; - } - auto peer_node = peer_out_anchor->GetOwnerNode(); - auto op_desc = peer_node->GetOpDesc(); - GE_CHECK_NOTNULL(op_desc); - if (op_desc->GetName() == kGetDynamicDimsName && op_desc->GetType() == GETDYNAMICDIMS) { - GELOGD("Start get info of %s.", op_desc->GetName().c_str()); - auto input_addr = ModelUtils::GetInputDataAddrs(runtime_param_, node->GetOpDesc()); - auto input_size = ModelUtils::GetInputSize(node->GetOpDesc()); - if (input_addr.empty() || input_size.empty()) { - GELOGE(PARAM_INVALID, "Not set output of %s", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - auto input_desc = node->GetOpDesc()->GetInputDescPtr(get_dynamic_dims_index); - GE_CHECK_NOTNULL(input_desc); - if (input_desc->GetShape().GetDims().empty()) { - GELOGE(PARAM_INVALID, "Not set output desc shape of %s.", op_desc->GetName().c_str()); - return PARAM_INVALID; - } - netoutput_last_input_addr_ = input_addr[get_dynamic_dims_index]; - netoutput_last_input_size_ = input_size[get_dynamic_dims_index]; - shape_of_cur_dynamic_dims_ = input_desc->GetShape().GetDims().at(0); - GELOGD("Shape of cur dynamic dims is %zu, size is %ld, addr is %p.", shape_of_cur_dynamic_dims_, - netoutput_last_input_size_, netoutput_last_input_addr_); - } - return SUCCESS; -} - -Status DavinciModel::GetGearAndRealOutSizeInfo(size_t input_count, const NodePtr &node) { - GELOGD("Start get gear and real output size info of %s, input count is %zu.", node->GetName().c_str(), input_count); - merge_nodes_gear_and_real_out_size_info_.clear(); - for (size_t idx = 0; idx < input_count; ++idx) { - auto in_anchor = node->GetAllInDataAnchors().at(idx); - auto peer_out_anchor = in_anchor->GetPeerOutAnchor(); - if (peer_out_anchor == nullptr) { - continue; - } - auto peer_node = peer_out_anchor->GetOwnerNode(); - auto op_desc = peer_node->GetOpDesc(); - GE_CHECK_NOTNULL(op_desc); - if ((peer_node->GetType() == MERGE) && (op_desc->HasAttr(ATTR_INSERT_BY_MBATCH))) { - if (GetRealOutputSizeOfMerge(idx, peer_node) != SUCCESS) { - GELOGE(PARAM_INVALID, "Get real output size of %s failed.", peer_node->GetName().c_str()); - return PARAM_INVALID; - } - } - } - return SUCCESS; -} - -Status DavinciModel::GetRealOutputSizeOfMerge(size_t input_index, const NodePtr &merge_node) { - GELOGD("Start get output size of %s, which is %zu input to netoutput.", merge_node->GetName().c_str(), input_index); - std::map, int64_t> gear_and_real_out_size_info; - for (auto &in_anchor : merge_node->GetAllInDataAnchors()) { - auto peer_out_anchor = in_anchor->GetPeerOutAnchor(); - if (peer_out_anchor == nullptr) { - continue; - } - auto in_node = peer_out_anchor->GetOwnerNode(); - GELOGD("Input node of merge is %s.", in_node->GetName().c_str()); - auto op_desc = in_node->GetOpDesc(); - GE_CHECK_NOTNULL(op_desc); - string batch_label; - if (AttrUtils::GetStr(op_desc, ATTR_NAME_BATCH_LABEL, batch_label)) { - size_t batch_index = static_cast(stoi(batch_label.substr(batch_label.rfind('_') + 1))); - GELOGD("Batch index of %s is %zu.", op_desc->GetName().c_str(), batch_index); - if (batch_index > all_gears_info_.size()) { - GELOGE(PARAM_INVALID, "The value of ATTR_NAME_BATCH_LABEL is invalid."); - return PARAM_INVALID; - } - - const vector output_size_list = ModelUtils::GetOutputSize(op_desc); - int output_index = ge::AnchorUtils::GetIdx(peer_out_anchor); - auto tensor_desc = op_desc->GetOutputDescPtr(output_index); - GE_CHECK_NOTNULL(tensor_desc); - int64_t data_size = 0; - if (TensorUtils::GetTensorSizeInBytes(*tensor_desc, data_size) != GRAPH_SUCCESS) { - GELOGE(FAILED, "Get tensor size in bytes failed."); - return FAILED; - } - gear_and_real_out_size_info[all_gears_info_[batch_index]] = data_size; - GELOGD("Get real gear index is: %zu, gear info is %s, size is %ld, tensor size is %ld", - batch_index, formats::JoinToString(all_gears_info_[batch_index]).c_str(), - output_size_list[output_index], data_size); - } - } - merge_nodes_gear_and_real_out_size_info_[input_index] = gear_and_real_out_size_info; - return SUCCESS; -} - -Status DavinciModel::GetGearAndRealOutShapeInfo(size_t input_count, const OpDescPtr &op_desc) { - GELOGD("Start to get dynamic output dims of %s.", op_desc->GetName().c_str()); - merge_nodes_gear_and_real_out_shape_info_.clear(); - std::vector dynamic_output_shape_info; - if (!AttrUtils::GetListStr(op_desc, ATTR_NAME_DYNAMIC_OUTPUT_DIMS, dynamic_output_shape_info)) { - GELOGD("Can not get dynamic output dims attr"); - return SUCCESS; - } - GELOGI("Dynamic output shape info is %s", formats::JoinToString(dynamic_output_shape_info).c_str()); - std::vector> dynamic_output_shape; - ParseDynamicOutShape(dynamic_output_shape_info, dynamic_output_shape); - // idx: input_index to netoutput - for (size_t idx = 0; idx < input_count; ++idx) { - std::map, vector> gear_and_real_out_shape_info; - for (auto &it : dynamic_output_shape) { - auto gear_index = static_cast(it[0]); - if (gear_index > all_gears_info_.size()) { - GELOGE(PARAM_INVALID, "The value of cur index: %zu is invalid.", static_cast(it[0])); - return PARAM_INVALID; - } - - if (static_cast(it[1]) == idx) { - vector output_shape; - for (size_t i = 2; i < it.size(); ++i) { - output_shape.emplace_back(it[i]); - } - gear_and_real_out_shape_info[all_gears_info_[gear_index]] = output_shape; - GELOGD("Get real gear index is: %zu, gear info is %s, output shape is %s.", - gear_index, formats::JoinToString(all_gears_info_[gear_index]).c_str(), - formats::JoinToString(output_shape).c_str()); - } - } - merge_nodes_gear_and_real_out_shape_info_[idx] = gear_and_real_out_shape_info; - } - return SUCCESS; -} - -void DavinciModel::ParseDynamicOutShape(const std::vector &str_info, - std::vector> &vec_info) { - for (size_t i = 0; i < str_info.size(); ++i) { - std::vector shape; - std::vector dims = ge::StringUtils::Split(str_info[i], ','); - for (const auto &dim : dims) { - if (dim.empty()) { - continue; - } - shape.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal)); - } - GELOGI("Shape from attr is %s.", formats::JoinToString(shape).c_str()); - vec_info.emplace_back(shape); - } -} - -/// @ingroup ge -/// @brief LabelSet Op Initialize. -/// @param [in] op_desc: LabelSet Op descriptor. -/// @return Status -Status DavinciModel::InitLabelSet(const OpDescPtr &op_desc) { - uint32_t label_index = 0; - if (!AttrUtils::GetInt(op_desc, ATTR_NAME_LABEL_SWITCH_INDEX, label_index)) { - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s attr [%s] not exist.", op_desc->GetName().c_str(), - ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); - return INTERNAL_ERROR; - } - if (label_index >= LabelNum()) { - GELOGE(INTERNAL_ERROR, "InitLabelSet: label index: %u >= label size: %u.", label_index, LabelNum()); - return INTERNAL_ERROR; - } - if (label_id_indication_.count(label_index) > 0) { - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s label index: %u already used.", op_desc->GetName().c_str(), label_index); - return INTERNAL_ERROR; - } - - rtStream_t stream = nullptr; - uint32_t stream_id = static_cast(op_desc->GetStreamId()); - if (stream_list_.size() == 1) { - stream = stream_list_[0]; - } else if (stream_list_.size() > stream_id) { - stream = stream_list_[stream_id]; - } else { - GELOGE(INTERNAL_ERROR, "InitLabelSet: stream index: %u >= stream size: %zu.", stream_id, stream_list_.size()); - return INTERNAL_ERROR; - } - - rtLabel_t rt_label = nullptr; - rtError_t rt_error = rtLabelCreateEx(&rt_label, stream); - if (rt_error != RT_ERROR_NONE || rt_label == nullptr) { - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s create label failed, error=0x%x.", op_desc->GetName().c_str(), rt_error); - return INTERNAL_ERROR; - } - - GELOGI("InitLabelSet: label[%u]=%p stream[%u]=%p.", label_index, rt_label, stream_id, stream); - label_id_indication_.insert(label_index); - label_list_[label_index] = rt_label; - return SUCCESS; -} - -Status DavinciModel::InitVariable(const OpDescPtr &op_desc) { - variable_op_list_.push_back(op_desc); - return SUCCESS; -} - -/// @ingroup ge -/// @brief ACL case, Load task list with queue. -/// @param [in] input_queue_ids: input queue ids from user, nums equal Data Op. -/// @param [in] output_queue_ids: input queue ids from user, nums equal NetOutput Op. -/// @return: 0 for success / others for failed -Status DavinciModel::SetQueIds(const std::vector &input_queue_ids, - const std::vector &output_queue_ids) { - if (input_queue_ids.empty() && output_queue_ids.empty()) { - GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "Param is empty"); - return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; - } - - input_queue_ids_ = input_queue_ids; - output_queue_ids_ = output_queue_ids; - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief ACL case, Load task list with queue. -/// @param [in] input_que_ids: input queue ids from user, nums equal Data Op. -/// @param [in] output_que_ids: input queue ids from user, nums equal NetOutput Op. -/// @return: 0 for success / others for failed -/// -Status DavinciModel::LoadWithQueue() { - if (input_queue_ids_.empty() && output_queue_ids_.empty()) { - return SUCCESS; - } - - if (input_queue_ids_.size() != new_input_data_info_.size()) { - GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "Input queue ids not match model: input_queue=%zu input_data=%zu", - input_queue_ids_.size(), new_input_data_info_.size()); - return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; - } - - if (output_queue_ids_.size() != new_output_data_info_.size()) { - GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, - "Output queue ids not match model: output_queue=%zu output_data=%zu", - output_queue_ids_.size(), new_output_data_info_.size()); - return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; - } - - GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed."); - // Binding input_queue and Data Op. - GE_CHK_STATUS_RET(BindInputQueue(), "Launch bind input queue failed."); - GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(input_mbuf_list_, new_input_outside_addrs_), "Launch zero copy failed."); - - // Binding output_queue and NetOutput Op. - GE_CHK_STATUS_RET(BindOutputQueue(), "Launch bind output queue failed."); - GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(output_mbuf_list_, new_output_outside_addrs_), "Launch zero copy failed."); - - GE_CHK_STATUS_RET(CpuActiveStream(), "Launch active entry stream failed."); - GE_CHK_STATUS_RET(CpuWaitEndGraph(), "Launch wait end graph failed."); - GE_CHK_STATUS_RET(BindEnqueue(), "Launch enqueue failed."); - GE_CHK_STATUS_RET(CpuModelRepeat(), "Launch model repeat failed."); - - return SUCCESS; -} - -/// @ingroup ge -/// @brief queue schedule, Bind input queue to Data output address. -/// @return: 0 for success / others for failed -Status DavinciModel::BindInputQueue() { - // Caller checked: input_queue_ids_.size() == input_size_list_.size() != input_addr_list_.size() - for (size_t i = 0; i < input_queue_ids_.size(); ++i) { - auto it = new_input_data_info_.find(i); - if (it == new_input_data_info_.end()) { - GELOGE(FAILED, "Input not match: tensor num=%zu, Queue id index=%zu", new_input_data_info_.size(), i); - return FAILED; - } - - uint32_t queue_id = input_queue_ids_[i]; - if (it->second.GetDataInfo().empty()) { - GELOGE(INTERNAL_ERROR, "the %zu input_queue not set data_info.", i); - return INTERNAL_ERROR; - } - uint32_t data_size = static_cast(it->second.GetDataInfo().at(0).first); - uintptr_t data_addr = reinterpret_cast(it->second.GetDataInfo().at(0).second); - GELOGI("BindInputToQueue: graph_%u index[%zu] queue id[%u] output addr[0x%lx] output size[%u]", - runtime_param_.graph_id, i, queue_id, data_addr, data_size); - - rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_INPUT_QUEUE); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - - if (CpuModelDequeue(queue_id) != SUCCESS) { - return INTERNAL_ERROR; - } - } - - return SUCCESS; -} - -/// @ingroup ge -/// @brief definiteness queue schedule, bind input queue to task. -/// @param [in] queue_id: input queue id from user. -/// @return: 0 for success / others for failed -Status DavinciModel::CpuModelDequeue(uint32_t queue_id) { - GELOGI("Set CpuKernel model dequeue task enter."); - std::shared_ptr dequeue_task = MakeShared(rt_entry_stream_); - if (dequeue_task == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelDequeue task failed."); - return MEMALLOC_FAILED; - } - - // Get DataOp Output address and bind to queue. - uintptr_t in_mbuf = 0; - Status status = dequeue_task->Init(queue_id, in_mbuf); - if (status != SUCCESS) { - return status; - } - - cpu_task_list_.push_back(dequeue_task); - input_mbuf_list_.push_back(in_mbuf); - GELOGI("Set CpuKernel model dequeue task success."); - return SUCCESS; -} - -Status DavinciModel::CpuTaskModelZeroCopy(std::vector &mbuf_list, - std::map &outside_addrs) { - GELOGI("Set CpuKernel model zero_copy task enter."); - std::shared_ptr zero_copy = MakeShared(rt_entry_stream_); - if (zero_copy == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskZeroCopy task failed."); - return MEMALLOC_FAILED; - } - - // mdc zero_copy not support l2 fusion - Status status = zero_copy->Init(mbuf_list, outside_addrs); - if (status != SUCCESS) { - return status; - } - cpu_task_list_.push_back(zero_copy); - GELOGI("Set CpuKernel model zero_copy task success."); - return SUCCESS; -} - -/// @ingroup ge -/// @brief queue schedule, bind output queue to NetOutput input address. -/// @return: 0 for success / others for failed -Status DavinciModel::BindOutputQueue() { - // Caller checked: input_queue_ids_.size() == input_size_list_.size() != input_addr_list_.size() - for (size_t i = 0; i < output_queue_ids_.size(); ++i) { - auto it = new_output_data_info_.find(i); - if (it == new_output_data_info_.end()) { - GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", new_output_data_info_.size(), i); - return FAILED; - } - - uint32_t queue_id = output_queue_ids_[i]; - if (it->second.GetDataInfo().empty()) { - GELOGE(INTERNAL_ERROR, "the %zu output_queue not set data_info.", i); - return INTERNAL_ERROR; - } - uint32_t data_size = static_cast(it->second.GetDataInfo().at(0).first); - uintptr_t data_addr = reinterpret_cast(it->second.GetDataInfo().at(0).second); - GELOGI("BindOutputToQueue: graph_%u index[%zu] queue id[%u] input addr[0x%lx] input size[%u]", - runtime_param_.graph_id, i, queue_id, data_addr, data_size); - - rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_OUTPUT_QUEUE); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - - Status status = CpuModelPrepareOutput(data_addr, data_size); - if (status != SUCCESS) { - return status; - } - } - - return SUCCESS; -} - -/// @ingroup ge -/// @brief definiteness queue schedule, bind output queue to task. -/// @param [in] addr: NetOutput Op input tensor address. -/// @param [in] size: NetOutput Op input tensor size. -/// @return: 0 for success / others for failed -Status DavinciModel::CpuModelPrepareOutput(uintptr_t addr, uint32_t size) { - GELOGI("Set CpuKernel model enqueue task enter."); - if (input_mbuf_list_.empty()) { - GELOGE(FAILED, "Need input mbuf for fill output mbuf head info."); - return FAILED; - } - - std::shared_ptr prepare_output = MakeShared(rt_entry_stream_); - if (prepare_output == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskPrepareOutput task failed."); - return MEMALLOC_FAILED; - } - - uintptr_t out_mbuf = 0; - if (prepare_output->Init(addr, size, input_mbuf_list_.back(), out_mbuf) != SUCCESS) { - return FAILED; - } - - cpu_task_list_.push_back(prepare_output); - output_mbuf_list_.push_back(out_mbuf); - GELOGI("Set CpuKernel model enqueue task success."); - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief definiteness queue schedule, active original model stream. -/// @return: 0 for success / others for failed -/// -Status DavinciModel::CpuActiveStream() { - GELOGI("Set CpuKernel active stream task enter."); - std::shared_ptr active_entry = MakeShared(rt_entry_stream_); - if (active_entry == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed."); - return MEMALLOC_FAILED; - } - - Status status = active_entry->Init(rt_head_stream_); - if (status != SUCCESS) { - return status; - } - - cpu_task_list_.push_back(active_entry); - GELOGI("Set CpuKernel active stream task success."); - return SUCCESS; -} - -/// @ingroup ge -/// @brief definiteness queue schedule, wait for end graph. -/// @return: 0 for success / others for failed -Status DavinciModel::CpuWaitEndGraph() { - GELOGI("Set CpuKernel wait end graph task enter."); - std::shared_ptr wait_endgraph = MakeShared(rt_entry_stream_); - if (wait_endgraph == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskWaitEndGraph task failed."); - return MEMALLOC_FAILED; - } - - Status status = wait_endgraph->Init(runtime_model_id_); - if (status != SUCCESS) { - return status; - } - - cpu_task_list_.push_back(wait_endgraph); - GELOGI("Set CpuKernel wait end graph task success."); - return SUCCESS; -} - -Status DavinciModel::BindEnqueue() { - for (size_t i = 0; i < output_queue_ids_.size(); ++i) { - auto it = new_output_data_info_.find(i); - if (it == new_output_data_info_.end()) { - GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", new_output_data_info_.size(), i); - return FAILED; - } - - uint32_t queue_id = output_queue_ids_[i]; - if (CpuModelEnqueue(queue_id, output_mbuf_list_[i]) != SUCCESS) { - return INTERNAL_ERROR; - } - } - return SUCCESS; -} - -Status DavinciModel::CpuModelEnqueue(uint32_t queue_id, uintptr_t out_mbuf) { - GELOGI("Set CpuKernel model enqueue task enter."); - std::shared_ptr model_enqueue = MakeShared(rt_entry_stream_); - if (model_enqueue == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelEnqueue task failed."); - return MEMALLOC_FAILED; - } - - Status status = model_enqueue->Init(queue_id, out_mbuf); - if (status != SUCCESS) { - return status; - } - cpu_task_list_.push_back(model_enqueue); - GELOGI("Set CpuKernel model enqueue task enter."); - return SUCCESS; -} - -/// @ingroup ge -/// @brief definiteness queue schedule, repeat run model. -/// @return: 0 for success / others for failed -Status DavinciModel::CpuModelRepeat() { - GELOGI("Set CpuKernel repeat task enter."); - std::shared_ptr model_repeat = MakeShared(rt_entry_stream_); - if (model_repeat == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelRepeat task failed."); - return MEMALLOC_FAILED; - } - - Status status = model_repeat->Init(runtime_model_id_); - if (status != SUCCESS) { - return status; - } - - cpu_task_list_.push_back(model_repeat); - GELOGI("Set CpuKernel repeat task success."); - return SUCCESS; -} - -Status DavinciModel::GetInputOutputDescInfo(vector &input_desc, - vector &output_desc) { - if (input_addrs_list_.empty() || input_addrs_list_[0].size() != 1) { - GELOGI("data_op_list_ is empty or input_desc size is not 1."); - } else { - vector input_formats; - GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed."); - } - - vector output_formats; - GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), "get output desc info failed"); - return SUCCESS; -} - -Status DavinciModel::GetInputOutputDescInfo(vector &input_desc, - vector &output_desc, - vector &input_formats, - vector &output_formats) { - if (input_addrs_list_.empty() || input_addrs_list_[0].size() != 1) { - GELOGE(FAILED, "OP List Pointer is null or input_desc size is not 1!"); - return FAILED; - } - - GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed"); - - GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), "get output desc info failed"); - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Get dynamic batch_info -/// @param [out] batch_info -/// @param [out] dynamic_type -/// @return execute result -/// -Status DavinciModel::GetDynamicBatchInfo(std::vector> &batch_info, int32_t &dynamic_type) const { - dynamic_type = dynamic_type_; - batch_info = batch_info_; - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Get combined dynamic dims info -/// @param [out] batch_info -/// @return None -/// -void DavinciModel::GetCombinedDynamicDims(std::vector> &batch_info) const { - batch_info.clear(); - batch_info = combined_batch_info_; -} - -/// -/// @ingroup ge -/// @brief Get user designate shape order -/// @param [out] user_input_shape_order -/// @return None -/// -void DavinciModel::GetUserDesignateShapeOrder(std::vector &user_input_shape_order) const { - user_input_shape_order.clear(); - user_input_shape_order = user_designate_shape_order_; -} - -/// -/// @ingroup ge -/// @brief Get AIPP input info -/// @param [in] index -/// @param [out] aipp_info -/// @return execute result -/// -Status DavinciModel::GetAIPPInfo(uint32_t index, AippConfigInfo &aipp_info) { - GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index); - OpDescPtr data_op = data_op_list_[index]; - if (!data_op->HasAttr(ATTR_NAME_AIPP)) { - GELOGW("GetAIPPInfo: there is not AIPP related with index %u.", index); - return ACL_ERROR_GE_AIPP_NOT_EXIST; - } - - std::unique_ptr aipp_params(new (std::nothrow) domi::AippOpParams()); - GE_CHECK_NOTNULL(aipp_params); - - ge::GeAttrValue::NAMED_ATTRS aipp_attr; - GE_CHK_BOOL_RET_STATUS(AttrUtils::GetNamedAttrs(data_op, ATTR_NAME_AIPP, aipp_attr), GE_AIPP_NOT_EXIST, - "Data node do not contain param aipp!"); - GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aipp_params.get()), "get aipp params failed"); - GELOGI("GetAIPPInfo: node data: %s, type: %s, current index: %u, current node related input rank: %u", - data_op->GetName().c_str(), data_op->GetType().c_str(), index, aipp_params->related_input_rank()); - - GE_CHK_STATUS_RET(AippUtils::ConvertAippParams2AippInfo(aipp_params.get(), aipp_info), - "convert aipp params to aipp config info failed"); - - return SUCCESS; -} - -Status DavinciModel::GetAippType(uint32_t index, InputAippType &type, size_t &aipp_index) { - GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index); - // Set default value - type = DATA_WITHOUT_AIPP; - aipp_index = 0xFFFFFFFF; // default invalid value - OpDescPtr data_op = data_op_list_[index]; - GE_CHECK_NOTNULL(data_op); - if (!data_op->HasAttr(ATTR_DATA_RELATED_AIPP_MODE)) { - GELOGW("There is no aipp releated info with index %u.", index); - return SUCCESS; - } - std::string data_mode; - (void)AttrUtils::GetStr(data_op, ATTR_DATA_RELATED_AIPP_MODE, data_mode); - if (data_mode == "static_aipp") { - type = DATA_WITH_STATIC_AIPP; - } else if (data_mode == "dynamic_aipp") { - type = DATA_WITH_DYNAMIC_AIPP; - } else if (data_mode == "dynamic_aipp_conf") { - type = DYNAMIC_AIPP_NODE; - } else { - GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, - "The info of aipp releated info %s is invalid with index %u.", data_mode.c_str(), index); - return ACL_ERROR_GE_AIPP_MODE_INVALID; - } - - if (type == DATA_WITH_DYNAMIC_AIPP) { - string releated_name; - (void)AttrUtils::GetStr(data_op, ATTR_DATA_AIPP_DATA_NAME_MAP, releated_name); - for (size_t i = 0; i < data_op_list_.size(); ++i) { - GE_CHECK_NOTNULL(data_op_list_[i]); - if (data_op_list_[i]->GetName() == releated_name) { - GELOGI("Find aipp_data [%s] index %zu from index %u", releated_name.c_str(), i, index); - aipp_index = i; - } - } - if (aipp_index == 0xFFFFFFFF) { - GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "Can not find aipp data node from index %u", index); - return ACL_ERROR_GE_AIPP_NOT_EXIST; - } - } - return SUCCESS; -} - -void DavinciModel::SetDynamicSize(const std::vector &batch_num, int32_t dynamic_type) { - batch_size_.clear(); - if (batch_num.empty()) { - GELOGD("User has not set dynammic data"); - } - for (size_t i = 0; i < batch_num.size(); i++) { - batch_size_.emplace_back(batch_num[i]); - } - - dynamic_type_ = dynamic_type; -} - -void DavinciModel::GetCurShape(std::vector &batch_info, int32_t &dynamic_type) { - if (batch_size_.empty()) { - GELOGD("User does not set dynamic size"); - } - for (size_t i = 0; i < batch_size_.size(); i++) { - GELOGI("Start to get current shape"); - batch_info.emplace_back(batch_size_[i]); - } - - dynamic_type = dynamic_type_; -} - -void DavinciModel::GetModelAttr(vector &out_shape_info) { - out_shape_info.insert(out_shape_info.end(), dynamic_output_shape_info_.begin(), dynamic_output_shape_info_.end()); -} - -Status DavinciModel::GetInputOutputDescInfoForZeroCopy(vector &input_desc, - vector &output_desc, - std::vector &input_formats, - std::vector &output_formats) { - if (input_addrs_list_.empty() || input_addrs_list_[0].size() != kOutputNum) { - GELOGE(FAILED, "OP List Pointer is null or input_desc size is not 1!"); - return FAILED; - } - - GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed"); - - GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), "get ouput desc info failed"); - - GE_CHK_BOOL_RET_STATUS(output_desc.size() == output_memory_size_list_.size(), INTERNAL_ERROR, - "output_desc size[%zu] not equal output_size_list_[%zu] size!", output_desc.size(), - output_memory_size_list_.size()); - - /// For function zero copy,the momery should be aligned by 512 bytes. - /// And, because of the cce op limit, size should be lager than the real shape size. The memory should be padded by 32 - /// bytes. - /// *size equals to ((tensorDesc->dataSize + 2 * 32 - 1) / 32) * 32; - for (size_t i = 0; i < output_memory_size_list_.size(); i++) { - output_desc[i].size = output_memory_size_list_[i]; - } - - return SUCCESS; -} - -void DavinciModel::SetInputDimsInfo(const vector &model_input_dims, Format &format, - InputOutputDescInfo &input) { - uint32_t n, c, h, w; - n = format == FORMAT_NHWC ? NHWC_DIM_N : NCHW_DIM_N; - c = format == FORMAT_NHWC ? NHWC_DIM_C : NCHW_DIM_C; - h = format == FORMAT_NHWC ? NHWC_DIM_H : NCHW_DIM_H; - w = format == FORMAT_NHWC ? NHWC_DIM_W : NCHW_DIM_W; - - if (model_input_dims.size() == static_cast(NORMAL_TENSOR_SIZE)) { - input.shape_info.num = model_input_dims[n]; - input.shape_info.height = model_input_dims[h]; - input.shape_info.width = model_input_dims[w]; - input.shape_info.channel = model_input_dims[c]; - } - for (size_t k = 0; k < model_input_dims.size(); ++k) { - input.shape_info.dims.push_back(model_input_dims[k]); - } - return; -} - -void DavinciModel::CreateInputDimsInfo(const OpDescPtr &op_desc, Format format, InputOutputDescInfo &input) { - if (is_new_model_desc_ && op_desc->HasAttr(ATTR_NAME_INPUT_DIMS)) { - // When static aipp is set, need to get the model input dims which processed by aipp - vector model_input_dims; - (void)AttrUtils::GetListInt(op_desc, ATTR_NAME_INPUT_DIMS, model_input_dims); - SetInputDimsInfo(model_input_dims, format, input); - return; - } - // judge if this data is linked dynamic aipp first, multiply batch has been considered - if (op_desc->HasAttr(ATTR_DYNAMIC_AIPP_INPUT_DIMS)) { - vector dynamic_aipp_input_dims; - (void)AttrUtils::GetListInt(op_desc, ATTR_DYNAMIC_AIPP_INPUT_DIMS, dynamic_aipp_input_dims); - SetInputDimsInfo(dynamic_aipp_input_dims, format, input); - return; - } else { - // judge if this data is multiply batch - if (!op_desc->HasAttr(ATTR_MBATCH_ORIGIN_INPUT_DIMS)) { - vector input_dims = op_desc->GetInputDescPtr(0)->GetShape().GetDims(); - SetInputDimsInfo(input_dims, format, input); - return; - } else { - vector origin_input_dims; - (void)AttrUtils::GetListInt(op_desc, ATTR_MBATCH_ORIGIN_INPUT_DIMS, origin_input_dims); - SetInputDimsInfo(origin_input_dims, format, input); - return; - } - } -} - -Status DavinciModel::InitInputDescInfo(const vector &data_op_list) { - for (const auto &op_desc : data_op_list) { - GE_CHECK_NOTNULL(op_desc->GetInputDescPtr(0)); - - InputOutputDescInfo input; - Format format = op_desc->GetInputDescPtr(0)->GetFormat(); - CreateInputDimsInfo(op_desc, format, input); - - input.data_type = op_desc->GetInputDescPtr(0)->GetDataType(); - input.name = op_desc->GetName(); - int64_t input_size = 0; - GE_CHK_STATUS_RET(TensorUtils::GetSize(*op_desc->GetInputDescPtr(0), input_size), "get input size failed."); - input.size = input_size; - input_formats_.push_back(format); - input_descs_.push_back(input); - } - return SUCCESS; -} - -Status DavinciModel::GetInputDescInfo(vector &input_descs, vector &input_formats) { - input_descs.insert(input_descs.end(), input_descs_.begin(), input_descs_.end()); - input_formats.insert(input_formats.end(), input_formats_.begin(), input_formats_.end()); - - // cause GetInputDescInfo called not only once, set is_new_model_desc_ to false after calc the model input dims - is_new_model_desc_ = false; - return SUCCESS; -} - -void DavinciModel::CreateOutput(uint32_t index, const OpDescPtr &op_desc, InputOutputDescInfo &output, - uint32_t &format_result) { - /// netoutput input tensor desc - GE_IF_BOOL_EXEC(op_desc->GetInputDescPtr(index) == nullptr, GELOGE(FAILED, "OpDesc GetInputDescPtr is nullptr"); - return ); - Format format = op_desc->GetInputDescPtr(index)->GetFormat(); - GeShape shape = op_desc->GetInputDescPtr(index)->GetShape(); - DataType data_type = op_desc->GetInputDescPtr(index)->GetDataType(); - - int64_t dims[] = {1, 1, 1, 1}; - format_result = format; - if (format == FORMAT_ND) { // for ND tensor - for (size_t i = 0; i < shape.GetDimNum() && i < (sizeof(dims) / sizeof(dims[0])); i++) { - dims[i] = shape.GetDim(i); - } - } else { // FOR FORMAT_NHWC or FORMAT_NCHW - dims[0] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_N : NCHW_DIM_N); // 0: first dim - dims[1] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_C : NCHW_DIM_C); // 1: second dim - dims[2] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_H : NCHW_DIM_H); // 2: third dim - dims[3] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_W : NCHW_DIM_W); // 3: forth dim - } - output.shape_info.num = dims[0]; // 0: first dim - output.shape_info.channel = dims[1]; // 1: second dim - output.shape_info.height = dims[2]; // 2: third dim - output.shape_info.width = dims[3]; // 3: forth dim - - if (op_desc->GetInputDescPtr(index)->GetFormat() == FORMAT_FRACTAL_Z) { // FraczToHWCK - int64_t k = shape.GetDim(0); // 0: first dim - int64_t c = shape.GetDim(1); // 1: second dim - int64_t h = shape.GetDim(2); // 2: third dim - int64_t w = shape.GetDim(3); // 3: forth dim - output.shape_info.dims.push_back(h); - output.shape_info.dims.push_back(w); - output.shape_info.dims.push_back(c); - output.shape_info.dims.push_back(k); - format_result = FORMAT_HWCN; - } else { - for (size_t j = 0; j < shape.GetDimNum(); j++) { - output.shape_info.dims.push_back(shape.GetDim(j)); - } - } - - int64_t tensor_size = 0; - if (AttrUtils::GetInt(op_desc->GetInputDescPtr(index), ATTR_NAME_SPECIAL_OUTPUT_SIZE, tensor_size) - && (tensor_size > 0)) { - GELOGI("netoutput[%s] [%d]th input has special size [%ld]", op_desc->GetName().c_str(), index, tensor_size); - } else { - (void)TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size); // no need to check value - } - output.size = static_cast(tensor_size); - output.data_type = op_desc->GetInputDescPtr(index)->GetDataType(); -} - -Status DavinciModel::InitOutputDescInfo(const vector &output_op_list) { - GELOGD("Output node size: %zu", output_op_list.size()); - for (const auto &op_desc : output_op_list) { - uint32_t out_size = static_cast(op_desc->GetInputsSize()); - for (uint32_t index = 0; index < out_size; index++) { - string output_name; - InputOutputDescInfo output; - uint32_t format_result; - CreateOutput(index, op_desc, output, format_result); - - std::vector src_name = op_desc->GetSrcName(); - std::vector src_index = op_desc->GetSrcIndex(); - GE_CHK_BOOL_RET_STATUS(src_name.size() > index && src_index.size() > index, INTERNAL_ERROR, - "construct output_name failed."); - // forward compatbility, if old om has no out_node_name, need to return output follow origin way - if (out_size == out_node_name_.size()) { - // neweast plan, the index will add to name during generate model. - bool contains_colon = out_node_name_[index].find(":") != std::string::npos; - output_name = - contains_colon ? out_node_name_[index] : out_node_name_[index] + ":" + std::to_string(src_index[index]); - } else { - output_name = std::string("output_") + std::to_string(index) + "_" + src_name[index] + "_" + - std::to_string(src_index[index]); - } - output.name = output_name; - output_descs_.push_back(output); - output_formats_.push_back(format_result); - } - } - return SUCCESS; -} - -Status DavinciModel::GetOutputDescInfo(vector &output_descs, vector &output_formats) { - output_descs.insert(output_descs.end(), output_descs_.begin(), output_descs_.end()); - output_formats.insert(output_formats.end(), output_formats_.begin(), output_formats_.end()); - return SUCCESS; -} - -Status DavinciModel::CopyInputData(const InputData &input_data, bool device_data) { - rtMemcpyKind_t kind = device_data ? RT_MEMCPY_DEVICE_TO_DEVICE : RT_MEMCPY_HOST_TO_DEVICE; - const std::vector &blobs = input_data.blobs; - for (const auto &data : new_input_data_info_) { - if (data.first >= blobs.size()) { - GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld, op_name(%s)", blobs.size(), - new_input_data_info_.size(), data.first, data.second.GetDataInfo().at(0).first, - data.second.GetOpName().c_str()); - return FAILED; - } - - const DataBuffer &data_buf = blobs[data.first]; - if (data_buf.length == 0) { - GELOGW("No data need to memcpy!"); - return SUCCESS; - } - uint64_t data_size = data.second.GetDataSize(); - GE_CHK_BOOL_RET_STATUS(data_size >= data_buf.length, PARAM_INVALID, - "input data size(%lu) does not match model required size(%lu), op_name(%s) ret failed.", - data_buf.length, data_size, data.second.GetOpName().c_str()); - void *mem_addr = data.second.GetBasicAddr(); - void *data_buf_addr = reinterpret_cast(reinterpret_cast(data_buf.data)); - uint64_t data_buf_length = data_buf.length; - GELOGI("CopyPlainData memcpy graph_%u type[F] input[%s] rank[%u] dst[%p] src[%p] mem_size[%lu] datasize[%lu]", - runtime_param_.graph_id, data.second.GetOpName().c_str(), data.first, mem_addr, data_buf_addr, data_size, - data_buf_length); - GE_CHK_RT_RET(rtMemcpy(mem_addr, data_size, data_buf_addr, data_buf_length, kind)); - } - - return SUCCESS; -} - -Status DavinciModel::SyncVarData() { - GELOGI("Sync var data, model id:%u", model_id_); - Status ret = SUCCESS; - - OpDescPtr global_step = GetVariableOp(NODE_NAME_GLOBAL_STEP); - if (global_step != nullptr) { - auto v_output_size = ModelUtils::GetOutputSize(global_step); - auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param_, global_step); - if (v_output_size.empty() || v_output_addr.empty()) { - GELOGE(PARAM_INVALID, "global step op:%s not set output", global_step->GetName().c_str()); - return PARAM_INVALID; - } - std::vector v_step; - v_step.push_back(iterator_count_); - GE_CHK_RT_RET(rtMemcpy(v_output_addr[0], v_output_size[0], v_step.data(), v_step.size() * sizeof(uint64_t), - RT_MEMCPY_HOST_TO_DEVICE)); - } - - return ret; -} - -Status DavinciModel::InitModelProfile() { - for (const auto &task : task_list_) { - GE_CHECK_NOTNULL(task); - const FusionOpInfo *fusion_op_info = task->GetFusionOpInfo(); - // when type is RT_MODEL_TASK_KERNEL, ctx is not null - if ((fusion_op_info == nullptr) || fusion_op_info->original_op_names.empty()) { - continue; - } - - GELOGI("task.id = %u, opNum = %zu", task->GetTaskID(), fusion_op_info->original_op_names.size()); - op_id_map_.insert(std::make_pair(fusion_op_info->op_index, task->GetTaskID())); - } - - std::set task_id_set; - using CIT = std::multimap::const_iterator; - using Range = std::pair; - for (const auto &task : task_list_) { - GE_CHECK_NOTNULL(task); - const FusionOpInfo *fusion_op_info = task->GetFusionOpInfo(); - if ((fusion_op_info == nullptr) || fusion_op_info->original_op_names.empty()) { - continue; - } - - if (task_id_set.count(task->GetTaskID()) > 0) { - continue; - } - - const auto &op_desc = GetOpByIndex(fusion_op_info->op_index); - GE_CHK_BOOL_EXEC(op_desc != nullptr, return FAILED, "index: %u out of range", fusion_op_info->op_index); - - ProfileInfo profile; - profile.fusion_info = *fusion_op_info; - Range range = op_id_map_.equal_range(fusion_op_info->op_index); - for (CIT range_idx = range.first; range_idx != range.second; ++range_idx) { - profile.task_count++; - task_id_set.insert(range_idx->second); - } - - // memory info - TaskMemInfo &mem_info = profile.memory_info; - const auto input_size = ModelUtils::GetInputSize(op_desc); - const auto output_size = ModelUtils::GetOutputSize(op_desc); - const auto workspace_size = ModelUtils::GetWorkspaceSize(op_desc); - const auto weight_size = ModelUtils::GetWeightSize(op_desc); - mem_info.input_size = std::accumulate(input_size.begin(), input_size.end(), 0); - mem_info.output_size = std::accumulate(output_size.begin(), output_size.end(), 0); - mem_info.workspace_size = std::accumulate(workspace_size.begin(), workspace_size.end(), 0); - mem_info.weight_size = std::accumulate(weight_size.begin(), weight_size.end(), 0); - mem_info.total_size = mem_info.weight_size + mem_info.input_size + mem_info.output_size + mem_info.workspace_size; - - profile_list_.emplace_back(profile); - } - - GELOGI("fusion task size: %zu, profile info size: %zu", op_id_map_.size(), profile_list_.size()); - return SUCCESS; -} - -Status DavinciModel::SinkModelProfile() { - // profiling plugin must be registered - auto &prof_mgr = ProfilingManager::Instance(); - ReporterData reporter_data{}; - // report model data tag name - std::string tag_name("model_load_info_" + std::to_string(this->Id())); - GE_CHK_BOOL_EXEC(memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN, tag_name.c_str(), tag_name.size()) == EOK, - return FAILED, "Sink model tag memcpy error."); - - // Model Header - std::string name = om_name_.empty() ? name_ : om_name_; - size_t name_len = name.size(); - reporter_data.deviceId = device_id_; - reporter_data.data = (unsigned char *)&name_len; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - reporter_data.data = (unsigned char *)name.c_str(); - reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - uint32_t model_id = this->Id(); - reporter_data.data = (unsigned char *)&model_id; - reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - // Load Start/End Time - int64_t start_time = this->GetLoadBeginTime(); - reporter_data.data = (unsigned char *)&start_time; - reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - int64_t end_time = this->GetLoadEndTime(); - reporter_data.data = (unsigned char *)&end_time; - reporter_data.dataLen = sizeof(int64_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - using CIT = std::multimap::const_iterator; - using Range = std::pair; - for (const ProfileInfo &profile : profile_list_) { - // op name after fusion - string fusion_op_name = profile.fusion_info.op_name; - int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); - reporter_data.data = (unsigned char *)&fusion_op_name_len; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - reporter_data.data = (unsigned char *)fusion_op_name.c_str(); - reporter_data.dataLen = fusion_op_name_len; - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - // original op name before fusion - uint32_t op_num = profile.fusion_info.original_op_names.size(); - reporter_data.data = (unsigned char *)&op_num; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - for (uint32_t k = 0; k < op_num; k++) { - std::string op_name = profile.fusion_info.original_op_names[k]; - int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); - reporter_data.data = (unsigned char *)&op_name_len; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - reporter_data.data = (unsigned char *)op_name.c_str(); - reporter_data.dataLen = op_name_len; - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - } - - // stream id info - uint32_t streamId = profile.fusion_info.stream_id; - reporter_data.data = (unsigned char *)&streamId; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - // memory info - reporter_data.data = (unsigned char *)&profile.memory_info; - reporter_data.dataLen = sizeof(profile.memory_info); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - // task info - reporter_data.data = (unsigned char *)&profile.task_count; - reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - Range task_range = op_id_map_.equal_range(profile.fusion_info.op_index); - for (CIT idx = task_range.first; idx != task_range.second; ++idx) { - uint32_t task_id = idx->second; - reporter_data.data = (unsigned char *)&task_id; - reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - } - } - - return SUCCESS; -} - -Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { - // profiling plugin must be registered - auto &prof_mgr = ProfilingManager::Instance(); - ReporterData reporter_data{}; - // report model data tag name - std::string tag_name; - tag_name.append("model_time_info_") - .append(std::to_string(this->Id())) - .append("_") - .append(std::to_string(current_data.index)); - - GE_CHK_BOOL_EXEC(memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN, tag_name.c_str(), tag_name.size()) == EOK, - return FAILED, "Sink model tag memcpy error."); - // device id - reporter_data.deviceId = device_id_; - - // Model Header - string name; - if (!om_name_.empty()) { - name = om_name_; - } else { - name = name_; - } - size_t name_len = name.size(); - reporter_data.data = (unsigned char *)&name_len; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - reporter_data.data = (unsigned char *)name.c_str(); - reporter_data.dataLen = name.size(); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u.", this->Id()); - - // request id - uint64_t request_id = current_data.request_id; - reporter_data.data = (unsigned char *)&request_id; - reporter_data.dataLen = sizeof(uint32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); - - // thread id - int32_t thread_id = GetDataInputTid(); - reporter_data.data = (unsigned char *)&thread_id; - reporter_data.dataLen = sizeof(int32_t); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); - - // time info - time_info_.modelId = this->Id(); - reporter_data.data = (unsigned char *)&time_info_; - reporter_data.dataLen = sizeof(struct timeInfo); - GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, - "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); - - return SUCCESS; -} - -void DavinciModel::SetProfileTime(ModelProcStage stage, int64_t endTime) { - int64_t time = endTime; - - if (time == 0) { - mmTimespec timespec = mmGetTickCount(); - time = timespec.tv_sec * 1000 * 1000 * 1000 + timespec.tv_nsec; // 1000 ^ 3 converts second to nanosecond - } - - switch (stage) { - case MODEL_LOAD_START: - load_begin_time_ = time; - break; - case MODEL_LOAD_END: - load_end_time_ = time; - break; - case MODEL_PRE_PROC_START: - time_info_.processBeginTime = time; - break; - case MODEL_PRE_PROC_END: - time_info_.processEndTime = time; - break; - case MODEL_INFER_START: - time_info_.inferenceBeginTime = time; - break; - case MODEL_INFER_END: - time_info_.inferenceEndTime = time; - break; - case MODEL_AFTER_PROC_START: - time_info_.dumpBeginTime = time; - break; - case MODEL_AFTER_PROC_END: - time_info_.dumpEndTime = time; - break; - default: - break; - } - return; -} - -/// -/// @ingroup ge -/// @brief send Output Op result to upper layer -/// @already malloced in ModelLoad, no need to malloc again -/// @param [in] data_id: the index of output_data -/// @param [in/out] output_data: real user output_data -/// @param [in] kind: the kind of rtMemcpy -/// @return Status result -/// @author -/// -Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, rtMemcpyKind_t kind) { - if (output_addrs_list_.empty()) { - Status ret = SyncVarData(); - return ret; - } - - output_data.index = data_id; - output_data.model_id = model_id_; - if (output_data.blobs.size() != new_output_data_info_.size()) { - GELOGE(FAILED, "Output data buffer num=%zu not equal model data num=%zu", output_data.blobs.size(), - new_output_data_info_.size()); - return FAILED; - } - - std::vector &blobs = output_data.blobs; - size_t idx = 0; - for (const auto &output : new_output_data_info_) { - if (output.first >= blobs.size()) { - GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld", blobs.size(), - new_input_data_info_.size(), output.first, output.second.GetDataInfo().at(0).first); - return FAILED; - } - - if ((kind == RT_MEMCPY_DEVICE_TO_DEVICE) && (copy_only_addrs_.count(output.second.GetBasicAddr()) == 0)) { - continue; // Skip: Feed by zero copy. - } - - DataBuffer &buffer = blobs[output.first]; - uint64_t mem_size = static_cast(output.second.GetDataSize()); - if ((buffer.length == 0) || (mem_size == 0)) { - GELOGI("Length of data is zero, No need copy. output tensor index=%u", output.first); - continue; - } - if (is_dynamic_) { - GELOGI("No need to check output data size."); - } else if (buffer.length < mem_size) { - GELOGE(FAILED, "Tensor data size=%lu, buffer size=%lu", mem_size, buffer.length); - return FAILED; - } else if (buffer.length > mem_size) { - GELOGW("Tensor data size=%lu, buffer size=%lu", mem_size, buffer.length); - } - int64_t data_size = output.second.GetDataSize(); - - if (is_online_infer_dynamic_) { - if (merge_nodes_gear_and_real_out_size_info_.find(idx) != merge_nodes_gear_and_real_out_size_info_.end()) { - auto gear_and_real_out_size_info = merge_nodes_gear_and_real_out_size_info_[idx]; - data_size = gear_and_real_out_size_info[cur_dynamic_dims_]; - } - } - uint64_t buffer_length = buffer.length; - void *buffer_addr = reinterpret_cast(reinterpret_cast(buffer.data)); - - GELOGI("[IMAS]CopyPlainData memcpy graph_%u type[F] output[%u] memaddr[%p] mem_size[%lu] datasize[%lu]", - runtime_param_.graph_id, output.first, output.second.GetBasicAddr(), data_size, buffer_length); - GE_CHK_RT_RET(rtMemcpy(buffer_addr, buffer_length, output.second.GetBasicAddr(), data_size, kind)); - idx++; - } - return SUCCESS; -} - -Status DavinciModel::InitOutputTensorInfo(const OpDescPtr &op_desc) { - size_t input_num = op_desc->GetInputsSize(); - if (is_getnext_sink_dynamic_) { - input_num = input_num - kGetDynamicDimsCount; - } - - for (size_t i = 0; i < input_num; ++i) { - int64_t size = 0; - auto input_desc = op_desc->GetInputDescPtr(i); - GE_CHECK_NOTNULL(input_desc); - auto ret = TensorUtils::GetTensorSizeInBytes(*input_desc, size); - GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, - GELOGE(ret, "Get size from TensorDesc failed, op:%s, input id:%zu", op_desc->GetName().c_str(), i); - return ret); - std::vector output_shape = input_desc->GetShape().GetDims(); - if (is_online_infer_dynamic_) { - if (merge_nodes_gear_and_real_out_size_info_.find(i) != merge_nodes_gear_and_real_out_size_info_.end()) { - auto gear_and_real_out_size_info = merge_nodes_gear_and_real_out_size_info_[i]; - size = gear_and_real_out_size_info[cur_dynamic_dims_]; - auto gear_and_real_out_shape_info = merge_nodes_gear_and_real_out_shape_info_[i]; - output_shape = gear_and_real_out_shape_info[cur_dynamic_dims_]; - is_dynamic_ = true; - } - } - GELOGI("Output size is %ld, output shape is %s.", size, formats::JoinToString(output_shape).c_str()); - output_buffer_size_.push_back(size); - output_shape_info_.push_back(output_shape); - } - - return SUCCESS; -} - -Status DavinciModel::GenOutputTensorInfo(OutputData *output_data, vector &outputs) { - GE_CHECK_NOTNULL(output_data); - if (!output_data->blobs.empty()) { - GELOGI("No need to generate output tensor info, model id:%u", model_id_); - return SUCCESS; - } - - GELOGI("Output blobs size:%zu, model id:%u", output_buffer_size_.size(), model_id_); - for (size_t i = 0; i < output_buffer_size_.size(); ++i) { - std::unique_ptr data_buf(new (std::nothrow) uint8_t[output_buffer_size_[i]]); - if (data_buf == nullptr) { - GELOGE(GE_GRAPH_MALLOC_FAILED, "Malloc buffer failed."); - return GE_GRAPH_MALLOC_FAILED; - } - output_data->blobs.push_back({data_buf.get(), static_cast(output_buffer_size_[i]), false}); - ge::OutputTensorInfo output; - output.dims = output_shape_info_[i]; - output.data = std::move(data_buf); - output.length = output_buffer_size_[i]; - outputs.emplace_back(std::move(output)); - GELOGD("Output index:%zu, output dims is %s, data length:%lu.", i, - formats::JoinToString(output.dims).c_str(), output.length); - } - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief send Output Op result to upper layer -/// @already malloced in ModelLoad, no need to malloc again -/// @param [in] data_id: the index of output_data -/// @param [in] rslt_flg: result flag -/// @param [in] seq_end_flag: sequence end flag -/// @param [out] output_data: real user output_data -/// @return Status result -/// @author -/// -Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const bool seq_end_flag, - OutputData *output_data) { - GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null."); - std::vector outputs; - - // return result is not required - if (!rslt_flg && !seq_end_flag) { - GELOGW("Compute failed, model id: %u", model_id_); - auto model_manager = ModelManager::GetInstance(); - GE_CHECK_NOTNULL(model_manager); - auto exception_infos = model_manager->GetExceptionInfos(); - if (exception_infos.size() > 0) { - GE_CHK_STATUS_RET(data_dumper_.DumpExceptionInfo(exception_infos), "Dump exception info failed"); - } else { - GELOGI("Exception info is null"); - } - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed."); - return INTERNAL_ERROR; - } - - if (output_addrs_list_.empty()) { - GELOGW("Output tensor list is empty, model id: %u", model_id_); - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed."); - return INTERNAL_ERROR; - } - - GE_CHECK_NOTNULL(output_data); - output_data->index = data_id; - output_data->model_id = model_id_; - - if (is_getnext_sink_dynamic_) { - GELOGD("Reinit cur dynamic dims when getnext sink dynamic."); - cur_dynamic_dims_.clear(); - cur_dynamic_dims_.resize(shape_of_cur_dynamic_dims_); - auto ret = rtMemcpy(cur_dynamic_dims_.data(), shape_of_cur_dynamic_dims_ * sizeof(int64_t), - netoutput_last_input_addr_, netoutput_last_input_size_, RT_MEMCPY_DEVICE_TO_HOST); - GE_CHK_RT_RET(ret); - } - - GELOGD("Cur dynamic dims is %s.", formats::JoinToString(cur_dynamic_dims_).c_str()); - if (GenOutputTensorInfo(output_data, outputs) != SUCCESS) { - return INTERNAL_ERROR; - } - - if (CopyOutputData(data_id, *output_data, RT_MEMCPY_DEVICE_TO_HOST) != SUCCESS) { - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed"); - return INTERNAL_ERROR; - } - - if (seq_end_flag) { - GELOGW("End of sequence, model id: %u", model_id_); - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, END_OF_SEQUENCE, outputs), "OnCompute Done failed."); - return END_OF_SEQUENCE; - } - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed"); - return SUCCESS; -} -/// -/// @ingroup ge -/// @brief return not output to upper layer for cloud case -/// @param [in] data_id -/// @return Status result -/// -Status DavinciModel::ReturnNoOutput(uint32_t data_id) { - GELOGI("ReturnNoOutput model id:%u", model_id_); - - GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null!"); - std::vector outputs; - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed."); - return SUCCESS; -} - -void *DavinciModel::Run(DavinciModel *model) { - GE_CHK_BOOL_EXEC(model != nullptr, - CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); - return nullptr, "model_pointer is null!") - bool seq_end_flag = false; - uint32_t model_id = model->Id(); - uint32_t device_id = model->GetDeviceId(); - - GELOGI("Model Run thread start, model_id:%u.", model_id); - rtError_t rt_ret = rtSetDevice(static_cast(device_id)); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(FAILED, "Model run rtsetdevice failed."); - return nullptr; - } - // DeviceReset before thread run finished! - GE_MAKE_GUARD(not_used_var, [&] { GE_CHK_RT(rtDeviceReset(device_id)); }); - - while (model->RunFlag()) { - bool rslt_flg = true; - if (model->GetDataInputer() == nullptr) { - GELOGW("Data inputer is nullptr."); - CsaInteract::GetInstance().StoreInternalErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); - break; - } - - std::shared_ptr data_wrapper; - Status ret = model->GetDataInputer()->Pop(data_wrapper); - if (data_wrapper == nullptr || ret != SUCCESS) { - GELOGI("data_wrapper is null!"); - continue; - } - GELOGI("Getting the input data, model_id:%u", model_id); - GE_IF_BOOL_EXEC(!model->RunFlag(), break); - - InputData current_data = data_wrapper->GetInput(); - GELOGI("Model thread Run begin, model id:%u, data index:%u.", model_id, current_data.index); - GE_TIMESTAMP_START(Model_SyncVarData); - ret = model->SyncVarData(); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); - continue, "Copy input data to model failed."); // [No need to check value] - GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(Model_SyncVarData, "Model Run SyncVarData")); - - GELOGI("Copy input data, model id:%u", model_id); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), - model->SetProfileTime(MODEL_PRE_PROC_START)); - ret = model->CopyInputData(current_data, false); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); - continue, "Copy input data to model failed."); // [No need to check value] - if (model->is_online_infer_dynamic_ && !model->is_getnext_sink_dynamic_) { - model->cur_dynamic_dims_.clear(); - GE_IF_BOOL_EXEC(current_data.blobs.empty(), break); - auto shape_data_buffer_data = current_data.blobs.back().data; - auto shape_data_buffer_length = current_data.blobs.back().length; - model->cur_dynamic_dims_.assign(reinterpret_cast(shape_data_buffer_data), - reinterpret_cast(shape_data_buffer_data) + - shape_data_buffer_length / sizeof(int64_t)); - GELOGD("Data: cur dynamic dims is %s", formats::JoinToString(model->cur_dynamic_dims_).c_str()); - delete[] reinterpret_cast(current_data.blobs.back().data); - current_data.blobs.pop_back(); - } - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); - GE_TIMESTAMP_START(rtModelExecute); - GELOGI("rtModelExecute start."); - rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; - (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - GELOGI("rtModelExecute end"); - GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); - - GE_TIMESTAMP_START(rtStreamSynchronize); - GELOGI("rtStreamSynchronize start."); - rt_ret = rtStreamSynchronize(model->rt_model_stream_); - if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { - seq_end_flag = true; - } - if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { - GELOGI("The model with multiple datasets aborts normally."); - } else { - GE_IF_BOOL_EXEC( - rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); - (void)model->ReturnResult(current_data.index, false, seq_end_flag, - data_wrapper->GetOutput()); // [No need to check value] - CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); - continue); - } - - GELOGI("rtStreamSynchronize end."); - GE_IF_BOOL_EXEC(model->is_first_execute_, - GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), - model->SetProfileTime(MODEL_AFTER_PROC_START)); - GE_TIMESTAMP_START(ReturnResult3); - // copy output data from device to host - GE_IF_BOOL_EXEC(!model->output_addrs_list_.empty(), - (void)model->ReturnResult(current_data.index, rslt_flg, false, data_wrapper->GetOutput())) - // copy output data from device to host for variable graph - GE_IF_BOOL_EXEC(model->output_addrs_list_.empty(), (void)model->ReturnNoOutput(current_data.index)); - GE_IF_BOOL_EXEC(model->is_first_execute_, - GE_TIMESTAMP_EVENT_END(ReturnResult3, "GraphExcute::CopyDataFromDeviceToHost")); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), - model->SetProfileTime(MODEL_AFTER_PROC_END)); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), (void)model->SinkTimeProfile(current_data)); - - model->iterator_count_++; - model->is_first_execute_ = false; - GELOGI("run iterator count is %lu", model->iterator_count_); - } - - CsaInteract::GetInstance().WriteInternalErrorCode(); - GELOGI("Model run end, model id:%u", model->model_id_); - return nullptr; -} - -/// -/// @ingroup ge -/// @brief call API provided by data inputer to destroy thread -/// @param [in] no -/// @return Status Destroy result -/// @author -/// -Status DavinciModel::DestroyThread() { - GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, "data_inputer_ is nullptr."); - - run_flg_ = false; - - data_inputer_->Stop(); - - if (thread_id_.joinable()) { - thread_id_.join(); - } - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief create model std::thread, -/// @brief start to execute Model -/// @param [in] no -/// @return Status create model thread and execute result -/// @author -/// -Status DavinciModel::ModelRunStart() { - GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, "data_inputer_ is nullptr."); - - LockRunFlg(); - GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); }); - - GE_CHK_BOOL_RET_STATUS(!run_flg_, INTERNAL_ERROR, "Model already started."); - - run_flg_ = true; - - // create stream instance which rt_model_handel is running on - GE_CHK_RT_RET(rtStreamCreate(&rt_model_stream_, priority_)); - is_inner_model_stream_ = true; - - string opt = "0"; - (void)ge::GetContext().GetOption(OPTION_GE_MAX_DUMP_OP_NUM, opt); // option may not be set up, no need to check value - int64_t maxDumpOpNum = std::strtol(opt.c_str(), nullptr, kDecimal); - maxDumpOpNum_ = maxDumpOpNum; - - CREATE_STD_THREAD(thread_id_, DavinciModel::Run, this); - GELOGI("model tread create success, model id:%u.", model_id_); - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief call API provided by data inputer and destroy model Thread -/// @param [in] no -/// @return Status Destroy result -/// @author -/// -Status DavinciModel::ModelRunStop() { - LockRunFlg(); - GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); }); - - GE_IF_BOOL_EXEC(!run_flg_, return SUCCESS); - - GE_CHK_STATUS_RET(DestroyThread(), "DestoyThead failed."); - - return SUCCESS; -} - -void DavinciModel::UnbindTaskSinkStream() { - // unbinding hcom stream - UnbindHcomStream(); - if (is_stream_list_bind_) { - for (size_t i = 0; i < stream_list_.size(); i++) { - // unbind rt_model_handle and streams - GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, stream_list_[i]) != RT_ERROR_NONE, - "Unbind stream from model failed! Index: %zu", i); - } - } - - if (is_inner_model_stream_) { - if (!input_queue_ids_.empty() || !output_queue_ids_.empty()) { - GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_model_stream_) != RT_ERROR_NONE, "Unbind stream failed!"); - } - // destroy stream that is bound with rt_model - GE_LOGW_IF(rtStreamDestroy(rt_model_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed.") - } - - if (is_pure_head_stream_ && rt_head_stream_ != nullptr) { - GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_head_stream_) != RT_ERROR_NONE, "Unbind stream failed!"); - GE_LOGW_IF(rtStreamDestroy(rt_head_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed."); - rt_head_stream_ = nullptr; - } - - if (rt_entry_stream_ != nullptr) { - GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_entry_stream_) != RT_ERROR_NONE, "Unbind stream failed!"); - GE_LOGW_IF(rtStreamDestroy(rt_entry_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed."); - rt_entry_stream_ = nullptr; - } -} - -void *DavinciModel::GetRunAddress(void *addr) const { - if (fixed_mem_base_ == reinterpret_cast(mem_base_)) { - return addr; - } - - uintptr_t ptr = reinterpret_cast(addr); - if ((fixed_mem_base_ <= ptr) && (ptr < fixed_mem_base_ + runtime_param_.mem_size)) { - return mem_base_ + (ptr - fixed_mem_base_); - } else { - return addr; - } -} - -Status DavinciModel::CreateKnownZeroCopyMap(const vector &inputs, const vector &outputs) { - GELOGI("in, inputs size: %zu, input addr size: %zu, outputs size: %zu, output addr size: %zu", - inputs.size(), input_addrs_list_.size(), outputs.size(), output_addrs_list_.size()); - if (inputs.size() > input_addrs_list_.size()) { - GELOGE(FAILED, "input data addr %zu should less than input op num %zu.", inputs.size(), input_addrs_list_.size()); - return FAILED; - } - // remove zero copy addr in last iteration - known_input_data_info_.clear(); - known_output_data_info_.clear(); - for (size_t i = 0; i < inputs.size(); ++i) { - const vector &addr_list = input_addrs_list_[i]; - void *addr = GetRunAddress(addr_list[kDataIndex]); - known_input_data_info_[addr] = inputs[i]; - GELOGI("input %zu, v addr %p, r addr %p, p addr %p", i, addr_list[kDataIndex], addr, inputs[i]); - } - - if (output_addrs_list_.empty()) { - GELOGW("output op num in graph is %zu", output_addrs_list_.size()); - return SUCCESS; - } - const vector &addr_list = output_addrs_list_.front(); - for (size_t i = 0; i < addr_list.size() && i < outputs.size(); ++i) { - void *addr = GetRunAddress(addr_list[i]); - known_output_data_info_[addr] = outputs[i]; - GELOGI("output %zu, v addr %p, r addr %p, p addr %p", i, addr_list[i], addr, outputs[i]); - } - - GELOGI("success, known input data info size: %zu, known output data info size: %zu", - known_input_data_info_.size(), known_output_data_info_.size()); - return SUCCESS; -} - -void DavinciModel::SetTotalIOAddrs(const vector &io_addrs) { - if (fixed_mem_base_ == reinterpret_cast(mem_base_)) { - total_io_addrs_.insert(total_io_addrs_.end(), io_addrs.begin(), io_addrs.end()); - return; - } - - for (size_t i = 0; i < io_addrs.size(); ++i) { - total_io_addrs_.emplace_back(GetRunAddress(io_addrs[i])); - } -} - -Status DavinciModel::UpdateKnownZeroCopyAddr(vector &total_io_addrs) { - if (fixed_mem_base_ != reinterpret_cast(mem_base_)) { - for (size_t i = 0; i < total_io_addrs.size(); ++i) { - total_io_addrs[i] = GetRunAddress(total_io_addrs[i]); - } - } - - for (size_t i = 0; i < total_io_addrs.size(); ++i) { - auto it_in = known_input_data_info_.find(total_io_addrs[i]); - if (it_in != known_input_data_info_.end()) { - GELOGI("input %zu, v addr %p, p addr %p", i, total_io_addrs[i], known_input_data_info_.at(total_io_addrs[i])); - total_io_addrs[i] = known_input_data_info_.at(total_io_addrs[i]); - } - auto it_out = known_output_data_info_.find(total_io_addrs[i]); - if (it_out != known_output_data_info_.end()) { - GELOGI("output %zu, v addr %p, p addr %p", i, total_io_addrs[i], known_output_data_info_.at(total_io_addrs[i])); - total_io_addrs[i] = known_output_data_info_.at(total_io_addrs[i]); - } - } - GELOGI("success, total io addrs size: %zu", total_io_addrs.size()); - return SUCCESS; -} - -Status DavinciModel::UpdateKnownNodeArgs(const vector &inputs, const vector &outputs) { - GELOGI("DavinciModel::UpdateKnownNodeArgs in"); - GE_CHK_STATUS_RET(CreateKnownZeroCopyMap(inputs, outputs), - "DavinciModel::UpdateKnownNodeArgs create map for input/output zero copy."); - if (!base_addr_not_changed_) { - total_io_addrs_.clear(); - orig_total_io_addrs_.clear(); - for (size_t task_index = 0; task_index < task_list_.size(); ++task_index) { - auto &task = task_list_[task_index]; - if (task != nullptr) { - Status ret = task->UpdateArgs(); - if (ret != SUCCESS) { - GELOGE(FAILED, "task %zu created by davinci model is nullptr.", task_index); - return FAILED; - } - } - } - // cache latest iterator io addr - orig_total_io_addrs_ = total_io_addrs_; - } else { - total_io_addrs_ = orig_total_io_addrs_; - } - GE_CHK_STATUS_RET(UpdateKnownZeroCopyAddr(total_io_addrs_), "DavinciModel::UpdateKnownZeroCopyAddr failed."); - - if (total_args_size_ == 0) { - GELOGW("DavinciModel::UpdateKnownNodeArgs device args %p, dst size %u, pass rtMemcpy.", args_, total_args_size_); - } else { - uint32_t total_addr_size = total_io_addrs_.size() * sizeof(uint64_t); - GELOGI("DavinciModel::UpdateKnownNodeArgs device args %p, dst size %u, src size %u", args_, total_args_size_, - total_addr_size); - - Status rt_ret = - rtMemcpy(args_, total_args_size_, total_io_addrs_.data(), total_addr_size, RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(rt_ret, "rtMemcpy error, ret: Ox%X", rt_ret); return FAILED;) - } - - GELOGI("DavinciModel::UpdateKnownNodeArgs success"); - return SUCCESS; -} - -Status DavinciModel::InitTaskInfo(domi::ModelTaskDef &model_task_def) { - GELOGI("InitTaskInfo in, task size %d", model_task_def.task().size()); - task_list_.resize(model_task_def.task_size()); - for (int i = 0; i < model_task_def.task_size(); ++i) { - // dynamic shape will create task_list_ before - const domi::TaskDef &task = model_task_def.task(i); - if (this->task_list_[i] == nullptr) { - task_list_[i] = TaskInfoFactory::Instance().Create(static_cast(task.type())); - } - GE_CHECK_NOTNULL(task_list_[i]); - Status ret = task_list_[i]->Init(task, this); - if (ret != SUCCESS) { - GELOGE(ret, "Task index %d init failed.", i); - return ret; - } - } - GELOGI("InitTaskInfo out"); - return SUCCESS; -} - -Status DavinciModel::MallocKnownArgs() { - GELOGI("DavinciModel::MallocKnownArgs in"); - const auto &model_task_def = ge_model_->GetModelTaskDefPtr(); - if (model_task_def->task_size() == 0) { - GELOGW("DavinciModel::MallocKnownArgs davincimodel has no task info."); - return SUCCESS; - } - task_list_.resize(model_task_def->task_size()); - for (int32_t i = 0; i < model_task_def->task_size(); ++i) { - const domi::TaskDef &taskdef = model_task_def->task(i); - task_list_[i] = TaskInfoFactory::Instance().Create(static_cast(taskdef.type())); - GE_CHECK_NOTNULL(task_list_[i]); - Status ret = task_list_[i]->CalculateArgs(taskdef, this); - if (ret != SUCCESS) { - GELOGE(ret, "TaskInfo CalculateArgs failed."); - return ret; - } - } - // malloc args memory - if (total_args_size_ == 0) { - GELOGW("DavinciModel::MallocKnownArgs total_args_size_ equals to zero."); - return SUCCESS; - } - - rtError_t rt_ret = rtMalloc(&args_, total_args_size_, RT_MEMORY_HBM); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - // malloc dynamic and static hybrid memory - if (total_hybrid_args_size_ != 0) { - rt_ret = rtMalloc(&hybrid_addrs_, total_hybrid_args_size_, RT_MEMORY_HBM); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - } - // malloc fixed addr memory, eg: rts op - if (total_fixed_addr_size_ != 0) { - GELOGI("Begin to allocate fixed addr."); - rt_ret = rtMalloc(&fixed_addrs_, total_fixed_addr_size_, RT_MEMORY_HBM); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - } - - GELOGI("DavinciModel::MallocKnownArgs success, total args size %u. total fixed addr size %ld", total_args_size_, - total_fixed_addr_size_); - return SUCCESS; -} - -Status DavinciModel::DistributeTask() { - GELOGI("do Distribute."); - for (auto &task : cpu_task_list_) { - if (task == nullptr) { - GELOGW("task is null"); - continue; - } - GE_CHK_STATUS_RET(task->Distribute()); - } - - task_desc_info_.clear(); - bool flag = GetL1FusionEnableOption(); - char skt_enable_env[MMPA_MAX_PATH] = { 0x00 }; - INT32 res = mmGetEnv("SKT_ENABLE", skt_enable_env, MMPA_MAX_PATH); - int64_t env_flag = (res == EN_OK) ? std::strtol(skt_enable_env, nullptr, kDecimal) : 0; - if (env_flag != 0) { - flag = true; - } - - const auto &model_task_def = ge_model_->GetModelTaskDefPtr(); - for (size_t task_index = 0; task_index < task_list_.size(); ++task_index) { - auto &task_def = model_task_def->task(task_index); - auto &task = task_list_.at(task_index); - GE_CHK_STATUS_RET(task->Distribute(), "Task[%zu] distribute fail", task_index); - // for data dump - auto op_index = std::max(task_def.kernel().context().op_index(), - task_def.kernel_ex().op_index()); - OpDescPtr op = GetOpByIndex(op_index); - GE_CHECK_NOTNULL(op); - - if (reinterpret_cast(task->GetDumpArgs()) != nullptr) { - bool call_dump = GetDumpProperties().IsLayerNeedDump(name_, om_name_, op->GetName()) && task->CallSaveDumpInfo(); - if (call_dump || is_op_debug_reg_) { - SaveDumpTask(task->GetTaskID(), task->GetStreamId(), op, task->GetDumpArgs()); - } - } - - auto task_type = static_cast(task_def.type()); - bool no_need_profiling = (task_type != RT_MODEL_TASK_KERNEL) && (task_type != RT_MODEL_TASK_KERNEL_EX); - GE_IF_BOOL_EXEC(no_need_profiling, continue); - - SaveDumpOpInfo(runtime_param_, op, task->GetTaskID(), task->GetStreamId()); - // Load task info for profiling - TaskDescInfo task_desc_info; - if (!om_name_.empty()) { - task_desc_info.model_name = om_name_; - } else { - task_desc_info.model_name = name_; - } - task_desc_info.op_name = op->GetName(); - task_desc_info.block_dim = task_def.kernel().block_dim(); - task_desc_info.task_id = task->GetTaskID(); - task_desc_info.stream_id = task->GetStreamId(); - task_desc_info.shape_type = "static"; - task_desc_info.cur_iter_num = 0; - task_desc_info_.emplace_back(task_desc_info); - if (flag) { - if (task->GetSktTaskID() != 0xFFFFFFFF) { - TaskDescInfo task_desc_info; - string op_name = "super_kernel_" + to_string(task_index); - task_desc_info.op_name = op_name; - task_desc_info.task_id = task->GetSktTaskID(); - task_desc_info_.emplace_back(task_desc_info); - } - } - } - // launch dump kernel to aicpu - GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "Load dump info failed."); - return SUCCESS; -} - -void DavinciModel::SetEndGraphId(uint32_t task_id, uint32_t stream_id) { - auto all_dump_model = GetDumpProperties().GetAllDumpModel(); - bool findByOmName = all_dump_model.find(om_name_) != all_dump_model.end(); - bool findByModelName = all_dump_model.find(name_) != all_dump_model.end(); - if (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end() || findByOmName || findByModelName) { - GELOGI("start save end_graph_info to dumper, task_id is %u, stream_id is %u", task_id, stream_id); - data_dumper_.SaveEndGraphId(task_id, stream_id); - } -} - -/// -/// @ingroup ge -/// @brief Set copy only for No task feed NetOutput address. -/// @return None. -/// -void DavinciModel::SetCopyOnlyOutput() { - for (const auto &output_outside_addrs : new_output_outside_addrs_) { - ZeroCopyOffset output_outside = output_outside_addrs.second; - for (uint32_t out_count = 0; out_count < output_outside.GetAddrCount(); ++out_count) { - auto &addrs_mapping_list = output_outside.GetOutsideAddrs(); - std::map> virtual_args_addrs = addrs_mapping_list[out_count]; - for (const auto &virtual_args_addr : virtual_args_addrs) { - const auto &args_addrs = virtual_args_addr.second; - if (args_addrs.empty()) { // No task feed Output addr, Need copy directly. - GELOGI("[ZCPY] just copy %p to netoutput.", virtual_args_addr.first); - copy_only_addrs_.insert(virtual_args_addr.first); - } - } - } - } -} - -/// -/// @ingroup ge -/// @brief Set disabled input zero copy addr. -/// @param [in] const void *addr: address of task -/// @return None. -/// -void DavinciModel::DisableZeroCopy(const void *addr) { - if (real_virtual_addrs_.find(addr) == real_virtual_addrs_.end()) { - return; - } - - // Data link to RTS Op directly. - std::lock_guard lock(outside_addrs_mutex_); - GELOGI("[ZCPY] disable zero copy of %p.", addr); - copy_only_addrs_.insert(addr); -} - -/// -/// @ingroup ge -/// @brief Save outside address used info for ZeroCopy. -/// @param [in] const OpDescPtr &op_desc: current op desc -/// @param [in] const std::vector &outside_addrs: address of task -/// @param [in] const void *info: task args -/// @param [in] const char *args: task args -/// @param [in] size_t size: size of task args -/// @param [in] size_t offset: offset of task args -/// @return None. -/// -void DavinciModel::SetZeroCopyAddr(const OpDescPtr &op_desc, const std::vector &outside_addrs, const void *info, - void *args, size_t size, size_t offset) { - // Internal call has ensured that op_desc is not nullptr - GELOGD("[ZCPY] SetZeroCopyAddr for %s.", op_desc->GetName().c_str()); - size_t nums = outside_addrs.size(); - ZeroCopyTask zero_copy_task(op_desc->GetName(), static_cast(args), size); - for (size_t i = 0; i < nums; ++i) { - std::lock_guard lock(outside_addrs_mutex_); - - for (auto &input_outside_addrs : new_input_outside_addrs_) { - ZeroCopyOffset &input_outside = input_outside_addrs.second; - input_outside.SetOutsideAddrsValue(zero_copy_task, outside_addrs[i], args, offset + i * kAddrLen); - } - - for (auto &output_outside_addrs : new_output_outside_addrs_) { - ZeroCopyOffset &output_outside = output_outside_addrs.second; - output_outside.SetOutsideAddrsValue(zero_copy_task, outside_addrs[i], args, offset + i * kAddrLen); - } - } - - string batch_label; - if (!AttrUtils::GetStr(op_desc, ATTR_NAME_BATCH_LABEL, batch_label) || batch_label.empty()) { - zero_copy_task.SetBatchLabel(kDefaultBatchLable); - } else { - zero_copy_task.SetBatchLabel(batch_label); - } - - std::lock_guard lock(outside_addrs_mutex_); - if (zero_copy_task.IsTaskArgsSet()) { - zero_copy_task.SetOriginalArgs(info, offset + nums * kAddrLen); - zero_copy_tasks_.emplace_back(zero_copy_task); - } -} - -/// -/// @ingroup ge -/// @brief Copy Check input size and model op size. -/// @param [in] const int64_t &input_size: input size. -/// @param [in] const int64_t &op_size: model op size. -/// @param [in] is_dynamic: dynamic batch input flag. -/// @return true if success -/// -bool DavinciModel::CheckInputAndModelSize(const int64_t &input_size, const int64_t &op_size, bool is_dynamic) { - if (is_dynamic) { // dynamic is max size. - GELOGI("No need to check input and model size."); - return true; - } - - if (input_size > op_size) { - GELOGW( - "Input size [%ld] is bigger than om size need [%ld], " - "MAY cause inference result ERROR, please check model input", - input_size, op_size); - } - - if (is_dynamic_aipp_) { - GELOGI("This is dynamic aipp model, no need to judge smaller input size"); - return true; - } - // Judge overflow first - if (input_size > (INT64_MAX - kDataMemAlignSizeCompare)) { - GELOGI("The Input size [%ld] is smaller than model size [%ld] and is in the range of 64 bytes", input_size, - op_size); - return true; - } - // The input and model input size can not be exactly equal because user input is not definite. - if ((input_size + kDataMemAlignSizeCompare) < op_size) { - GELOGE(FAILED, "Input size [%ld] can not be smaller than op size [%ld] after 64-byte alignment", input_size, - op_size); - return false; - } - return true; -} - -/// -/// @ingroup ge -/// @brief Copy Inputs and Outputs addr to model for direct use. -/// @param [in] const InputData &input_data: model input data. -/// @param [in] OutputData &output_data: model output data. -/// @param [in] bool is_dynamic_input: whether is dynamic input, true: is dynamic input; false: not is dynamic input -/// @return SUCCESS handle successfully / PARAM_INVALID for failed -/// -Status DavinciModel::CopyModelData(const InputData &input_data, OutputData &output_data, bool is_dynamic) { - if (UpdateIoTaskArgs(new_input_data_info_, true, input_data.blobs, is_dynamic, input_data.batch_label) != SUCCESS) { - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[ZCPY] Update input data to model failed."); - return ACL_ERROR_GE_PARAM_INVALID; - } - - if (UpdateIoTaskArgs(new_output_data_info_, false, output_data.blobs, is_dynamic, input_data.batch_label) != - SUCCESS) { - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[ZCPY] Update output data to model failed."); - return ACL_ERROR_GE_PARAM_INVALID; - } - - for (ZeroCopyTask &task : zero_copy_tasks_) { - GE_CHK_STATUS_RET(task.DistributeParam(is_async_mode_, rt_model_stream_), "[ZCPY] Update args failed."); - } - - output_data.index = input_data.index; - output_data.model_id = model_id_; - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief Copy Data addr to model for direct use. -/// @param [in] data_info: model memory addr/size map { data_index, { tensor_size, tensor_addr } }. -/// @param [in] is_input: input data or output data -/// @param [in] blobs: user input/output data list. -/// @param [in] is_dynamic: whether is dynamic input, true: is dynamic input; false: not is dynamic input -/// @param [in] batch_label: batch label for multi-batch scenes -/// @return SUCCESS handle successfully / others handle failed -/// -Status DavinciModel::UpdateIoTaskArgs(const std::map &data_info, bool is_input, - const vector &blobs, bool is_dynamic, const string &batch_label) { - string input_or_output = "input"; - is_input ? input_or_output = "input" : input_or_output = "output"; - if (blobs.size() != data_info.size()) { - GELOGE(FAILED, "Verify %s data num failed: model requires %zu, but user actually feeds %zu", - input_or_output.c_str(), data_info.size(), blobs.size()); - return FAILED; - } - - for (const auto &data : data_info) { - if (data.first >= blobs.size()) { // check data index. - GELOGE(FAILED, "Verify %s data num failed: can not find No.%u data, because user only feeds %zu", - input_or_output.c_str(), data.first, blobs.size()); - return FAILED; - } - - const DataBuffer &buffer = blobs[data.first]; // index of data. - if (buffer.data == nullptr) { - GELOGE(FAILED, "data_buf.data is nullptr, index=%u", data.first); - return FAILED; - } - - if (!CheckInputAndModelSize(buffer.length, data.second.GetDataSize(), is_dynamic)) { - GELOGE(FAILED, "Check input size and model size failed, op[%s]", data.second.GetOpName().c_str()); - return FAILED; - } - - void *basic_addr = data.second.GetBasicAddr(); - uint64_t data_size = data.second.GetDataSize(); - if (copy_only_addrs_.count(basic_addr) > 0) { - if (is_input) { - GELOGI("[IMAS] Find addr %p need direct copy from user malloc input %p", basic_addr, buffer.data); - if (rtMemcpy(basic_addr, data_size, buffer.data, buffer.length, RT_MEMCPY_DEVICE_TO_DEVICE) != RT_ERROR_NONE) { - GELOGE(FAILED, "Non-zero copy data node copy failed"); - return FAILED; - } - } - GELOGI("No need to exeucte zero copy task because this addr %p need direct copy.", basic_addr); - continue; - } - - for (size_t count = 0; count < data.second.GetDataCount(); ++count) { - int64_t size = data.second.GetDataInfo().at(count).first; - void *addr = data.second.GetDataInfo().at(count).second; - void *buffer_addr = reinterpret_cast(reinterpret_cast(buffer.data) + - data.second.GetRelativeOffset().at(count)); - GELOGI("[ZCPY] Copy %s blobs_index %u, virtual_addr: %p, size: %ld, user_data_addr: %p, batch_label: %s", - input_or_output.c_str(), data.first, addr, size, buffer_addr, batch_label.c_str()); - // For input data, just copy for rts task. - for (ZeroCopyTask &task : zero_copy_tasks_) { - if (task.GetBatchLabel() != kDefaultBatchLable && task.GetBatchLabel() != batch_label) { - continue; - } - uintptr_t addr_val = reinterpret_cast(addr); - if (task.UpdateTaskParam(addr_val, buffer_addr) != SUCCESS) { - return FAILED; - } - } - } - } - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief get unique identification for op when load two or more models -/// @param [in] const OpDescPtr: current op. -/// @param [in] string identification: unique identification for current op. -/// @return SUCCESS handle successfully / others handle failed -/// -void DavinciModel::GetUniqueId(const OpDescPtr &op_desc, std::string &unique_identification) { - std::string session_graph_id; - GE_IF_BOOL_EXEC(AttrUtils::GetStr(*op_desc, ATTR_NAME_SESSION_GRAPH_ID, session_graph_id), - GELOGD("Get original type of session_graph_id.")); - if (session_graph_id.empty()) { - return; - } else if (session_graph_id.find("-1") != string::npos) { - unique_identification = session_graph_id + "_" + to_string(model_id_); - } else { - unique_identification = session_graph_id; - } -} - -/// -/// @ingroup ge -/// @brief For TVM Op, avoid Addr Reuse. -/// @return void* -/// -const char *DavinciModel::GetRegisterStub(const string &binfile, const string &session_graph_id) { - string binfile_key; - if (session_graph_id.empty()) { - binfile_key = binfile; - } else { - binfile_key = session_graph_id + "_" + binfile; - } - auto it = tvm_bin_kernel_.find(binfile_key); - if (it != tvm_bin_kernel_.end()) { - return it->c_str(); - } else { - it = tvm_bin_kernel_.insert(tvm_bin_kernel_.end(), binfile_key); - return it->c_str(); - } -} - -/// -/// @ingroup ge -/// @brief Constant Op Init. -/// @return Status -/// -Status DavinciModel::InitConstant(const OpDescPtr &op_desc) { - auto v_weights = ModelUtils::GetWeights(op_desc); - auto v_output_size = ModelUtils::GetOutputSize(op_desc); - auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param_, op_desc); - GE_IF_BOOL_EXEC(v_weights.empty() || v_output_size.empty() || v_output_addr.empty(), - GELOGE(PARAM_INVALID, "const op:%s not set output", op_desc->GetName().c_str()); - return PARAM_INVALID;); - - GeTensor *tensor = const_cast(v_weights[0].get()); - GE_IF_BOOL_EXEC(static_cast(v_output_size[0]) < tensor->GetData().size(), - GELOGE(PARAM_INVALID, "output size:%ld less than weight data size:%zu", v_output_size[0], - tensor->GetData().size()); - return PARAM_INVALID;); - - GE_IF_BOOL_EXEC(tensor->GetData().size() == 0, GELOGW("const op:%s has no weight data.", op_desc->GetName().c_str()); - return SUCCESS;); - - auto desc = tensor->GetTensorDesc(); - if (desc.GetDataType() == DT_STRING) { - GeShape tensor_shape = desc.GetShape(); - /// if tensor is a scaler, it's shape size if zero, according ge_tensor.cc. - /// the logic of GetShapeSize is wrong, the scaler tensor's GetShapeSize is zero - /// and that of unknown shape is zero too. - /// unknown shape will not appear here, so we can use zero judge a tensor is scaler or not - int64_t elem_num = tensor_shape.GetShapeSize(); - if (elem_num == 0 && tensor_shape.GetDims().size() == 0) { - elem_num = 1; - } - uint64_t *buff = reinterpret_cast(tensor->MutableData().data()); - GE_CHK_BOOL_RET_STATUS(ge::CheckInt64Uint32MulOverflow(elem_num, kBytes) == SUCCESS, FAILED, - "Shape size is invalid"); - uint64_t offset = static_cast(elem_num * kBytes); - - uint64_t hbm_raw_data_base_addr = - static_cast(reinterpret_cast(v_output_addr[0])) + offset; - for (int64_t i = elem_num - 1; i >= 0; --i) { - buff[i] = hbm_raw_data_base_addr + (buff[i] - buff[0]); - } - } - GELOGI("[IMAS]InitConstant memcpy graph_%u type[V] name[%s] output[%d] memaddr[%p] mem_size[%lu] datasize[%zu]", - runtime_param_.graph_id, op_desc->GetName().c_str(), 0, v_output_addr[0], v_output_size[0], - tensor->GetData().size()); - GE_CHK_RT_RET(rtMemcpy(v_output_addr[0], v_output_size[0], tensor->GetData().data(), tensor->GetData().size(), - RT_MEMCPY_HOST_TO_DEVICE)); - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief TVM Op Init. -/// @return Status -/// -Status DavinciModel::InitTbeHandle(const OpDescPtr &op_desc) { - auto kernel = ge_model_->GetTBEKernelStore().FindKernel(op_desc->GetName()); - auto tbe_kernel = (kernel != nullptr) ? kernel : op_desc->TryGetExtAttr(OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr()); - if (tbe_kernel == nullptr) { - GELOGE(INTERNAL_ERROR, "TBE: %s can't find tvm bin file!", op_desc->GetName().c_str()); - return INTERNAL_ERROR; - } - - std::string session_graph_model_id; - GetUniqueId(op_desc, session_graph_model_id); - const char *bin_file_key = GetRegisterStub(op_desc->GetName(), session_graph_model_id); // from set, always valid. - TBEHandleStore &kernel_store = TBEHandleStore::GetInstance(); - - std::lock_guard lock(tvm_bin_mutex_); - if (rtQueryFunctionRegistered(bin_file_key) != RT_ERROR_NONE) { - void *bin_handle = nullptr; - if (!kernel_store.FindTBEHandle(bin_file_key, bin_handle)) { - GELOGD("TBE: can't find the kernel_name[%s] in HandleMap", bin_file_key); - - rtDevBinary_t binary; - std::string json_string; - GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, TVM_ATTR_NAME_MAGIC, json_string), - GELOGD("Get original type of session_graph_id.")); - if (json_string == "RT_DEV_BINARY_MAGIC_ELF_AICPU") { - binary.magic = RT_DEV_BINARY_MAGIC_ELF_AICPU; - } else if (json_string == "RT_DEV_BINARY_MAGIC_ELF") { - binary.magic = RT_DEV_BINARY_MAGIC_ELF; - } else if (json_string == "RT_DEV_BINARY_MAGIC_ELF_AIVEC") { - binary.magic = RT_DEV_BINARY_MAGIC_ELF_AIVEC; - } else { - GELOGE(PARAM_INVALID, "TBE: Invalid parameter magic number! json: %s", json_string.c_str()); - return PARAM_INVALID; - } - - binary.version = 0; - binary.data = tbe_kernel->GetBinData(); - binary.length = tbe_kernel->GetBinDataSize(); - - GELOGD("TBE: binary.length: %lu", binary.length); - GE_CHK_RT_RET(rtDevBinaryRegister(&binary, &bin_handle)); - - std::string meta_data; - GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, TVM_ATTR_NAME_METADATA, meta_data), - GELOGI("Get original type of json_string")); - GELOGD("TBE: meta data: %s", meta_data.empty() ? "null" : meta_data.c_str()); - GE_IF_BOOL_EXEC(!meta_data.empty(), GE_CHK_RT_RET(rtMetadataRegister(bin_handle, meta_data.c_str()))); - - kernel_store.StoreTBEHandle(bin_file_key, bin_handle, tbe_kernel); - } else { - GELOGI("TBE: find the kernel_name[%s] in HandleMap", bin_file_key); - kernel_store.ReferTBEHandle(bin_file_key); - } - - std::string kernel_name; - GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, op_desc->GetName() + "_kernelname", kernel_name), - GELOGD("Get original type of kernel_name")); - GE_CHK_RT_RET(rtFunctionRegister(bin_handle, bin_file_key, bin_file_key, kernel_name.c_str(), 0)); - used_tbe_handle_map_[bin_file_key] = 1; // Init used num to 1. - return SUCCESS; - } - - // Kernel registed, Increase used num in store. - StoreTbeHandle(bin_file_key); - return SUCCESS; -} - -void DavinciModel::StoreTbeHandle(const std::string &handle_key) { - // Online mode FE may call rtFunctionRegister. - TBEHandleStore &kernel_store = TBEHandleStore::GetInstance(); - - auto it = used_tbe_handle_map_.find(handle_key); - if (it != used_tbe_handle_map_.end()) { - // GE registered, increase reference. - kernel_store.ReferTBEHandle(handle_key); - it->second++; - return; - } - - void *bin_handle = nullptr; - if (kernel_store.FindTBEHandle(handle_key, bin_handle)) { - // GE registered, increase reference. - used_tbe_handle_map_[handle_key] = 1; // Init used num to 1. - kernel_store.ReferTBEHandle(handle_key); - } -} - -void DavinciModel::CleanTbeHandle() { - TBEHandleStore &kernel_store = TBEHandleStore::GetInstance(); - - kernel_store.EraseTBEHandle(used_tbe_handle_map_); - used_tbe_handle_map_.clear(); - tvm_bin_kernel_.clear(); -} - -/// -/// @ingroup ge -/// @brief insert active_stream_indication_ -/// @return Status -/// -Status DavinciModel::InitStreamActive(const OpDescPtr &op_desc) { - if (op_desc->HasAttr(ATTR_NAME_SWITCH_BRANCH_NODE_LABEL)) { - std::vector active_stream_list; - GE_CHK_BOOL_EXEC(AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list), - return INTERNAL_ERROR, "StreamActiveOp get attr ACTIVE_STREAM failed."); - - for (size_t j = 0; j < active_stream_list.size(); ++j) { - active_stream_indication_.insert(active_stream_list[j]); - GELOGI("flowctrl_op_index_map node:%s, active_stream_id=%u.", op_desc->GetName().c_str(), active_stream_list[j]); - } - } - - return SUCCESS; -} - -Status DavinciModel::InitStreamSwitch(const OpDescPtr &op_desc) { - std::vector active_stream_list; - GE_LOGI_IF(!ge::AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list), - "GetInt ACTIVE_STREAM_LIST failed."); - if (active_stream_list.size() != kTrueBranchStreamNum) { - GELOGE(INTERNAL_ERROR, "Stream num of switch true branch must be %u.", kTrueBranchStreamNum); - return INTERNAL_ERROR; - } - - uint32_t true_stream_id = active_stream_list.front(); - active_stream_indication_.insert(true_stream_id); - GELOGI("flowctrl_op_index_map node:%s, true_stream_id=%u.", op_desc->GetName().c_str(), true_stream_id); - - return SUCCESS; -} - -Status DavinciModel::InitStreamSwitchN(const OpDescPtr &op_desc) { - std::vector active_stream_list; - if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list)) { - GELOGE(INTERNAL_ERROR, "StreamSwitchNOp get attr ACTIVE_STREAM failed."); - return INTERNAL_ERROR; - } - - for (size_t j = 0; j < active_stream_list.size(); ++j) { - active_stream_indication_.insert(active_stream_list[j]); - GELOGI("StreamSwitchNOp node:%s, active_stream_id=%u.", op_desc->GetName().c_str(), active_stream_list[j]); - } - - uint32_t batch_num = 0; - if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, batch_num)) { - GELOGE(FAILED, "Failed to get attr ATTR_NAME_BATCH_NUM, StreamSwitchN: %s.", op_desc->GetName().c_str()); - return FAILED; - } - - return SetDynamicBatchInfo(op_desc, batch_num); -} - -Status DavinciModel::SetDynamicBatchInfo(const OpDescPtr &op_desc, uint32_t batch_num) { - batch_info_.clear(); - combined_batch_info_.clear(); - - (void)AttrUtils::GetInt(op_desc, ATTR_DYNAMIC_TYPE, dynamic_type_); - (void)AttrUtils::GetListStr(op_desc, ATTR_USER_DESIGNEATE_SHAPE_ORDER, user_designate_shape_order_); - for (uint32_t i = 0; i < batch_num; ++i) { - std::vector batch_shape; - const std::string attr_name = ATTR_NAME_PRED_VALUE + "_" + std::to_string(i); - if (!AttrUtils::GetListInt(op_desc, attr_name, batch_shape)) { - GELOGE(FAILED, "Get attr ATTR_NAME_PRED_VALUE failed, Node: %s", op_desc->GetName().c_str()); - batch_info_.clear(); - return FAILED; - } - batch_info_.emplace_back(batch_shape); - batch_shape.clear(); - const string attr_combined_batch = ATTR_NAME_COMBINED_BATCH + "_" + std::to_string(i); - if (AttrUtils::GetListInt(op_desc, attr_combined_batch, batch_shape)) { - combined_batch_info_.emplace_back(batch_shape); - } - } - - return SUCCESS; -} - -Status DavinciModel::InitCase(const OpDescPtr &op_desc) { - uint32_t batch_num = 0; - if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, batch_num)) { - GELOGI("Not multi-batch Node: %s", op_desc->GetName().c_str()); - return SUCCESS; - } - - return SetDynamicBatchInfo(op_desc, batch_num); -} - -bool DavinciModel::IsBroadCastOpData(const ge::NodePtr &var_node) { - for (auto out_anchor : var_node->GetAllOutDataAnchors()) { - GE_RT_FALSE_CHECK_NOTNULL(out_anchor); - for (auto in_anchor : out_anchor->GetPeerInDataAnchors()) { - GE_RT_FALSE_CHECK_NOTNULL(in_anchor); - ge::NodePtr dst_node = in_anchor->GetOwnerNode(); - GE_RT_FALSE_CHECK_NOTNULL(dst_node); - if (dst_node->GetType() == HCOMBROADCAST || dst_node->GetType() == HVDCALLBACKBROADCAST) { - return true; - } - } - } - return false; -} - -/// -/// @ingroup ge -/// @brief Init model stream for NN model. -/// @param [in] stream user input model stream. -/// @return Status -/// -Status DavinciModel::InitModelStream(rtStream_t stream) { - ExecuteMode curr_mode = is_async_mode_ ? ASYNCHRONIZATION : SYNCHRONIZATION; - GE_CHK_BOOL_RET_STATUS((curr_mode == last_execute_mode_) || (last_execute_mode_ == INITIALIZATION), INTERNAL_ERROR, - "NnExecute not support mix execute."); - last_execute_mode_ = curr_mode; - - // asynchronize mode, use user input stream. - if (is_async_mode_) { - rt_model_stream_ = stream; - is_inner_model_stream_ = false; - return SUCCESS; - } - - // synchronize mode, use forbidden stream. - if (stream != nullptr) { - if ((rt_model_stream_ != nullptr) && is_inner_model_stream_) { - GE_LOGW_IF(rtStreamDestroy(rt_model_stream_) != RT_ERROR_NONE, "Destroy rt_stream failed!"); - } - - rt_model_stream_ = stream; - is_inner_model_stream_ = false; - return SUCCESS; - } - - if (rt_model_stream_ == nullptr) { - GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_model_stream_, priority_, RT_STREAM_FORBIDDEN_DEFAULT)); - is_inner_model_stream_ = true; - } - - return SUCCESS; -} - -/// -/// @ingroup ge -/// @brief ACL case, do not start new thread, return execute result. -/// @param [in] stream execute model stream. -/// @param [in] async_mode is asynchronize mode. -/// @param [in] input_data model input data. -/// @param [out] output_data model output data. -/// -Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputData &input_data, - OutputData &output_data) { - is_async_mode_ = async_mode; - GELOGD("Model Run begin, model id:%u, data index:%u, flag:%d.", model_id_, input_data.index, is_async_mode_); - GE_CHK_STATUS_RET(InitModelStream(stream), "Init model stream failed."); - is_dynamic_ = input_data.is_dynamic_batch; - - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_PRE_PROC_START)); - Status ret = CopyModelData(input_data, output_data, is_dynamic_); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Copy input data to model failed. model id: %u", - model_id_); - - GELOGD("current_data.index=%u", input_data.index); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_PRE_PROC_END)); - - if (!task_list_.empty()) { - GELOGD("rtModelExecute do"); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_INFER_START)); - rtError_t rt_ret = rtModelExecute(rt_model_handle_, rt_model_stream_, 0); - GE_CHK_RT_EXEC(rt_ret, return RT_ERROR_TO_GE_STATUS(rt_ret)); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_INFER_END)); - GELOGD("rtModelExecute end"); - } - - if (!is_async_mode_) { - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_AFTER_PROC_START)); - ret = CopyOutputData(input_data.index, output_data, RT_MEMCPY_DEVICE_TO_DEVICE); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_INTERNAL_ERROR, - "Copy Output data to user failed."); - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_AFTER_PROC_END)); - } - - // report model time data - GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), (void)SinkTimeProfile(input_data)); - GELOGD("Model run end, model id:%u", model_id_); - return SUCCESS; -} - -// Add active entry stream for special env. -Status DavinciModel::AddHeadStream() { - if (active_stream_list_.empty()) { - GELOGE(INTERNAL_ERROR, "Active stream is empty, stream list size: %zu, stream indication size: %zu.", - stream_list_.size(), active_stream_indication_.size()); - return INTERNAL_ERROR; - } - - if (active_stream_list_.size() == 1) { - GELOGI("Just one active stream, take as head stream."); - rt_head_stream_ = active_stream_list_[0]; - is_pure_head_stream_ = false; - } else { - // Create stream which rt_model_handel running on, this is S0, TS stream. - GELOGI("Multiple active stream: %zu, create head stream.", active_stream_list_.size()); - GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_head_stream_, priority_, RT_STREAM_PERSISTENT)); - GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, rt_head_stream_, RT_INVALID_FLAG)); // Not active. - is_pure_head_stream_ = true; - - for (auto s : active_stream_list_) { - std::shared_ptr active_entry = MakeShared(rt_head_stream_); - if (active_entry == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed."); - return MEMALLOC_FAILED; - } - - Status status = active_entry->Init(s); - if (status != SUCCESS) { - return status; - } - - cpu_task_list_.emplace_back(active_entry); - } - } - - // Create entry stream active head stream. AICPU stream. - GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_entry_stream_, priority_, RT_STREAM_AICPU)); - GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, rt_entry_stream_, RT_HEAD_STREAM)); - return SUCCESS; -} - -Status DavinciModel::InitEntryTask() { - if (deploy_type_ == AICPU_DEPLOY_CROSS_THREAD) { - GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed."); - return CpuActiveStream(); - } else { - return LoadWithQueue(); - } -} - -uint8_t *DavinciModel::MallocFeatureMapMem(size_t data_size) { - uint8_t *mem_base = nullptr; - const string purpose("feature map,used for op input and output."); - char ge_static_mem_env[MMPA_MAX_PATH] = { 0x00 }; - INT32 res = mmGetEnv(kEnvGeuseStaticMemory, ge_static_mem_env, MMPA_MAX_PATH); - if (res == EN_OK) { - data_size = static_cast(VarManager::Instance(session_id_)->GetGraphMemoryMaxSize()); - string memory_key = std::to_string(0) + "_f"; - mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, memory_key, data_size, GetDeviceId()); - } else { - mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, data_size, GetDeviceId()); - } - - if (mem_base != nullptr) { - GE_CHK_RT(rtMemset(mem_base, data_size, 0U, data_size)); - } - return mem_base; -} - -uint8_t *DavinciModel::MallocP2PMem(size_t p2p_data_size) { - uint8_t *p2p_mem_base = nullptr; - const string purpose("p2p memory, used for some op related to hcom"); - if (std::getenv(kEnvGeuseStaticMemory) != nullptr) { - string p2p_memory_key = std::to_string(0) + "_p"; - p2p_mem_base = - MemManager::Instance(RT_MEMORY_P2P_DDR)->MallocMemory(purpose, p2p_memory_key, p2p_data_size, GetDeviceId()); - } else { - p2p_mem_base = MemManager::Instance(RT_MEMORY_P2P_DDR)->MallocMemory(purpose, p2p_data_size, GetDeviceId()); - } - return p2p_mem_base; -} - -uint8_t *DavinciModel::MallocWeightsMem(size_t weights_size) { - uint8_t *weights_mem_base = nullptr; - const string purpose("weights memory in inference network."); - char ge_static_mem_env[MMPA_MAX_PATH] = { 0x00 }; - INT32 res = mmGetEnv(kEnvGeuseStaticMemory, ge_static_mem_env, MMPA_MAX_PATH); - if (res == EN_OK) { - string weight_memory_key = std::to_string(0) + "_w"; - weights_mem_base = - MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, weight_memory_key, weights_size, GetDeviceId()); - } else { - weights_mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, weights_size, GetDeviceId()); - } - return weights_mem_base; -} - -void DavinciModel::FreeFeatureMapMem() { - char ge_static_mem_env[MMPA_MAX_PATH] = { 0x00 }; - INT32 res = mmGetEnv(kEnvGeuseStaticMemory, ge_static_mem_env, MMPA_MAX_PATH); - if (res == EN_OK && is_inner_mem_base_) { - string weight_memory_key = std::to_string(0) + "_f"; - if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(weight_memory_key) != nullptr) { - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weight_memory_key, GetDeviceId()), - "failed to free weight memory"); - } - mem_base_ = nullptr; - } else { - GE_IF_BOOL_EXEC(mem_base_ != nullptr && is_inner_mem_base_, - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(mem_base_, GetDeviceId()), - "failed to free feature_map memory"); - mem_base_ = nullptr); - } -} - -void DavinciModel::FreeP2PMem() { - if (std::getenv(kEnvGeuseStaticMemory) != nullptr) { - std::string p2p_memory_key = std::to_string(0) + "_p"; - if (MemManager::Instance(RT_MEMORY_P2P_DDR)->GetMemoryAddr(p2p_memory_key) != nullptr) { - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_memory_key, GetDeviceId()), - "failed to free p2p memory"); - } - p2p_mem_base_ = nullptr; - } else { - GE_IF_BOOL_EXEC(p2p_mem_base_ != nullptr && is_inner_mem_base_, - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_mem_base_, GetDeviceId()), - "failed to free p2p memory"); - p2p_mem_base_ = nullptr); - } -} - -void DavinciModel::FreeWeightsMem() { - char ge_static_mem_env[MMPA_MAX_PATH] = { 0x00 }; - INT32 res = mmGetEnv(kEnvGeuseStaticMemory, ge_static_mem_env, MMPA_MAX_PATH); - if (res == EN_OK) { - string memory_key = std::to_string(0) + "_w"; - if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(memory_key) != nullptr) { - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(memory_key, GetDeviceId()), - "failed to free feature_map memory"); - } - weights_mem_base_ = nullptr; - } else { - GE_IF_BOOL_EXEC(weights_mem_base_ != nullptr && weights_mem_base_ != mem_base_ && is_inner_weight_base_, - GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weights_mem_base_, GetDeviceId()), - "failed to free weight memory"); - weights_mem_base_ = nullptr); - } -} - -Status DavinciModel::TransAllVarData(ComputeGraphPtr &graph, uint32_t graph_id) { - rtContext_t ctx = nullptr; - rtError_t rt_ret = rtCtxGetCurrent(&ctx); - if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Failed to get current context, error_code is: 0x%X.", rt_ret); - return RT_ERROR_TO_GE_STATUS(rt_ret); - } - - std::vector variable_node_list; - for (ge::NodePtr &node : graph->GetDirectNode()) { - if (node == nullptr) { - continue; - } - if (node->GetType() != VARIABLE) { - continue; - } - variable_node_list.emplace_back(node); - } - - GE_CHK_STATUS_RET_NOLOG( - TransVarDataUtils::TransAllVarData(variable_node_list, session_id_, ctx, graph_id, kThreadNum)); - return SUCCESS; -} - -void DavinciModel::SetDataDumperArgs(const ComputeGraphPtr &compute_graph) { - data_dumper_.SetModelName(name_); - data_dumper_.SetModelId(model_id_); - data_dumper_.SetOmName(om_name_); - data_dumper_.SetComputeGraph(compute_graph); - data_dumper_.SetRefInfo(saved_task_addrs_); - data_dumper_.SetL1FusionAddr(l1_fusion_addr_); - - int32_t device_id = 0; - rtError_t rt_ret = rtGetDevice(&device_id); - if (rt_ret != RT_ERROR_NONE || device_id < 0) { - GELOGE(RT_FAILED, "Call rtGetDevice failed, ret = 0x%X, device_id = %d.", rt_ret, device_id); - return; - } - data_dumper_.SetDeviceId(device_id); - - // set loop count addr - auto get_var_addr = [](const OpDescPtr &op, const RuntimeParam &runtime_param) -> void *{ - if (op != nullptr) { - auto v_output_size = ModelUtils::GetOutputSize(op); - auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param, op); - if (v_output_size.empty() || v_output_addr.empty()) { - return nullptr; - } - return v_output_addr[0]; - } - GELOGD("op is null."); - return nullptr; - }; - - data_dumper_.SetLoopAddr(get_var_addr(GetVariableOp(NODE_NAME_GLOBAL_STEP), runtime_param_), - get_var_addr(GetVariableOp(NODE_NAME_FLOWCTRL_LOOP_PER_ITER), runtime_param_), - get_var_addr(GetVariableOp(NODE_NAME_FLOWCTRL_LOOP_COND), runtime_param_)); -} - -uint32_t DavinciModel::GetFlowctrlIndex(uint32_t op_index) { - std::lock_guard lock(flowctrl_op_index_internal_map_mutex_); - return (++flowctrl_op_index_internal_map_[op_index]) - 1; -} - -void DavinciModel::PushHcclStream(rtStream_t value) { - std::lock_guard lock(all_hccl_stream_list_mutex_); - all_hccl_stream_list_.push_back(value); -} - -void DavinciModel::SaveHcclFollowStream(int64_t main_stream_id, rtStream_t stream) { - std::lock_guard lock(capacity_of_stream_mutex_); - main_follow_stream_mapping_[main_stream_id].emplace_back(stream); -} - -Status DavinciModel::GetComputeGraphInfo(vector &graph_desc_info) { - auto &all_op_desc = data_dumper_.GetAllOpDescInfo(); - for (auto &op_desc : all_op_desc) { - ComputeGraphDescInfo compute_graph_info; - if (!om_name_.empty()) { - compute_graph_info.model_name = om_name_; - } else { - compute_graph_info.model_name = name_; - } - compute_graph_info.op_name = op_desc.op_name; - compute_graph_info.op_type = op_desc.op_type; - compute_graph_info.input_format = op_desc.input_format; - compute_graph_info.input_shape = op_desc.input_shape; - compute_graph_info.input_data_type = op_desc.input_data_type; - compute_graph_info.output_format = op_desc.output_format; - compute_graph_info.output_shape = op_desc.output_shape; - compute_graph_info.output_data_type = op_desc.output_data_type; - - graph_desc_info.emplace_back(compute_graph_info); - } - return SUCCESS; -} - -void DavinciModel::SetTotalFixedAddrsSize(string tensor_name, int64_t fix_addr_size) { - if (tensor_name_to_fixed_addr_size_.find(tensor_name) == tensor_name_to_fixed_addr_size_.end()) { - tensor_name_to_fixed_addr_size_[tensor_name] = total_fixed_addr_size_; - total_fixed_addr_size_ += fix_addr_size; - } -} - -Status DavinciModel::InitOrigInputInfo(const OpDescPtr &op_desc) { - if (!op_desc->HasAttr(ATTR_NAME_AIPP_INPUTS) || !op_desc->HasAttr(ATTR_NAME_AIPP_OUTPUTS)) { - GELOGI("there is not AIPP related with index %s.", op_desc->GetName().c_str()); - OriginInputInfo orig_input_info = { FORMAT_NULL, DT_UNDEFINED, 0 }; - orig_input_info_.emplace_back(orig_input_info); - return SUCCESS; - } - - vector inputs; - if (AttrUtils::GetListStr(op_desc, ATTR_NAME_AIPP_INPUTS, inputs) && !inputs.empty()) { - std::string input = inputs[kAippOriginInputIndex]; - GELOGI("GetOrigInputInfo: origin input str: %s", input.c_str()); - std::vector infos = ge::StringUtils::Split(input, ':'); - if (infos.size() != kAippInfoNum) { - GELOGW("origin input str is invalid."); - } - OriginInputInfo orig_input_info; - orig_input_info.format = TypeUtils::SerialStringToFormat(infos[kAippInfoFormat]); - orig_input_info.data_type = TypeUtils::SerialStringToDataType(infos[kAippInfoDataType]); - orig_input_info.dim_num = std::strtol(infos[kAippInfoDimNum].c_str(), nullptr, kDecimal); - orig_input_info_.emplace_back(orig_input_info); - } else { - OriginInputInfo orig_input_info = { FORMAT_RESERVED, DT_UNDEFINED, 0 }; - orig_input_info_.emplace_back(orig_input_info); - } - - return SUCCESS; -} - -Status DavinciModel::GetOrigInputInfo(uint32_t index, OriginInputInfo &orig_input_info) { - GE_CHK_BOOL_RET_STATUS(index < orig_input_info_.size(), PARAM_INVALID, "Index %u is invalid.", index); - const OriginInputInfo &input_info = orig_input_info_[index]; - if (input_info.format == FORMAT_NULL && input_info.data_type == DT_UNDEFINED) { - GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "GetOrigInputInfo: there is not AIPP related with index %u.", index); - return ACL_ERROR_GE_AIPP_NOT_EXIST; - } - - if (input_info.format != FORMAT_RESERVED && input_info.data_type == DT_UNDEFINED) { - orig_input_info = input_info; - } - - return SUCCESS; -} - -void DavinciModel::ParseAIPPInfo(std::string in_out_info, InputOutputDims &dims_info) { - GELOGI("ParseAIPPInfo: origin str: %s", in_out_info.c_str()); - std::vector infos = ge::StringUtils::Split(in_out_info, ':'); - if (infos.size() != kAippInfoNum) { - GELOGW("origin input str is invalid."); - } - dims_info.name = infos[kAippInfoTensorName]; - dims_info.size = std::strtol(infos[kAippInfoTensorSize].c_str(), nullptr, kDecimal); - dims_info.dim_num = std::strtol(infos[kAippInfoDimNum].c_str(), nullptr, kDecimal); - - std::vector dims = ge::StringUtils::Split(infos[kAippInfoShape], ','); - for (const auto &dim : dims) { - if (dim.empty()) { - continue; - } - dims_info.dims.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal)); - } -} - -Status DavinciModel::GetAllAippInputOutputDims(uint32_t index, std::vector &input_dims, - std::vector &output_dims) { - GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index); - OpDescPtr data_op = data_op_list_[index]; - if (!data_op->HasAttr(ATTR_NAME_AIPP_INPUTS) || !data_op->HasAttr(ATTR_NAME_AIPP_OUTPUTS)) { - GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "GetAllAippInputOutputDims: there is not AIPP related with index %u.", index); - return ACL_ERROR_GE_AIPP_NOT_EXIST; - } - - vector inputs; - if (AttrUtils::GetListStr(data_op, ATTR_NAME_AIPP_INPUTS, inputs) && !inputs.empty()) { - GELOGI("GetAllAippInputOutputDims: Data: %s has %zu related aippInfo.", data_op->GetName().c_str(), inputs.size()); - for (auto it : inputs) { - InputOutputDims input_info; - ParseAIPPInfo(it, input_info); - input_dims.emplace_back(input_info); - GELOGD("GetAllAippInputOutputDims Aipp origin input dims info: %s", it.c_str()); - - ConstGeTensorDescPtr data_input_desc = data_op->GetInputDescPtr(kDataIndex); - int64_t data_input_size; - (void)TensorUtils::GetSize(*(data_op->GetInputDescPtr(kDataIndex)), data_input_size); - GELOGD( - "GetAllAippInputOutputDims related Data[%d]: tensor_name is %s, dim_num is %zu, tensor_size: %zu, format: " - "%s, data_type: %s, shape: %s .", - index, data_op->GetName().c_str(), data_input_desc->GetShape().GetDimNum(), data_input_size, - TypeUtils::FormatToSerialString(data_input_desc->GetFormat()).c_str(), - TypeUtils::DataTypeToSerialString(data_input_desc->GetDataType()).c_str(), - formats::JoinToString(data_input_desc->GetShape().GetDims()).c_str()); - } - } - - vector outputs; - if (AttrUtils::GetListStr(data_op, ATTR_NAME_AIPP_OUTPUTS, outputs) && !outputs.empty()) { - for (auto it : outputs) { - InputOutputDims output_info; - ParseAIPPInfo(it, output_info); - output_dims.emplace_back(output_info); - GELOGD("GetAllAippInputOutputDims Aipp output dims info: %s", it.c_str()); - } - } - - return SUCCESS; -} - -int64_t DavinciModel::GetFixedAddrsSize(string tensor_name) { - if (tensor_name_to_fixed_addr_size_.find(tensor_name) != tensor_name_to_fixed_addr_size_.end()) { - return tensor_name_to_fixed_addr_size_[tensor_name]; - } else { - return total_fixed_addr_size_; - } -} - -} // namespace ge diff --git a/ge/graph/load/new_model_manager/davinci_model.h.bak b/ge/graph/load/new_model_manager/davinci_model.h.bak deleted file mode 100644 index 61fd8cb5..00000000 --- a/ge/graph/load/new_model_manager/davinci_model.h.bak +++ /dev/null @@ -1,1044 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef GE_GRAPH_LOAD_NEW_MODEL_MANAGER_DAVINCI_MODEL_H_ -#define GE_GRAPH_LOAD_NEW_MODEL_MANAGER_DAVINCI_MODEL_H_ - -#include -#include -#include -#include -#include -#include - -#include "common/ge_types.h" -#include "common/helper/model_helper.h" -#include "common/helper/om_file_helper.h" -#include "common/opskernel/ge_task_info.h" -#include "common/properties_manager.h" -#include "common/types.h" -#include "framework/common/util.h" -#include "graph/debug/ge_attr_define.h" -#include "graph/load/new_model_manager/aipp_utils.h" -#include "graph/load/new_model_manager/data_dumper.h" -#include "graph/load/new_model_manager/data_inputer.h" -#include "graph/load/new_model_manager/model_utils.h" -#include "graph/load/new_model_manager/zero_copy_offset.h" -#include "graph/load/new_model_manager/zero_copy_task.h" -#include "graph/model.h" -#include "graph/node.h" -#include "graph/op_desc.h" -#include "graph/operator.h" -#include "graph/utils/attr_utils.h" -#include "graph/utils/tensor_utils.h" -#include "mmpa/mmpa_api.h" -#include "proto/task.pb.h" -#include "task_info/task_info.h" -#include "graph/common/local_context.h" - -using std::mutex; -using std::thread; -using std::multimap; - -namespace ge { -// op debug need 2048 bits buffer -const size_t kOpDebugMemorySize = 2048UL; -const size_t kDebugP2pSize = 8UL; - -typedef enum tagModelProcStage { - MODEL_LOAD_START = 1, - MODEL_LOAD_END, - MODEL_PRE_PROC_START, - MODEL_PRE_PROC_END, - MODEL_INFER_START, - MODEL_INFER_END, - MODEL_AFTER_PROC_START, - MODEL_AFTER_PROC_END, - MODEL_PROC_INVALID, -} ModelProcStage; - -struct timeInfo { - uint32_t modelId; - int64_t processBeginTime; - int64_t processEndTime; - int64_t inferenceBeginTime; - int64_t inferenceEndTime; - int64_t dumpBeginTime; - int64_t dumpEndTime; -}; - -// For super kernel -struct SuperKernelTaskInfo { - uint32_t last_block_dim; - uint32_t last_args_size; - uint32_t last_task_id; - uint32_t last_stream_id; - void *last_stream; - void *last_sm_desc; - vector kernel_list; - vector arg_list; - vector dump_flag_list; - vector op_desc_list; - vector dump_args_list; - uint32_t last_dump_flag; - int64_t last_group_key; - uintptr_t last_dump_args; - OpDescPtr last_op; -}; - -struct TaskMemInfo { - int64_t input_size{0}; - int64_t output_size{0}; - int64_t weight_size{0}; - int64_t workspace_size{0}; - int64_t total_size{0}; -}; - -struct ProfileInfo { - FusionOpInfo fusion_info; - TaskMemInfo memory_info; - uint32_t task_count{0}; -}; - -enum ExecuteMode { - INITIALIZATION, - SYNCHRONIZATION, - ASYNCHRONIZATION, -}; - -// comments -class DavinciModel { - public: - /// - /// @ingroup ge - /// @brief DavinciModel constructor - /// @author - /// - DavinciModel(int32_t priority, const shared_ptr &listener); - - /// - /// @ingroup ge - /// @brief DavinciModel desctructor, free Parse and Init resources - /// @author - /// - ~DavinciModel(); - - /// - /// @ingroup ge - /// @brief apply model to model_def_ - /// - Status Assign(const GeModelPtr &ge_model); - - /// - /// @ingroup ge - /// @brief DavinciModel initialization, including Stream, ccHandle, Event, DataInputer, etc - /// @return execute result - /// @author - /// - Status Init(void *dev_ptr = nullptr, size_t memsize = 0, void *weight_ptr = nullptr, size_t weightsize = 0); - - /// - /// @ingroup ge - /// @brief ACL case, Load task list with queue. - /// @param [in] input_que_ids: input queue ids from user, nums equal Data Op. - /// @param [in] output_que_ids: input queue ids from user, nums equal NetOutput Op. - /// @return: 0 for success / others for fail - /// - Status SetQueIds(const vector &input_queue_ids, const vector &output_queue_ids); - - /// - /// @ingroup ge - /// @brief Get DataInputer - /// @return model ID - /// - uint32_t Id() const { return model_id_; } - - /// - /// @ingroup ge - /// @brief Get DataInputer - /// @return model ID - /// - void SetId(uint32_t model_id) { model_id_ = model_id; } - - static void *Run(DavinciModel *model_pointer); - - /// - /// @ingroup ge - /// @brief NnExecute - /// @param [in] stream execute stream - /// @param [in] async_mode is asynchronize mode. - /// @param [in] input_data model input data - /// @param [out] output_data model output data - /// - Status NnExecute(rtStream_t stream, bool async_mode, const InputData &input_data, OutputData &output_data); - - /// - /// @ingroup ge - /// @brief lock mutex run flag - /// @author - /// - void LockRunFlg() { mux_run_flg_.lock(); } - - /// - /// @ingroup ge - /// @brief unlock mutex run flag - /// @author - /// - void UnlockRunFlg() { mux_run_flg_.unlock(); } - - /// - /// @ingroup ge - /// @brief get DataInputer - /// @return DataInputer pointer - /// - DataInputer *const GetDataInputer() const { return data_inputer_; } - - // get Stream number - uint32_t StreamNum() const { return runtime_param_.stream_num; } - - // get Event number - uint32_t EventNum() const { return runtime_param_.event_num; } - - // get Lable number - uint32_t LabelNum() const { return runtime_param_.label_num; } - - // get batch number - uint32_t BatchNum() const { return runtime_param_.batch_num; } - - // get session id - uint64_t SessionId() const { return runtime_param_.session_id; } - - // get model priority - int32_t Priority() const { return priority_; } - - // get total mem size - size_t TotalMemSize() const { return runtime_param_.mem_size; } - - const map &P2PMemInfos() const { return runtime_param_.memory_infos; } - - // model name - string Name() const { return name_; } - - // om_name - string OmName() const { return om_name_; } - - // version - uint32_t Version() const { return version_; } - - // get total weights mem size - size_t TotalWeightsMemSize() const { return runtime_param_.weight_size; } - - size_t TotalVarMemSize() const { return runtime_param_.var_size; } - - // get base memory address - uint8_t *MemBase() { return mem_base_; } - - // get weight base memory address - uint8_t *WeightsMemBase() { return weights_mem_base_; } - - uint8_t *VarMemBase() { return var_mem_base_; } - - // get Event list - const vector &GetEventList() const { return event_list_; } - - const vector &GetStreamList() const { return stream_list_; } - - const vector &GetLabelList() const { return label_list_; } - - Status DestroyThread(); - - // get Op - OpDescPtr GetOpByIndex(uint32_t index) const { - if (op_list_.find(index) == op_list_.end()) { - return nullptr; - } - return op_list_.at(index); - } - - OpDescPtr GetVariableOp(const string &name) { - for (auto op_desc : variable_op_list_) { - if (op_desc != nullptr && op_desc->GetName() == name) { - return op_desc; - } - } - return nullptr; - } - - // get task info for profiling - const vector &GetTaskDescInfo() const { return task_desc_info_; } - - // get updated task info list - vector GetTaskList() { return task_list_; } - - // Modified from KernelTaskInfo. - SuperKernelTaskInfo &GetSuperKernelTaskInfo() { return skt_info_; } - - rtModel_t GetRtModelHandle() const { return rt_model_handle_; } - - rtStream_t GetRtModelStream() const { return rt_model_stream_; } - - uint64_t GetRtBaseAddr() const { return runtime_param_.logic_mem_base; } - - uint64_t GetRtWeightAddr() const { return runtime_param_.logic_weight_base; } - - uint64_t GetRtVarAddr() const { return runtime_param_.logic_var_base; } - - uint32_t GetFlowctrlIndex(uint32_t op_index); - - void PushHcclStream(rtStream_t value); - - bool IsBroadCastOpData(const NodePtr &var_node); - - /// - /// @ingroup ge - /// @brief For TVM Op, avoid Addr Reuse. - /// @return void* - /// - const char *GetRegisterStub(const string &tvm_binfile_key, const string &session_graph_model_id = ""); - - /// - /// @ingroup ge - /// @brief get model input and output desc info - /// @param [out] input_shape model input size - /// @param [out] output_shape model output size - /// @return execute result - /// - Status GetInputOutputDescInfo(vector &input_desc, vector &output_desc); - - Status GetInputOutputDescInfo(vector &input_desc, vector &output_desc, - vector &inputFormats, vector &output_formats); - - /// - /// @ingroup ge - /// @brief Get dynamic batch_info - /// @param [out] batch_info - /// @param [out] dynamic_type - /// @return execute result - /// - Status GetDynamicBatchInfo(vector> &batch_info, int32_t &dynamic_type) const; - - /// - /// @ingroup ge - /// @brief Get combined dynamic dims info - /// @param [out] batch_info - /// @return None - /// - void GetCombinedDynamicDims(vector> &batch_info) const; - - void GetUserDesignateShapeOrder(vector &user_input_shape_order) const; - - void GetCurShape(vector &batch_info, int32_t &dynamic_type); - - void GetModelAttr(vector &dynamic_output_shape_info); - - /// - /// @ingroup ge - /// @brief Get AIPP input info - /// @param [in] index - /// @param [out] aipp_info - /// @return execute result - /// - Status GetAIPPInfo(uint32_t index, AippConfigInfo &aipp_info); - - Status GetAippType(uint32_t index, InputAippType &type, size_t &aipp_index); - - /// - /// @ingroup ge - /// @brief Get model_id. - /// @return model_id - /// - uint32_t GetModelId() const { return model_id_; } - - /// - /// @ingroup ge - /// @brief get unique identification for op when load two or more models - /// @param [in] op_desc : current op. - /// @param [in] string identification: unique identification for current op. - /// @return None - /// - void GetUniqueId(const OpDescPtr &op_desc, string &unique_identification); - - /// - /// @ingroup ge - /// @brief get model input and output desc for zero copy - /// @param [out] input_shape model input size - /// @param [out] output_shape model output size - /// @return execute result - /// - Status GetInputOutputDescInfoForZeroCopy(vector &input_desc, - vector &output_desc, - vector &inputFormats, vector &output_formats); - - Status ReturnResult(uint32_t data_id, const bool rslt_flg, const bool seq_end_flg, OutputData *output_data); - - Status ReturnNoOutput(uint32_t data_id); - - Status ModelRunStart(); - - /// - /// @ingroup ge - /// @brief stop run model - /// @return Status - /// - Status ModelRunStop(); - - /// - /// @ingroup ge - /// @brief model run flag - /// @return Status - /// - bool RunFlag() const { return run_flg_; } - - /// - /// @ingroup ge - /// @brief Set Session Id - /// @return void - /// - void SetSessionId(uint64_t session_id) { session_id_ = session_id; } - - /// - /// @ingroup ge - /// @brief Get Session Id - /// @return sessionID - /// - uint64_t GetSessionId() const { return session_id_; } - - /// - /// @ingroup ge - /// @brief SetDeviceId - /// @return void - /// - void SetDeviceId(uint32_t device_id) { device_id_ = device_id; } - - /// - /// @ingroup ge - /// @brief Get device Id - /// @return device id - /// - uint32_t GetDeviceId() const { return device_id_; } - - bool NeedDestroyAicpuKernel() const { return need_destroy_aicpu_kernel_; } - - Status UpdateSessionId(uint64_t session_id); - - const RuntimeParam &GetRuntimeParam() { return runtime_param_; } - - int32_t GetDataInputTid() const { return dataInputTid; } - void SetDataInputTid(int32_t data_input_tid) { dataInputTid = data_input_tid; } - - void DisableZeroCopy(const void *addr); - - bool GetOpDugReg() const { return is_op_debug_reg_; } - - /// - /// @ingroup ge - /// @brief Save outside address of Data or NetOutput used info for ZeroCopy. - /// @param [in] const OpDescPtr &op_desc: current op desc - /// @param [in] const vector &outside_addrs: address of task - /// @param [in] const void *args_offset: arguments address save the address. - /// @return None. - /// - void SetZeroCopyAddr(const OpDescPtr &op_desc, const vector &outside_addrs, const void *info, void *args, - size_t size, size_t offset); - - void SetDynamicSize(const vector &batch_num, int32_t dynamic_type); - - bool GetL1FusionEnableOption() { return is_l1_fusion_enable_; } - - void SetProfileTime(ModelProcStage stage, int64_t endTime = 0); - - int64_t GetLoadBeginTime() { return load_begin_time_; } - - int64_t GetLoadEndTime() { return load_end_time_; } - - Status ReportProfilingData(); - - void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { - data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); - } - - void SaveDumpTask(uint32_t task_id, uint32_t stream_id, const shared_ptr &op_desc, uintptr_t args) { - data_dumper_.SaveDumpTask(task_id, stream_id, op_desc, args); - } - - void SetEndGraphId(uint32_t task_id, uint32_t stream_id); - DavinciModel &operator=(const DavinciModel &model) = delete; - - DavinciModel(const DavinciModel &model) = delete; - - const map> &GetHcclFolowStream() { - return main_follow_stream_mapping_; - } - void SaveHcclFollowStream(int64_t main_stream_id, rtStream_t stream); - - void InitRuntimeParams(); - Status InitVariableMem(); - - void UpdateMemBase(uint8_t *mem_base) { - runtime_param_.mem_base = mem_base; - mem_base_ = mem_base; - } - void SetTotalArgsSize(uint32_t args_size) { total_args_size_ += args_size; } - uint32_t GetTotalArgsSize() { return total_args_size_; } - void *GetCurrentArgsAddr(uint32_t offset) { - void *cur_args = static_cast(args_) + offset; - return cur_args; - } - void SetTotalIOAddrs(const vector &io_addrs); - void SetHybridArgsSize(uint32_t args_size) { total_hybrid_args_size_ += args_size; } - uint32_t GetHybridArgsSize() { - return total_hybrid_args_size_; - } - void *GetCurrentHybridArgsAddr(uint32_t offset) { - void *cur_args = static_cast(hybrid_addrs_) + offset; - return cur_args; - } - void SetTotalFixedAddrsSize(string tensor_name, int64_t fix_addr_size); - int64_t GetFixedAddrsSize(string tensor_name); - void *GetCurrentFixedAddr(int64_t offset) const { - void *cur_addr = static_cast(fixed_addrs_) + offset; - return cur_addr; - } - - uint32_t GetFixedAddrOutputIndex(string tensor_name) { - if (tensor_name_to_peer_output_index_.find(tensor_name) != tensor_name_to_peer_output_index_.end()) { - return tensor_name_to_peer_output_index_[tensor_name]; - } - return UINT32_MAX; - } - void SetKnownNode(bool known_node) { known_node_ = known_node; } - bool IsKnownNode() { return known_node_; } - Status MallocKnownArgs(); - Status UpdateKnownNodeArgs(const vector &inputs, const vector &outputs); - Status CreateKnownZeroCopyMap(const vector &inputs, const vector &outputs); - Status UpdateKnownZeroCopyAddr(vector &total_io_addrs); - void SetKnownNodeAddrNotChanged(bool base_addr_not_changed) { base_addr_not_changed_ = base_addr_not_changed; } - - Status GetOrigInputInfo(uint32_t index, OriginInputInfo &orig_input_info); - Status GetAllAippInputOutputDims(uint32_t index, vector &input_dims, - vector &output_dims); - void SetModelDescVersion(bool is_new_model_desc) { is_new_model_desc_ = is_new_model_desc; } - // om file name - void SetOmName(string om_name) { om_name_ = om_name; } - - void SetDumpProperties(const DumpProperties &dump_properties) { data_dumper_.SetDumpProperties(dump_properties); } - const DumpProperties &GetDumpProperties() const { return data_dumper_.GetDumpProperties(); } - - bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const { - return data_dumper_.GetOpDescInfo(stream_id, task_id, op_desc_info); - } - - private: - // memory address of weights - uint8_t *weights_mem_base_; - uint8_t *var_mem_base_; - // memory address of model - uintptr_t fixed_mem_base_; // Initial of mem_base_, keep forever. - uint8_t *mem_base_; - uint8_t *p2p_mem_base_; - bool is_inner_mem_base_; - bool is_inner_weight_base_; - bool is_inner_p2p_mem_base_; - // input data manager - DataInputer *data_inputer_; - int64_t load_begin_time_; - int64_t load_end_time_; - struct timeInfo time_info_; - int32_t dataInputTid; - - void *GetRunAddress(void *addr) const; - - /// - /// @ingroup ge - /// @brief Copy Check input size and model op size. - /// @param [in] const int64_t &input_size: input size. - /// @param [in] const int64_t &op_size: model op size. - /// @param [in] is_dynamic: dynamic batch input flag. - /// @return true if success - /// - bool CheckInputAndModelSize(const int64_t &input_size, const int64_t &op_size, bool is_dynamic); - - /// - /// @ingroup ge - /// @brief Set copy only for No task feed NetOutput address. - /// @return None. - /// - void SetCopyOnlyOutput(); - - /// - /// @ingroup ge - /// @brief Copy Input/Output to model for direct use. - /// @param [in] const InputData &input_data: user input data info. - /// @param [in/out] OutputData &output_data: user output data info. - /// @param [in] bool is_dynamic: whether is dynamic input, true: is dynamic input; false: not is dynamic input - /// @return SUCCESS handle successfully / others handle failed - /// - Status CopyModelData(const InputData &input_data, OutputData &output_data, bool is_dynamic); - - /// - /// @ingroup ge - /// @brief Copy Data addr to model for direct use. - /// @param [in] data_info: model memory addr/size map { data_index, { tensor_size, tensor_addr } }. - /// @param [in] is_input: input data or output data - /// @param [in] blobs: user input/output data list. - /// @param [in] is_dynamic: whether is dynamic input, true: is dynamic input; false: not is dynamic input - /// @param [in] batch_label: batch label for multi-batch scenes - /// @return SUCCESS handle successfully / others handle failed - /// - Status UpdateIoTaskArgs(const map &data_info, bool is_input, - const vector &blobs, bool is_dynamic, const string &batch_label); - - Status CopyInputData(const InputData &input_data, bool device_data = false); - - Status CopyOutputData(uint32_t data_id, OutputData &output_data, rtMemcpyKind_t kind); - - Status SyncVarData(); - - Status InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weight_size); - Status InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size); - - void CreateInputDimsInfo(const OpDescPtr &op_desc, Format format, InputOutputDescInfo &input); - - void SetInputDimsInfo(const vector &model_input_dims, Format &format, InputOutputDescInfo &input); - - Status GetInputDescInfo(vector &input_desc, vector &input_formats); - Status GetOutputDescInfo(vector &output_desc, vector &output_formats); - - Status InitTaskInfo(domi::ModelTaskDef &modelTaskInfo); - - void UnbindHcomStream(); - - Status DistributeTask(); - - uint8_t *MallocFeatureMapMem(size_t data_size); - - uint8_t *MallocWeightsMem(size_t weights_size); - - uint8_t *MallocP2PMem(size_t p2p_data_size); - - void FreeFeatureMapMem(); - - void FreeWeightsMem(); - - void FreeP2PMem(); - - void ReleaseTask(); - - void UnbindTaskSinkStream(); - - bool IsAicpuKernelConnectSpecifiedLayer(); - - /// - /// @ingroup ge - /// @brief Reduce memory usage after task sink. - /// @return: void - /// - void Shrink(); - - /// - /// @ingroup ge - /// @brief Travel all nodes and do some init. - /// @param [in] compute_graph: ComputeGraph to load. - /// @return Status - /// - Status InitNodes(const ComputeGraphPtr &compute_graph); - - /// - /// @ingroup ge - /// @brief Data Op Initialize. - /// @param [in] ComputeGraphPtr: root graph of the model. - /// @param [in] NodePtr: Data Op. - /// @param [in/out] data_op_index: index of courrent count. - /// @param [in/out] data_by_index: Data ordered by index. - /// @return Status - /// - Status InitDataOp(const ComputeGraphPtr &graph, const NodePtr &node, uint32_t &data_op_index, - map &data_by_index); - - /// - /// @ingroup ge - /// @brief Sort Data op list by index. - /// @param [in] data_by_index: map of Data Op. - /// @param [in] output_op_list: list of NetOutput op. - /// @return Status - /// - Status OptInputOutputInfo(const map &data_by_index, const vector &output_op_list); - - /// - /// @ingroup ge - /// @brief NetOutput Op Initialize. - /// @param [in] ComputeGraphPtr: root graph of the model. - /// @param [in] NodePtr: NetOutput Op. - /// @param [in/out] vector: All NetOutput node in model. - /// @return Status - /// - Status InitNetOutput(const ComputeGraphPtr &graph, const NodePtr &node, vector &output_op_list); - - /// - /// @ingroup ge - /// @brief Constant Op Init. - /// @return Status - /// - Status InitConstant(const OpDescPtr &op_desc); - - Status InitVariable(const OpDescPtr &op_desc); - - /// @ingroup ge - /// @brief LabelSet Op Initialize. - /// @param [in] op_desc: LabelSet Op descriptor. - /// @return Status - Status InitLabelSet(const OpDescPtr &op_desc); - - Status InitStreamSwitch(const OpDescPtr &op_desc); - - Status InitStreamActive(const OpDescPtr &op_desc); - - Status InitStreamSwitchN(const OpDescPtr &op_desc); - - /// - /// @ingroup ge - /// @brief Case Op Init. - /// @return Status - /// - Status InitCase(const OpDescPtr &op_desc); - - Status SetDynamicBatchInfo(const OpDescPtr &op_desc, uint32_t batch_num); - - /// - /// @ingroup ge - /// @brief TVM Op Init. - /// @return Status - /// - Status InitTbeHandle(const OpDescPtr &op_desc); - - void StoreTbeHandle(const string &handle_key); - void CleanTbeHandle(); - - /// - /// @ingroup ge - /// @brief Make active stream list and bind to model. - /// @return: 0 for success / others for fail - /// - Status BindModelStream(); - - /// - /// @ingroup ge - /// @brief Init model stream for NN model. - /// @return Status - /// - Status InitModelStream(rtStream_t stream); - - /// - /// @ingroup ge - /// @brief ACL, Load task list with queue entrance. - /// @return: 0 for success / others for fail - /// - Status LoadWithQueue(); - - /// - /// @ingroup ge - /// @brief ACL, Bind Data Op addr to input queue. - /// @return: 0 for success / others for fail - /// - Status BindInputQueue(); - - Status CpuTaskModelZeroCopy(vector &mbuf_list, map &outside_addrs); - - /// - /// @ingroup ge - /// @brief ACL, Bind NetOutput Op addr to output queue. - /// @return: 0 for success / others for fail - /// - Status BindOutputQueue(); - Status CpuModelPrepareOutput(uintptr_t addr, uint32_t size); - - /// - /// @ingroup ge - /// @brief definiteness queue schedule, bind input queue to task. - /// @param [in] queue_id: input queue id from user. - /// @param [in] addr: Data Op output tensor address. - /// @param [in] size: Data Op output tensor size. - /// @return: 0 for success / others for fail - /// - Status CpuModelDequeue(uint32_t queue_id); - - /// - /// @ingroup ge - /// @brief definiteness queue schedule, bind output queue to task. - /// @param [in] queue_id: output queue id from user. - /// @param [in] addr: NetOutput Op input tensor address. - /// @param [in] size: NetOutput Op input tensor size. - /// @return: 0 for success / others for fail - /// - Status CpuModelEnqueue(uint32_t queue_id, uintptr_t addr, uint32_t size); - - /// - /// @ingroup ge - /// @brief definiteness queue schedule, active original model stream. - /// @return: 0 for success / others for fail - /// - Status CpuActiveStream(); - - /// - /// @ingroup ge - /// @brief definiteness queue schedule, wait for end graph. - /// @return: 0 for success / others for fail - /// - Status CpuWaitEndGraph(); - - Status BindEnqueue(); - Status CpuModelEnqueue(uint32_t queue_id, uintptr_t out_mbuf); - /// - /// @ingroup ge - /// @brief definiteness queue schedule, repeat run model. - /// @return: 0 for success / others for fail - /// - Status CpuModelRepeat(); - - Status InitEntryTask(); - Status AddHeadStream(); - - /// - /// @ingroup ge - /// @brief set ts device. - /// @return: 0 for success / others for fail - /// - Status SetTSDevice(); - - Status OpDebugRegister(); - - void OpDebugUnRegister(); - - void CheckHasHcomOp(); - - Status DoTaskSink(); - - void CreateOutput(uint32_t index, const OpDescPtr &op_desc, InputOutputDescInfo &output, uint32_t &format_result); - - Status TransAllVarData(ComputeGraphPtr &graph, uint32_t graph_id); - - // get desc info of graph for profiling - Status GetComputeGraphInfo(vector &graph_desc_info); - - void SetDataDumperArgs(const ComputeGraphPtr &compute_graph); - - Status InitModelProfile(); - Status SinkModelProfile(); - - Status SinkTimeProfile(const InputData ¤t_data); - - Status InitOutputTensorInfo(const OpDescPtr &op_desc); - Status GenOutputTensorInfo(OutputData *output_data, vector &outputs); - - Status InitInputDescInfo(const vector &data_op_list); - Status InitOutputDescInfo(const vector &output_op_list); - - void ParseAIPPInfo(string in_out_info, InputOutputDims &dims_info); - void SetLabelForDynamic(const NodePtr &node); - - void ParseDynamicOutShape(const vector &str_info, vector> &vec_info); - bool IsGetNextSinkDynamic(const OpDescPtr &op_desc); - void GetAllGearsInfo(const NodePtr &node); - Status GetGetDynamicDimsNodeInfo(const NodePtr &node); - Status GetGearAndRealOutSizeInfo(size_t input_count, const NodePtr &node); - Status GetRealOutputSizeOfMerge(size_t input_index, const NodePtr &merge_node); - Status GetGearAndRealOutShapeInfo(size_t input_count, const OpDescPtr &op_desc); - - bool is_weight_mem_has_inited_; - bool is_feature_map_mem_has_inited_; - - uint32_t model_id_; - uint32_t runtime_model_id_; - string name_; - - // used for inference data dump - string om_name_; - - uint32_t version_; - GeModelPtr ge_model_; - - bool need_destroy_aicpu_kernel_{false}; - vector out_node_name_; - - map op_list_; - - // data op_desc - vector data_op_list_; - - vector variable_op_list_; - - map new_input_data_info_; - map new_output_data_info_; - map new_input_outside_addrs_; - map new_output_outside_addrs_; - - set real_virtual_addrs_; - - // output op: save cce op actual needed memory size - vector output_memory_size_list_; - - thread thread_id_; - - shared_ptr listener_; - - bool run_flg_; - - mutex mux_run_flg_; - - int32_t priority_; - - vector stream_list_; - - mutex all_hccl_stream_list_mutex_; - vector all_hccl_stream_list_; - - // for reuse hccl_follow_stream - mutex capacity_of_stream_mutex_; - map> main_follow_stream_mapping_; - - vector event_list_; - - vector label_list_; - set label_id_indication_; - - mutex outside_addrs_mutex_; - vector zero_copy_tasks_; // Task used Data or NetOutput addr. - set copy_only_addrs_; // Address need copy to original place. - - vector task_list_; - // rt_moodel_handle - rtModel_t rt_model_handle_; - - rtStream_t rt_model_stream_; - - bool is_inner_model_stream_; - - bool is_async_mode_; // For NN execute, Async mode use rtMemcpyAsync on rt_model_stream_. - ExecuteMode last_execute_mode_; - - bool is_stream_list_bind_{false}; - bool is_pure_head_stream_{false}; - rtStream_t rt_head_stream_{nullptr}; - rtStream_t rt_entry_stream_{nullptr}; - rtAicpuDeployType_t deploy_type_{AICPU_DEPLOY_RESERVED}; - - // ACL queue schedule, save queue ids for Init. - vector cpu_task_list_; - vector input_queue_ids_; // input queue ids created by caller. - vector output_queue_ids_; // output queue ids created by caller. - vector input_mbuf_list_; // input mbuf created by dequeue task. - vector output_mbuf_list_; // output mbuf created by dequeue task. - - uint64_t session_id_; - - uint32_t device_id_; - - mutex flowctrl_op_index_internal_map_mutex_; - map flowctrl_op_index_internal_map_; - - vector active_stream_list_; - set active_stream_indication_; - - set hcom_streams_; - RuntimeParam runtime_param_; - - static mutex tvm_bin_mutex_; - set tvm_bin_kernel_; - - map used_tbe_handle_map_; - - // for profiling task and graph info - vector task_desc_info_; - - int64_t maxDumpOpNum_; - // for data dump - DataDumper data_dumper_; - uint64_t iterator_count_; - bool is_l1_fusion_enable_; - map saved_task_addrs_; - void *l1_fusion_addr_ = nullptr; - - bool known_node_ = false; - uint32_t total_args_size_ = 0; - void *args_ = nullptr; - void *args_host_ = nullptr; - void *fixed_addrs_ = nullptr; - void *hybrid_addrs_ = nullptr; - uint32_t total_hybrid_args_size_ = 0; - int64_t total_fixed_addr_size_ = 0; - map known_input_data_info_; - map known_output_data_info_; - vector total_io_addrs_; - vector orig_total_io_addrs_; - bool base_addr_not_changed_ = false; - - vector> batch_info_; - vector> combined_batch_info_; - vector user_designate_shape_order_; - int32_t dynamic_type_ = 0; - bool is_dynamic_ = false; - - vector batch_size_; - // key: input tensor name, generally rts op; - // value: the fixed addr of input anchor, same as the peer output anchor addr of the peer op - map tensor_name_to_fixed_addr_size_; - - // key: input tensor name, generally rts op; value: the peer output anchor of the peer op - map tensor_name_to_peer_output_index_; - // if model is first execute - bool is_first_execute_; - // for op debug - mutex debug_reg_mutex_; - bool is_op_debug_reg_ = false; - void *op_debug_addr_ = nullptr; - void *p2p_debug_addr_ = nullptr; - bool is_new_model_desc_{false}; - bool is_online_infer_dynamic_ = false; - bool is_getnext_sink_dynamic_ = false; - vector cur_dynamic_dims_; - void *netoutput_last_input_addr_ = nullptr; - int64_t netoutput_last_input_size_ = 0; - size_t shape_of_cur_dynamic_dims_ = 0; - // key: input_index: input is merge node; value: each gear info and each output size - map, int64_t>> merge_nodes_gear_and_real_out_size_info_; - // key: input_index: input is merge node; value: each gear info and each output shape - map, vector>> merge_nodes_gear_and_real_out_shape_info_; - vector> all_gears_info_; - - multimap op_id_map_; - vector profile_list_; - - // For super kernel. - SuperKernelTaskInfo skt_info_; - - bool is_dynamic_aipp_ = false; - vector dynamic_output_shape_info_; - - vector> input_addrs_list_; - vector> output_addrs_list_; - - vector output_buffer_size_; - vector> output_shape_info_; - - map orig_input_info_; - - vector input_descs_; - vector input_formats_; - vector output_descs_; - vector output_formats_; -}; -} // namespace ge -#endif // GE_GRAPH_LOAD_NEW_MODEL_MANAGER_DAVINCI_MODEL_H_