From 3d71d6f8f7e3ae6e563a32569ae7ecb63388a7e9 Mon Sep 17 00:00:00 2001 From: lichun Date: Thu, 17 Dec 2020 10:06:15 +0800 Subject: [PATCH 01/18] Bugfix: fix the error of root model name which causes dump failure --- ge/generator/ge_generator.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ge/generator/ge_generator.cc b/ge/generator/ge_generator.cc index e50feae2..aecd87af 100644 --- a/ge/generator/ge_generator.cc +++ b/ge/generator/ge_generator.cc @@ -524,6 +524,19 @@ Status GeGenerator::GenerateModel(const Graph &graph, const string &file_name_pr GE_CHECK_NOTNULL(ge_root_model); GE_CHECK_NOTNULL(ge_root_model->GetRootGraph()); + ModelHelper model_helper; + string model_name = ""; + Status name_ret = model_helper.GetModelNameFromMergedGraphName(ge_root_model->GetRootGraph()->GetName(), + model_name); + if (name_ret != SUCCESS) { + ErrorManager::GetInstance().ATCReportErrMessage("E10000", {"parameter"}, {"output"}); + GELOGE(FAILED, "Get model_name failed. Param --output is invalid."); + return PARAM_INVALID; + } + map name_to_ge_model = ge_root_model->GetSubgraphInstanceNameToModel(); + GeModelPtr &ge_model = name_to_ge_model[ge_root_model->GetRootGraph()->GetName()]; + GE_RETURN_WITH_LOG_IF_FALSE(ge_model != nullptr, "ge_model cannot be null"); + ge_model->SetName(model_name); ret = impl_->SaveRootModel(file_name_prefix, ge_root_model, model); if (ret != SUCCESS) { GELOGE(ret, "Save model failed"); From 425b9b7a587ea8c96d979cf780a3fbed92e99129 Mon Sep 17 00:00:00 2001 From: wuweikang Date: Thu, 17 Dec 2020 13:56:51 +0800 Subject: [PATCH 02/18] modify LoadCustAicpuSo --- ge/graph/load/new_model_manager/model_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ge/graph/load/new_model_manager/model_manager.cc b/ge/graph/load/new_model_manager/model_manager.cc index fdc4915f..c0e1ac83 100755 --- a/ge/graph/load/new_model_manager/model_manager.cc +++ b/ge/graph/load/new_model_manager/model_manager.cc @@ -1296,8 +1296,8 @@ Status ModelManager::LoadCustAicpuSo(const OpDescPtr &op_desc, const string &so_ std::lock_guard lock(cust_aicpu_mutex_); CustAICPUKernelPtr aicpu_kernel = op_desc->TryGetExtAttr(OP_EXTATTR_CUSTAICPU_KERNEL, CustAICPUKernelPtr()); if (aicpu_kernel == nullptr) { - GELOGE(INTERNAL_ERROR, "cust aicpu op %s can't find kernel!", op_desc->GetName().c_str()); - return INTERNAL_ERROR; + GELOGI("cust aicpu op %s has no corresponding kernel!", op_desc->GetName().c_str()); + return SUCCESS; } // get current context From 2c7810aa0938f37aad58e5b5829b24e667027374 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 15:22:25 +0800 Subject: [PATCH 03/18] For const input in dynamic aicpu single_op. --- ge/single_op/single_op_model.cc | 4 +- ge/single_op/single_op_model.h | 2 +- ge/single_op/stream_resource.cc | 3 +- .../task/aicpu_kernel_task_builder.cc | 2 + ge/single_op/task/aicpu_task_builder.cc | 2 + ge/single_op/task/op_task.cc | 57 +++++++++++++++++-- ge/single_op/task/op_task.h | 2 + 7 files changed, 62 insertions(+), 10 deletions(-) diff --git a/ge/single_op/single_op_model.cc b/ge/single_op/single_op_model.cc index a4a4b623..25bf6855 100755 --- a/ge/single_op/single_op_model.cc +++ b/ge/single_op/single_op_model.cc @@ -473,10 +473,10 @@ Status SingleOpModel::BuildTaskListForDynamicOp(DynamicSingleOp &single_op) { return SUCCESS; } -Status SingleOpModel::BuildDynamicOp(DynamicSingleOp &single_op) { +Status SingleOpModel::BuildDynamicOp(StreamResource &resource, DynamicSingleOp &single_op) { single_op.num_inputs_ = data_ops_.size(); single_op.num_outputs_ = netoutput_op_->GetAllInputsSize(); - ParseOpModelParams(model_helper_, model_params_); + GE_CHK_STATUS_RET_NOLOG(InitModelMem(resource)); return BuildTaskListForDynamicOp(single_op); } } // namespace ge diff --git a/ge/single_op/single_op_model.h b/ge/single_op/single_op_model.h index c3164543..6d0109fe 100755 --- a/ge/single_op/single_op_model.h +++ b/ge/single_op/single_op_model.h @@ -52,7 +52,7 @@ class SingleOpModel { Status Init(); Status BuildOp(StreamResource &resource, SingleOp &single_op); - Status BuildDynamicOp(DynamicSingleOp &single_op); + Status BuildDynamicOp(StreamResource &resource, DynamicSingleOp &single_op); private: Status InitModel(); diff --git a/ge/single_op/stream_resource.cc b/ge/single_op/stream_resource.cc index 722a1024..db6b7c47 100755 --- a/ge/single_op/stream_resource.cc +++ b/ge/single_op/stream_resource.cc @@ -155,7 +155,8 @@ Status StreamResource::BuildDynamicOperator(const string &model_name, GE_CHECK_NOTNULL(new_op); GELOGI("To build operator: %s", model_name.c_str()); - GE_CHK_STATUS_RET(model.BuildDynamicOp(*new_op), "Build op failed. op = %s, ret = %u", model_name.c_str(), ret); + GE_CHK_STATUS_RET(model.BuildDynamicOp(*this, *new_op), + "Build op failed. op = %s, ret = %u", model_name.c_str(), ret); *single_op = new_op.get(); dynamic_op_map_[model_data.model_data] = std::move(new_op); return SUCCESS; diff --git a/ge/single_op/task/aicpu_kernel_task_builder.cc b/ge/single_op/task/aicpu_kernel_task_builder.cc index f8a2bd1b..8eb4a07d 100755 --- a/ge/single_op/task/aicpu_kernel_task_builder.cc +++ b/ge/single_op/task/aicpu_kernel_task_builder.cc @@ -66,6 +66,7 @@ Status AiCpuCCTaskBuilder::BuildTask(AiCpuCCTask &task, uint64_t kernel_id, cons const std::string &kernel_name = kernel_def_.kernel_name(); task.SetSoName(so_name); task.SetkernelName(kernel_name); + GE_CHECK_NOTNULL(op_desc_); task.op_desc_ = op_desc_; const auto &context = kernel_def_.context(); @@ -96,6 +97,7 @@ Status AiCpuCCTaskBuilder::BuildTask(AiCpuCCTask &task, uint64_t kernel_id, cons GELOGE(ret, "Init ext info failed."); return ret; } + GE_CHK_STATUS_RET(task.SetInputConst(), "AicpuCCTask set input_const failed."); if (task.GetUnknownType() == DEPEND_COMPUTE) { GELOGE(FAILED, "AiCpuCCTask unknown type is depend compute, it's not supported now."); diff --git a/ge/single_op/task/aicpu_task_builder.cc b/ge/single_op/task/aicpu_task_builder.cc index 0cc5c554..e0785282 100755 --- a/ge/single_op/task/aicpu_task_builder.cc +++ b/ge/single_op/task/aicpu_task_builder.cc @@ -88,6 +88,7 @@ namespace ge { return ret; } + GE_CHECK_NOTNULL(op_desc_); task.op_desc_ = op_desc_; task.num_inputs_ = op_desc_->GetInputsSize(); task.num_outputs_ = op_desc_->GetOutputsSize(); @@ -104,6 +105,7 @@ namespace ge { fwk_op_kernel.fwkKernelBase.fwk_kernel.extInfoAddr = reinterpret_cast(task.ext_info_addr_dev_); fwk_op_kernel.fwkKernelBase.fwk_kernel.extInfoLen = kernel_ext_info_size; } + GE_CHK_STATUS_RET(task.SetInputConst(), "AicpuTask set input_const failed."); GE_CHK_STATUS_RET(task.InitForSummaryAndCopy(), "AiCpuTask init for summary and copy task failed."); fwk_op_kernel.fwkKernelBase.fwk_kernel.sessionID = ULLONG_MAX; diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index 22433ec9..4e1e9863 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -369,6 +369,25 @@ Status AiCpuBaseTask::SetExtInfoAndType(const std::string &kernel_ext_info, uint return SUCCESS; } +Status SetInputConst() { + input_is_const_.clear(); + const vector v_is_input_const = op_desc_->GetIsInputConst(); + for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { + const GeTensorDescPtr tensor_desc = op_desc_->MutableInputDesc(static_cast(i)); + if (tensor_desc == nullptr) { + GELOGD("SingleOp: %s, Index: %zu, has no input", op_desc_->GetName().c_str(), i); + continue; + } + if (i < v_is_input_const.size() && v_is_input_const[i]) { + GELOGD("SingleOp: %s, Index: %zu, input is const", op_desc_->GetName().c_str(), i); + input_is_const_.push_back(true); + continue; + } + input_is_const_.push_back(false); + } + return SUCCESS; +} + Status AiCpuBaseTask::UpdateExtInfo(const std::vector &input_desc, std::vector &output_desc, rtStream_t stream) { @@ -379,9 +398,23 @@ Status AiCpuBaseTask::UpdateExtInfo(const std::vector &input_desc, } GE_CHECK_NOTNULL(aicpu_ext_handle_); - for (size_t i = 0; i < num_inputs_; ++i) { - GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(i, input_desc[i]), - "Input[%zu] update input shape failed.", i); + + size_t non_const_index = 0; + for (size_t input_index = 0; input_index < num_inputs_; input_index++) { + if (input_index < input_is_const_.size() && input_is_const_[input_index]) { + // get input_desc from op_desc if const input, num_inputs_ is op_desc_ input_size + auto const_input_desc = op_desc_->MutableInputDesc(static_cast(input_index)); + GE_CHECK_NOTNULL(const_input_desc); + GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(i, *const_input_desc), + "Input[%zu] update input shape failed.", i); + continue; + } + GE_CHK_BOOL_RET_STATUS(non_const_index < input_desc.size(), PARAM_INVALID, + "Input_desc size is %zu, but get non_const_index is %zu", + input_desc.size(), non_const_index); + GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(input_index, input_desc[non_const_index]), + "Input[%zu] update input shape failed.", input_index); + non_const_index++; } if (unknown_type_ != DEPEND_COMPUTE) { @@ -460,11 +493,23 @@ Status AiCpuBaseTask::UpdateIoAddr(const vector &inputs, const vecto GetIoAddr(arg_base, arg_num); // input number and output number was check in ValidateParams - for (size_t i = 0; i < inputs.size(); ++i) { - auto addr = inputs[i].data; + size_t non_const_index = 0; + for (size_t input_index = 0; input_index < num_inputs_; input_index++) { + if (input_index < input_is_const_.size() && input_is_const_[input_index]) { + // const input no need update addr + GE_CHECK_NOTNULL(arg_base); + GELOGD("AICpuTask input[%zu] addr = %p", input_index, *arg_base); + *arg_base; + continue; + } + GE_CHK_BOOL_RET_STATUS(non_const_index < inputs.size(), PARAM_INVALID, + "Input size is %zu, but get non_const_index is %zu", + inputs.size(), non_const_index); + auto addr = inputs[non_const_index].data; GE_CHECK_NOTNULL(addr); - GELOGD("AICpuTask input[%zu] addr = %p", i, addr); + GELOGD("AICpuTask input[%zu] addr = %p", input_index, addr); *arg_base++ = reinterpret_cast(addr); + non_const_index++; } for (size_t i = 0; i < outputs.size(); ++i) { diff --git a/ge/single_op/task/op_task.h b/ge/single_op/task/op_task.h index e2122b6f..761697cb 100644 --- a/ge/single_op/task/op_task.h +++ b/ge/single_op/task/op_task.h @@ -113,6 +113,7 @@ class AiCpuBaseTask : public OpTask { protected: Status UpdateIoAddr(const std::vector &inputs, const std::vector &outputs); + Status SetInputConst(); Status SetExtInfoAndType(const std::string &kernel_ext_info, uint64_t kernel_id); Status UpdateExtInfo(const std::vector &input_desc, @@ -127,6 +128,7 @@ class AiCpuBaseTask : public OpTask { UnknowShapeOpType unknown_type_ = DEPEND_IN_SHAPE; std::unique_ptr aicpu_ext_handle_; void *ext_info_addr_dev_ = nullptr; + vector input_is_const_; }; class AiCpuTask : public AiCpuBaseTask { From df480ba44f7f628299093f6c1d7ebf2a0f51b70a Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 15:50:39 +0800 Subject: [PATCH 04/18] For const input in dynamic aicpu single_op. --- ge/single_op/task/op_task.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index 4e1e9863..84f2d8d3 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -369,10 +369,10 @@ Status AiCpuBaseTask::SetExtInfoAndType(const std::string &kernel_ext_info, uint return SUCCESS; } -Status SetInputConst() { +Status AiCpuTask::SetInputConst() { input_is_const_.clear(); const vector v_is_input_const = op_desc_->GetIsInputConst(); - for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { + for (size_t i = 0; i < op_desc_->GetAllInputsSize(); ++i) { const GeTensorDescPtr tensor_desc = op_desc_->MutableInputDesc(static_cast(i)); if (tensor_desc == nullptr) { GELOGD("SingleOp: %s, Index: %zu, has no input", op_desc_->GetName().c_str(), i); @@ -381,7 +381,7 @@ Status SetInputConst() { if (i < v_is_input_const.size() && v_is_input_const[i]) { GELOGD("SingleOp: %s, Index: %zu, input is const", op_desc_->GetName().c_str(), i); input_is_const_.push_back(true); - continue; + continue; } input_is_const_.push_back(false); } @@ -405,7 +405,7 @@ Status AiCpuBaseTask::UpdateExtInfo(const std::vector &input_desc, // get input_desc from op_desc if const input, num_inputs_ is op_desc_ input_size auto const_input_desc = op_desc_->MutableInputDesc(static_cast(input_index)); GE_CHECK_NOTNULL(const_input_desc); - GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(i, *const_input_desc), + GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(input_index, *const_input_desc), "Input[%zu] update input shape failed.", i); continue; } @@ -498,8 +498,8 @@ Status AiCpuBaseTask::UpdateIoAddr(const vector &inputs, const vecto if (input_index < input_is_const_.size() && input_is_const_[input_index]) { // const input no need update addr GE_CHECK_NOTNULL(arg_base); - GELOGD("AICpuTask input[%zu] addr = %p", input_index, *arg_base); - *arg_base; + GELOGD("AICpuTask input[%zu] addr = %u", input_index, *arg_base); + arg_base++; continue; } GE_CHK_BOOL_RET_STATUS(non_const_index < inputs.size(), PARAM_INVALID, From 56b65a7a41fdc6c9ebc07e0b0754394ff6b51046 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 15:59:09 +0800 Subject: [PATCH 05/18] For const input in dynamic aicpu single_op. --- ge/single_op/task/op_task.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index 84f2d8d3..d1c24609 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -369,7 +369,7 @@ Status AiCpuBaseTask::SetExtInfoAndType(const std::string &kernel_ext_info, uint return SUCCESS; } -Status AiCpuTask::SetInputConst() { +Status AiCpuBaseTask::SetInputConst() { input_is_const_.clear(); const vector v_is_input_const = op_desc_->GetIsInputConst(); for (size_t i = 0; i < op_desc_->GetAllInputsSize(); ++i) { From 75d274bbb94ed5b1b5ef75b2b6d32186c9b5f936 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 16:17:30 +0800 Subject: [PATCH 06/18] For const input in dynamic aicpu single_op. --- ge/single_op/task/op_task.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index d1c24609..4f64251c 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -406,7 +406,7 @@ Status AiCpuBaseTask::UpdateExtInfo(const std::vector &input_desc, auto const_input_desc = op_desc_->MutableInputDesc(static_cast(input_index)); GE_CHECK_NOTNULL(const_input_desc); GE_CHK_STATUS_RET(aicpu_ext_handle_->UpdateInputShapeAndType(input_index, *const_input_desc), - "Input[%zu] update input shape failed.", i); + "Input[%zu] update input shape failed.", input_index); continue; } GE_CHK_BOOL_RET_STATUS(non_const_index < input_desc.size(), PARAM_INVALID, From 97fac611e1a79d65a5c806417eca8f2f8c7f7109 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 19:32:57 +0800 Subject: [PATCH 07/18] For const input in dynamic aicpu single_op. --- ge/single_op/task/aicpu_task_builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/single_op/task/aicpu_task_builder.cc b/ge/single_op/task/aicpu_task_builder.cc index e0785282..5fd4879e 100755 --- a/ge/single_op/task/aicpu_task_builder.cc +++ b/ge/single_op/task/aicpu_task_builder.cc @@ -105,7 +105,7 @@ namespace ge { fwk_op_kernel.fwkKernelBase.fwk_kernel.extInfoAddr = reinterpret_cast(task.ext_info_addr_dev_); fwk_op_kernel.fwkKernelBase.fwk_kernel.extInfoLen = kernel_ext_info_size; } - GE_CHK_STATUS_RET(task.SetInputConst(), "AicpuTask set input_const failed."); + GE_CHK_STATUS_RET(task.SetInputConst(), "AiCpuTask set input_const failed."); GE_CHK_STATUS_RET(task.InitForSummaryAndCopy(), "AiCpuTask init for summary and copy task failed."); fwk_op_kernel.fwkKernelBase.fwk_kernel.sessionID = ULLONG_MAX; From c5fcb5d2f22842bf0cc79828e9c075d357e21604 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 17 Dec 2020 19:33:27 +0800 Subject: [PATCH 08/18] For const input in dynamic aicpu single_op. --- ge/single_op/task/aicpu_kernel_task_builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/single_op/task/aicpu_kernel_task_builder.cc b/ge/single_op/task/aicpu_kernel_task_builder.cc index 8eb4a07d..34f1ba7b 100755 --- a/ge/single_op/task/aicpu_kernel_task_builder.cc +++ b/ge/single_op/task/aicpu_kernel_task_builder.cc @@ -97,7 +97,7 @@ Status AiCpuCCTaskBuilder::BuildTask(AiCpuCCTask &task, uint64_t kernel_id, cons GELOGE(ret, "Init ext info failed."); return ret; } - GE_CHK_STATUS_RET(task.SetInputConst(), "AicpuCCTask set input_const failed."); + GE_CHK_STATUS_RET(task.SetInputConst(), "AiCpuCCTask set input_const failed."); if (task.GetUnknownType() == DEPEND_COMPUTE) { GELOGE(FAILED, "AiCpuCCTask unknown type is depend compute, it's not supported now."); From 395a4a8a31a457bc4c42d05e5a8ea4982ce99d29 Mon Sep 17 00:00:00 2001 From: wxl Date: Thu, 17 Dec 2020 21:21:14 +0800 Subject: [PATCH 09/18] bugfix --- ge/ir_build/ge_ir_build.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index f181170c..cb63c4cb 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -318,6 +318,13 @@ graphStatus Impl::CheckOptions(const std::map &options if (it != options_.end() && (CheckDisableReuseMemoryParamValid(it->second) != GRAPH_SUCCESS)) { return GRAPH_PARAM_INVALID; } + // Check Input Format + if (options_.find(kInputFormat) != options_.end()) { + if (!options_[kInputFormat].empty() && !ge::TypeUtils::IsFormatValid(options_[kInputFormat].c_str())) { + GELOGE(ge::PARAM_INVALID, "user input format [%s] is not found!", options_[kInputFormat].c_str()); + return GRAPH_PARAM_INVALID; + } + } return GRAPH_SUCCESS; } From 3c54d47b191951feacc8368c2af79357daaaa609 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 18 Dec 2020 09:54:20 +0800 Subject: [PATCH 10/18] bugfix --- ge/ir_build/atc_ir_common.cc | 11 +++++++++++ ge/ir_build/atc_ir_common.h | 1 + ge/ir_build/ge_ir_build.cc | 5 +---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ge/ir_build/atc_ir_common.cc b/ge/ir_build/atc_ir_common.cc index 2a77e386..f03b2117 100755 --- a/ge/ir_build/atc_ir_common.cc +++ b/ge/ir_build/atc_ir_common.cc @@ -63,6 +63,17 @@ vector SplitInputShape(const std::string &input_shape) { } } // namespace +Status CheckInputFormat(string &input_format) { + if (input_format.empty()) { + return ge::SUCCESS; + } + if (!ge::TypeUtils::IsFormatValid(input_format.c_str())) { + GELOGE(ge::PARAM_INVALID, "user input format [%s] is not found!", input_format.c_str()); + return ge::PARAM_INVALID; + } + return ge::SUCCESS; +} + bool CheckDynamicBatchSizeInputShapeValid(unordered_map> shape_map, std::string &dynamic_batch_size) { int32_t size = 0; diff --git a/ge/ir_build/atc_ir_common.h b/ge/ir_build/atc_ir_common.h index 47361167..e7a89b63 100644 --- a/ge/ir_build/atc_ir_common.h +++ b/ge/ir_build/atc_ir_common.h @@ -75,6 +75,7 @@ Status CheckInsertOpConfParamValid(const std::string insert_op_conf); Status CheckDisableReuseMemoryParamValid(const std::string disable_reuse_memory); Status CheckEnableSingleStreamParamValid(const std::string enable_single_stream); Status CheckImplmodeParamValid(const std::string &optypelist_for_implmode, std::string &op_select_implmode); +Status CheckInputFormat(string &input_format); void PrintOptionMap(std::map &options, std::string tips); void EraseEndSemicolon(std::string ¶m); } diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index cb63c4cb..4c39c271 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -320,10 +320,7 @@ graphStatus Impl::CheckOptions(const std::map &options } // Check Input Format if (options_.find(kInputFormat) != options_.end()) { - if (!options_[kInputFormat].empty() && !ge::TypeUtils::IsFormatValid(options_[kInputFormat].c_str())) { - GELOGE(ge::PARAM_INVALID, "user input format [%s] is not found!", options_[kInputFormat].c_str()); - return GRAPH_PARAM_INVALID; - } + return CheckInputFormat(options_[kInputFormat]); } return GRAPH_SUCCESS; } From c745346a5ed4f085dd86792ff75d7f2a67f27ce7 Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Fri, 18 Dec 2020 11:13:29 +0800 Subject: [PATCH 11/18] modify log level & rm invalid head files --- ge/ge_local_engine/engine/host_cpu_engine.cc | 5 +-- ge/graph/manager/graph_manager.cc | 18 --------- ge/graph/manager/graph_mem_allocator.cc | 3 -- .../passes/switch_to_stream_switch_pass.cc | 12 ++---- ge/graph/preprocess/graph_preprocess.cc | 40 ------------------- 5 files changed, 6 insertions(+), 72 deletions(-) diff --git a/ge/ge_local_engine/engine/host_cpu_engine.cc b/ge/ge_local_engine/engine/host_cpu_engine.cc index c836d4d6..e17f73de 100755 --- a/ge/ge_local_engine/engine/host_cpu_engine.cc +++ b/ge/ge_local_engine/engine/host_cpu_engine.cc @@ -39,7 +39,7 @@ namespace { } \ ge_tensor = MakeShared(out_desc); \ GE_CHECK_NOTNULL(ge_tensor); \ - GELOGI("node:%s allocate output %zu success, size=%lld", op_desc->GetName().c_str(), i, data_num * sizeof(TYPE));\ + GELOGD("node:%s allocate output %zu success, size=%lld", op_desc->GetName().c_str(), i, data_num * sizeof(TYPE));\ if (ge_tensor->SetData(reinterpret_cast(buf.get()), data_num * sizeof(TYPE)) != GRAPH_SUCCESS) { \ GELOGE(MEMALLOC_FAILED, "Set data for output %zu of node %s failed.", i, op_desc->GetName().c_str()); \ return MEMALLOC_FAILED; \ @@ -50,8 +50,7 @@ namespace { } else { \ ge_tensor = outputs[i]; \ GE_CHECK_NOTNULL(ge_tensor); \ - GELOGI("node:%s existed output %zu, addr=%p, size=%lld", op_desc->GetName().c_str(), i, \ - reinterpret_cast(ge_tensor->GetData().data()), ge_tensor->GetData().size()); \ + GELOGD("node:%s existed output %zu", op_desc->GetName().c_str(), i); \ } \ auto tensor = TensorAdapter::AsTensor(*ge_tensor); \ auto tensor_name = op_desc->GetOutputNameByIndex(i); \ diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 2c2495b4..4f5c190d 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -23,25 +23,15 @@ #include #include #include -#include -#include "common/ge/ge_util.h" #include "common/math/math_util.h" #include "common/thread_pool.h" -#include "common/util.h" -#include "external/graph/types.h" -#include "framework/common/debug/ge_log.h" -#include "framework/common/ge_inner_error_codes.h" -#include "framework/common/ge_types.h" #include "analyzer/analyzer.h" #include "graph/common/ge_call_wrapper.h" #include "graph/common/local_context.h" #include "graph/common/transop_util.h" -#include "graph/debug/ge_attr_define.h" #include "graph/ge_context.h" #include "graph/ge_global_options.h" -#include "graph/ge_local_context.h" -#include "graph/manager/graph_mem_allocator.h" #include "graph/manager/util/rt_context_util.h" #include "graph/partition/dynamic_shape_partition.h" #include "graph/passes/enter_pass.h" @@ -61,8 +51,6 @@ #include "graph/passes/dimension_adjust_pass.h" #include "graph/passes/dimension_compute_pass.h" #include "graph/passes/flow_ctrl_pass.h" -#include "graph/passes/hccl_group_pass.h" -#include "graph/passes/hccl_memcpy_pass.h" #include "graph/passes/identity_pass.h" #include "graph/passes/input_output_connection_identify_pass.h" #include "graph/passes/iterator_op_pass.h" @@ -76,7 +64,6 @@ #include "graph/passes/permute_pass.h" #include "graph/passes/prune_pass.h" #include "graph/passes/ref_identity_delete_op_pass.h" -#include "graph/passes/replace_with_empty_const_pass.h" #include "graph/passes/reshape_recovery_pass.h" #include "graph/passes/reshape_remove_pass.h" #include "graph/passes/same_transdata_breadth_fusion_pass.h" @@ -86,13 +73,11 @@ #include "graph/passes/switch_logic_remove_pass.h" #include "graph/passes/switch_to_stream_switch_pass.h" #include "graph/passes/transop_breadth_fusion_pass.h" -#include "graph/passes/transop_depth_fusion_pass.h" #include "graph/passes/transop_nearby_allreduce_fusion_pass.h" #include "graph/passes/transop_symmetry_elimination_pass.h" #include "graph/passes/transop_without_reshape_fusion_pass.h" #include "graph/passes/transpose_transdata_pass.h" #include "graph/passes/variable_op_pass.h" -#include "graph/passes/variable_prepare_op_pass.h" #include "graph/passes/variable_ref_delete_op_pass.h" #include "graph/passes/variable_ref_useless_control_out_delete_pass.h" #include "graph/passes/end_of_sequence_add_control_pass.h" @@ -103,9 +88,6 @@ #include "graph/passes/memcpy_addr_async_pass.h" #include "graph/build/label_allocator.h" #include "graph/utils/tensor_adapter.h" -#include "graph/utils/type_utils.h" -#include "graph/graph_util.h" -#include "graph/types.h" #include "inc/pass_manager.h" #include "init/gelib.h" #include "ir_build/atc_ir_common.h" diff --git a/ge/graph/manager/graph_mem_allocator.cc b/ge/graph/manager/graph_mem_allocator.cc index b832986b..7ee7df20 100755 --- a/ge/graph/manager/graph_mem_allocator.cc +++ b/ge/graph/manager/graph_mem_allocator.cc @@ -16,10 +16,7 @@ #include "graph/manager/graph_mem_allocator.h" -#include #include - -#include "framework/common/debug/ge_log.h" #include "graph/manager/graph_caching_allocator.h" #include "graph/manager/rdma_pool_allocator.h" diff --git a/ge/graph/passes/switch_to_stream_switch_pass.cc b/ge/graph/passes/switch_to_stream_switch_pass.cc index f75a104f..a7b922e0 100644 --- a/ge/graph/passes/switch_to_stream_switch_pass.cc +++ b/ge/graph/passes/switch_to_stream_switch_pass.cc @@ -17,13 +17,8 @@ #include "graph/passes/switch_to_stream_switch_pass.h" #include #include "common/ge/ge_util.h" -#include "framework/common/debug/ge_log.h" -#include "framework/common/debug/log.h" -#include "framework/common/ge_inner_error_codes.h" -#include "framework/common/types.h" #include "ge/ge_api_types.h" #include "graph/common/omg_util.h" -#include "graph/debug/ge_attr_define.h" #include "graph/ge_context.h" #include "graph/utils/type_utils.h" @@ -125,12 +120,13 @@ void SwitchToStreamSwitchPass::MarkCycleDependence( if (visited.count(tmp_node) > 0) { continue; } - GELOGD("MarkCycleDependence: tmp_node=%s.", tmp_node->GetName().c_str()); for (const NodePtr &out_node : tmp_node->GetOutAllNodes()) { if (switch_nodes.find(out_node) == switch_nodes.end()) { out_nodes.push(out_node); continue; } + GELOGD("MarkCycleDependence: tmp_node=%s, switch_node=%s.", + tmp_node->GetName().c_str(), out_node->GetName().c_str()); GE_IF_BOOL_EXEC(SetCyclicDependenceFlag(out_node) != SUCCESS, GELOGW("set cyclic dependence attr failed."); return ); auto map_iter = switch_cyclic_map_.find(out_node); @@ -602,7 +598,7 @@ Status SwitchToStreamSwitchPass::AddConstNode(const ComputeGraphPtr &graph, cons /// Status SwitchToStreamSwitchPass::ModifySwitchInCtlEdges(const NodePtr &switch_node, const NodePtr &cast_node, const std::set &same_cond_switch) { - GELOGI("ModifySwitchInCtlEdges: switch_node=%s, active_node=%s", switch_node->GetName().c_str(), + GELOGD("ModifySwitchInCtlEdges: switch_node=%s, active_node=%s", switch_node->GetName().c_str(), cast_node->GetName().c_str()); std::string orig_switch_name = switch_node->GetName(); OpDescPtr switch_desc = switch_node->GetOpDesc(); @@ -653,7 +649,7 @@ Status SwitchToStreamSwitchPass::ModifySwitchInCtlEdges(const NodePtr &switch_no /// Status SwitchToStreamSwitchPass::ModifySwitchOutCtlEdges(const NodePtr &switch_node, const NodePtr &stream_switch, const NodePtr &active_node) { - GELOGI("ModifySwitchOutCtlEdges: switch_node=%s, stream_switch=%s, active_node=%s", switch_node->GetName().c_str(), + GELOGD("ModifySwitchOutCtlEdges: switch_node=%s, stream_switch=%s, active_node=%s", switch_node->GetName().c_str(), stream_switch->GetName().c_str(), active_node->GetName().c_str()); auto find_res = switch_node_map_.find(switch_node); GE_IF_BOOL_EXEC(find_res == switch_node_map_.end(), { diff --git a/ge/graph/preprocess/graph_preprocess.cc b/ge/graph/preprocess/graph_preprocess.cc index 2ee5e330..da862836 100644 --- a/ge/graph/preprocess/graph_preprocess.cc +++ b/ge/graph/preprocess/graph_preprocess.cc @@ -18,7 +18,6 @@ #include #include #include -#include #include "common/formats/format_transfers/format_transfer_fractal_nz.h" #include "common/formats/format_transfers/format_transfer_fractal_z.h" #include "common/formats/format_transfers/format_transfer_nchw_nc1hwc0.h" @@ -28,13 +27,9 @@ #include "common/helper/model_helper.h" #include "common/math/math_util.h" #include "common/op/ge_op_utils.h" -#include "common/util/error_manager/error_manager.h" -#include "common/formats/utils/formats_trans_utils.h" -#include "framework/common/debug/ge_log.h" #include "graph/common/ge_call_wrapper.h" #include "graph/common/local_context.h" #include "graph/common/transop_util.h" -#include "graph/debug/ge_attr_define.h" #include "graph/ge_context.h" #include "graph/shape_refiner.h" #include "graph/manager/graph_var_manager.h" @@ -44,29 +39,21 @@ #include "graph/passes/aicpu_constant_folding_pass.h" #include "graph/passes/assert_pass.h" #include "graph/passes/assign_pass.h" -#include "graph/passes/base_pass.h" #include "graph/passes/common_subexpression_elimination_pass.h" #include "graph/passes/cond_pass.h" #include "graph/passes/cond_remove_pass.h" #include "graph/passes/constant_folding_pass.h" -#include "graph/passes/constant_fuse_same_pass.h" -#include "graph/passes/control_trigger_pass.h" #include "graph/passes/dimension_adjust_pass.h" #include "graph/passes/dimension_compute_pass.h" #include "graph/passes/dropout_pass.h" #include "graph/passes/enter_pass.h" -#include "graph/passes/flow_ctrl_pass.h" #include "graph/passes/for_pass.h" -#include "graph/passes/get_original_format_pass.h" #include "graph/passes/guarantee_const_pass.h" #include "graph/passes/hccl_group_pass.h" #include "graph/passes/hccl_memcpy_pass.h" #include "graph/passes/identity_pass.h" #include "graph/passes/infershape_pass.h" -#include "graph/passes/iterator_op_pass.h" -#include "graph/passes/merge_pass.h" #include "graph/passes/net_output_pass.h" -#include "graph/passes/next_iteration_pass.h" #include "graph/passes/no_use_reshape_remove_pass.h" #include "graph/passes/parallel_concat_start_op_pass.h" #include "graph/passes/placeholder_with_default_pass.h" @@ -81,45 +68,18 @@ #include "graph/passes/shape_operate_op_remove_pass.h" #include "graph/passes/snapshot_pass.h" #include "graph/passes/stop_gradient_pass.h" -#include "graph/passes/subgraph_pass.h" -#include "graph/passes/switch_data_edges_bypass.h" -#include "graph/passes/switch_dead_branch_elimination.h" -#include "graph/passes/switch_logic_remove_pass.h" -#include "graph/passes/merge_to_stream_merge_pass.h" -#include "graph/passes/switch_to_stream_switch_pass.h" -#include "graph/passes/attach_stream_label_pass.h" #include "graph/passes/unused_const_pass.h" -#include "graph/passes/unused_op_remove_pass.h" #include "graph/passes/var_is_initialized_op_pass.h" #include "graph/passes/variable_prepare_op_pass.h" #include "graph/preprocess/insert_op/util_insert_aipp_op.h" -#include "graph/types.h" -#include "graph/utils/tensor_utils.h" #include "graph/utils/type_utils.h" #include "inc/pass_manager.h" #include "init/gelib.h" #include "multi_batch_copy_graph.h" -#include "runtime/dev.h" -#include "graph/passes/dimension_adjust_pass.h" -#include "graph/passes/link_gen_mask_nodes_pass.h" -#include "graph/passes/permute_pass.h" -#include "graph/passes/reshape_remove_pass.h" -#include "graph/passes/same_transdata_breadth_fusion_pass.h" -#include "graph/passes/transop_breadth_fusion_pass.h" -#include "graph/passes/transop_depth_fusion_pass.h" -#include "graph/passes/transop_nearby_allreduce_fusion_pass.h" - -#include "graph/passes/cast_remove_pass.h" #include "graph/passes/data_pass.h" -#include "graph/passes/transop_without_reshape_fusion_pass.h" -#include "graph/passes/transpose_transdata_pass.h" -#include "graph/passes/variable_op_pass.h" -#include "graph/passes/variable_prepare_op_pass.h" -#include "graph/passes/variable_ref_delete_op_pass.h" #include "graph/passes/mark_agnostic_pass.h" - namespace ge { namespace { static std::map output_type_str_to_datatype = { From 0dfbbe1c1eb576d2851ac757f51d4c4c9333a9a4 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 18 Dec 2020 14:09:49 +0800 Subject: [PATCH 12/18] bugfix --- ge/ir_build/atc_ir_common.cc | 6 ++++-- ge/ir_build/atc_ir_common.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ge/ir_build/atc_ir_common.cc b/ge/ir_build/atc_ir_common.cc index f03b2117..d51bbc54 100755 --- a/ge/ir_build/atc_ir_common.cc +++ b/ge/ir_build/atc_ir_common.cc @@ -63,12 +63,14 @@ vector SplitInputShape(const std::string &input_shape) { } } // namespace -Status CheckInputFormat(string &input_format) { +Status CheckInputFormat(const string &input_format) { if (input_format.empty()) { return ge::SUCCESS; } if (!ge::TypeUtils::IsFormatValid(input_format.c_str())) { - GELOGE(ge::PARAM_INVALID, "user input format [%s] is not found!", input_format.c_str()); + ErrorManager::GetInstance().ATCReportErrMessage( + "E10001", {"parameter", "value", "reason"}, {"--input_format", FLAGS_input_format, "input format is invalid!"}); + GELOGE(ge::PARAM_INVALID, "input format [%s] is invalid!", input_format.c_str()); return ge::PARAM_INVALID; } return ge::SUCCESS; diff --git a/ge/ir_build/atc_ir_common.h b/ge/ir_build/atc_ir_common.h index e7a89b63..b26c2f2b 100644 --- a/ge/ir_build/atc_ir_common.h +++ b/ge/ir_build/atc_ir_common.h @@ -75,7 +75,7 @@ Status CheckInsertOpConfParamValid(const std::string insert_op_conf); Status CheckDisableReuseMemoryParamValid(const std::string disable_reuse_memory); Status CheckEnableSingleStreamParamValid(const std::string enable_single_stream); Status CheckImplmodeParamValid(const std::string &optypelist_for_implmode, std::string &op_select_implmode); -Status CheckInputFormat(string &input_format); +Status CheckInputFormat(const string &input_format); void PrintOptionMap(std::map &options, std::string tips); void EraseEndSemicolon(std::string ¶m); } From feb37b2ff317d80c793fe99a05ec54a684d3ec13 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 18 Dec 2020 14:34:31 +0800 Subject: [PATCH 13/18] bugfix --- inc/external/ge/ge_api_types.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/inc/external/ge/ge_api_types.h b/inc/external/ge/ge_api_types.h index cce17f93..86b8aa2b 100644 --- a/inc/external/ge/ge_api_types.h +++ b/inc/external/ge/ge_api_types.h @@ -293,6 +293,8 @@ const std::string MDL_BANK_PATH_FLAG = "ge.mdl_bank_path"; // Configure op bank path const std::string OP_BANK_PATH_FLAG = "ge.op_bank_path"; +const std::string OP_BANK_UPDATE_FLAG = "ge.op_bank_update"; + // Graph run mode enum GraphRunMode { PREDICTION = 0, TRAIN }; @@ -366,6 +368,7 @@ static const char *const OP_COMPILER_CACHE_DIR = ge::OP_COMPILER_CACHE_DIR; static const char *const OP_COMPILER_CACHE_MODE = ge::OP_COMPILER_CACHE_MODE; static const char *const MDL_BANK_PATH = ge::MDL_BANK_PATH_FLAG.c_str(); static const char *const OP_BANK_PATH = ge::OP_BANK_PATH_FLAG.c_str(); +static const char *const OP_BANK_UPDATE = ge::OP_BANK_UPDATE_FLAG.c_str(); static const char *const OP_DEBUG_LEVEL = ge::OP_DEBUG_LEVEL.c_str(); // for interface: aclgrphBuildModel @@ -389,7 +392,8 @@ const std::set ir_builder_suppported_options = {INPUT_FORMAT, OP_COMPILER_CACHE_DIR, OP_COMPILER_CACHE_MODE, MDL_BANK_PATH, - OP_BANK_PATH}; + OP_BANK_PATH, + OP_BANK_UPDATE}; // for interface: aclgrphParse const std::set ir_parser_suppported_options = {INPUT_FORMAT, From 19091f166d720d6bdbfd708b8d615a8842658b96 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 18 Dec 2020 14:55:09 +0800 Subject: [PATCH 14/18] bugfix --- inc/external/ge/ge_api_types.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/inc/external/ge/ge_api_types.h b/inc/external/ge/ge_api_types.h index 86b8aa2b..cce17f93 100644 --- a/inc/external/ge/ge_api_types.h +++ b/inc/external/ge/ge_api_types.h @@ -293,8 +293,6 @@ const std::string MDL_BANK_PATH_FLAG = "ge.mdl_bank_path"; // Configure op bank path const std::string OP_BANK_PATH_FLAG = "ge.op_bank_path"; -const std::string OP_BANK_UPDATE_FLAG = "ge.op_bank_update"; - // Graph run mode enum GraphRunMode { PREDICTION = 0, TRAIN }; @@ -368,7 +366,6 @@ static const char *const OP_COMPILER_CACHE_DIR = ge::OP_COMPILER_CACHE_DIR; static const char *const OP_COMPILER_CACHE_MODE = ge::OP_COMPILER_CACHE_MODE; static const char *const MDL_BANK_PATH = ge::MDL_BANK_PATH_FLAG.c_str(); static const char *const OP_BANK_PATH = ge::OP_BANK_PATH_FLAG.c_str(); -static const char *const OP_BANK_UPDATE = ge::OP_BANK_UPDATE_FLAG.c_str(); static const char *const OP_DEBUG_LEVEL = ge::OP_DEBUG_LEVEL.c_str(); // for interface: aclgrphBuildModel @@ -392,8 +389,7 @@ const std::set ir_builder_suppported_options = {INPUT_FORMAT, OP_COMPILER_CACHE_DIR, OP_COMPILER_CACHE_MODE, MDL_BANK_PATH, - OP_BANK_PATH, - OP_BANK_UPDATE}; + OP_BANK_PATH}; // for interface: aclgrphParse const std::set ir_parser_suppported_options = {INPUT_FORMAT, From 8ddb16e8f758f29ffcd5470780ad86583557d391 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 18 Dec 2020 15:07:06 +0800 Subject: [PATCH 15/18] bugfix --- ge/ir_build/atc_ir_common.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/ir_build/atc_ir_common.cc b/ge/ir_build/atc_ir_common.cc index d51bbc54..77d749de 100755 --- a/ge/ir_build/atc_ir_common.cc +++ b/ge/ir_build/atc_ir_common.cc @@ -69,7 +69,7 @@ Status CheckInputFormat(const string &input_format) { } if (!ge::TypeUtils::IsFormatValid(input_format.c_str())) { ErrorManager::GetInstance().ATCReportErrMessage( - "E10001", {"parameter", "value", "reason"}, {"--input_format", FLAGS_input_format, "input format is invalid!"}); + "E10001", {"parameter", "value", "reason"}, {"--input_format", input_format, "input format is invalid!"}); GELOGE(ge::PARAM_INVALID, "input format [%s] is invalid!", input_format.c_str()); return ge::PARAM_INVALID; } From 8d6e9c5d148852064193174b7b525693bdb2f67d Mon Sep 17 00:00:00 2001 From: taoxudonghaha Date: Fri, 18 Dec 2020 15:08:18 +0800 Subject: [PATCH 16/18] solve static depend and add llt interception --- CMakeLists.txt | 7 +- build.sh | 14 ++-- cmake/FindModule.cmake | 15 +--- metadef | 2 +- parser | 2 +- tests/depends/runtime/src/runtime_stub.cc | 5 ++ tests/ut/common/graph/CMakeLists.txt | 79 +++++++++++-------- tests/ut/ge/CMakeLists.txt | 32 ++++++-- .../ge/graph/build/mem_assigner_unittest.cc | 2 + .../broadcast_args_kernel_unittest.cc | 1 - ...broadcast_gradient_args_kernel_unittest.cc | 1 - .../folding_kernel/empty_kernel_unittest.cc | 1 - .../graph/passes/variable_op_pass_unittest.cc | 1 + .../graph_ir/ge_operator_factory_unittest.cc | 5 +- .../ge/single_op/single_op_model_unittest.cc | 2 +- .../ge/single_op/stream_resource_unittest.cc | 2 + 16 files changed, 98 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a4d9c3a6..776a3232 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,8 @@ if (ENABLE_OPEN_SRC) find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH}) find_module(msprofiler_fwk libmsprofiler_fwk.a ${GE_LIB_PATH}) #find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH}) + elseif(ENABLE_GE_COV OR ENABLE_GE_UT) + add_subdirectory(tests) else() find_module(slog libslog.so ${ASCEND_ATC_DIR}) find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) @@ -132,11 +134,6 @@ if (ENABLE_OPEN_SRC) else() message(STATUS "PLATFORM param is invalid, should be train or inference, you choose nothing!") endif() - - if (ENABLE_GE_COV OR ENABLE_GE_UT) - add_subdirectory(tests) - endif() - endif() set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/metadef) diff --git a/build.sh b/build.sh index 3c9a537e..71a733d1 100644 --- a/build.sh +++ b/build.sh @@ -224,12 +224,14 @@ if [[ "X$ENABLE_GE_UT" = "Xon" || "X$ENABLE_GE_COV" = "Xon" ]]; then # fi # if [[ "X$ENABLE_GE_COV" = "Xon" ]]; then -# echo "Generating coverage statistics, please wait..." -# cd ${BASEPATH} -# rm -rf ${BASEPATH}/cov -# mkdir ${BASEPATH}/cov -# gcovr -r ./ --exclude 'third_party' --exclude 'build' --exclude 'tests' --exclude 'prebuild' --exclude 'inc' --print-summary --html --html-details -d -o cov/index.html -# fi + echo "Generating coverage statistics, please wait..." + cd ${BASEPATH} + rm -rf ${BASEPATH}/cov + mkdir ${BASEPATH}/cov + lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o cov/tmp.info + lcov --remove cov/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' -o cov/coverage.info + cd ${BASEPATH}/cov + genhtml coverage.info fi # generate output package in tar form, including ut/st libraries/executables diff --git a/cmake/FindModule.cmake b/cmake/FindModule.cmake index eab39b10..d0c85da2 100644 --- a/cmake/FindModule.cmake +++ b/cmake/FindModule.cmake @@ -3,16 +3,11 @@ name - find the library name path - find the library path #]] -function(find_module module name) +function(find_module module name path) if (TARGET ${module}) return() endif() - - set(options) - set(oneValueArgs) - set(multiValueArgs) - cmake_parse_arguments(MODULE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(path ${MODULE_UNPARSED_ARGUMENTS}) + add_library(${module} INTERFACE) find_library(${module}_LIBRARY_DIR NAMES ${name} NAMES_PER_DIR PATHS ${path} PATH_SUFFIXES lib ) @@ -21,9 +16,5 @@ function(find_module module name) if ("${${module}_LIBRARY_DIR}" STREQUAL "${module}_LIBRARY_DIR-NOTFOUND") message(FATAL_ERROR "${name} not found in ${path}") endif() - - add_library(${module} SHARED IMPORTED) - set_target_properties(${module} PROPERTIES - IMPORTED_LOCATION ${${module}_LIBRARY_DIR} - ) + target_link_libraries(${module} INTERFACE ${${module}_LIBRARY_DIR}) endfunction() diff --git a/metadef b/metadef index 129b50b4..5546f5f4 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 129b50b41f79d0dfeb9fe8987b1c19c9ac51eb8b +Subproject commit 5546f5f4701130f2dd11a6d69817dc37d52c497e diff --git a/parser b/parser index e9f7d019..dc250b93 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit e9f7d0197aba57eb5247cb1e029c10e393631c89 +Subproject commit dc250b93ec6b1f08938cbe4a20091fcf68635d7d diff --git a/tests/depends/runtime/src/runtime_stub.cc b/tests/depends/runtime/src/runtime_stub.cc index 75eefdd1..9b45e7e2 100644 --- a/tests/depends/runtime/src/runtime_stub.cc +++ b/tests/depends/runtime/src/runtime_stub.cc @@ -384,3 +384,8 @@ rtError_t rtModelExit(rtModel_t model, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtGetTaskIdAndStreamID(uint32_t *taskId, uint32_t *streamId) +{ + return RT_ERROR_NONE; +} diff --git a/tests/ut/common/graph/CMakeLists.txt b/tests/ut/common/graph/CMakeLists.txt index 2f8776e3..99b21182 100644 --- a/tests/ut/common/graph/CMakeLists.txt +++ b/tests/ut/common/graph/CMakeLists.txt @@ -61,58 +61,67 @@ set(UT_FILES ) set(SRC_FILES - #"${GE_CODE_DIR}/metadef/graph/option/ge_local_context.cc" - #"${GE_CODE_DIR}/metadef/graph/option/ge_context.cc" - #"${GE_CODE_DIR}/metadef/graph/anchor.cc" - #"${GE_CODE_DIR}/metadef/graph/ge_attr_value.cc" - #"${GE_CODE_DIR}/metadef/graph/attr_value.cc" - #"${GE_CODE_DIR}/metadef/graph/buffer.cc" - #"${GE_CODE_DIR}/metadef/graph/compute_graph.cc" - #"${GE_CODE_DIR}/metadef/graph/ge_attr_define.cc" - #"${GE_CODE_DIR}/metadef/graph/graph.cc" - #"${GE_CODE_DIR}/metadef/graph/gnode.cc" - #"${GE_CODE_DIR}/metadef/graph/ascend_string.cc" - #"${GE_CODE_DIR}/metadef/graph/model.cc" - #"${GE_CODE_DIR}/metadef/graph/model_serialize.cc" - #"${GE_CODE_DIR}/metadef/graph/node.cc" - #"${GE_CODE_DIR}/metadef/graph/op_desc.cc" - #"${GE_CODE_DIR}/metadef/graph/operator.cc" - #"${GE_CODE_DIR}/metadef/graph/operator_reg.cc" - #"${GE_CODE_DIR}/metadef/graph/operator_factory.cc" - #"${GE_CODE_DIR}/metadef/graph/operator_factory_impl.cc" - #"${GE_CODE_DIR}/metadef/graph/range_vistor.cc" - #"${GE_CODE_DIR}/metadef/graph/tensor.cc" - #"${GE_CODE_DIR}/metadef/graph/ge_tensor.cc" - #"${GE_CODE_DIR}/metadef/graph/shape_refiner.cc" - #"${GE_CODE_DIR}/metadef/graph/format_refiner.cc" - #"${GE_CODE_DIR}/metadef/graph/inference_context.cc" - #"${GE_CODE_DIR}/metadef/graph/detail/attributes_holder.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/anchor_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/graph_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/node_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/op_desc_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/type_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/ge_ir_utils.cc" - #"${GE_CODE_DIR}/metadef/graph/utils/tensor_utils.cc" + "${GE_CODE_DIR}/metadef/graph/option/ge_local_context.cc" + "${GE_CODE_DIR}/metadef/graph/option/ge_context.cc" + "${GE_CODE_DIR}/metadef/graph/anchor.cc" + "${GE_CODE_DIR}/metadef/graph/ge_attr_value.cc" + "${GE_CODE_DIR}/metadef/graph/attr_value.cc" + "${GE_CODE_DIR}/metadef/graph/buffer.cc" + "${GE_CODE_DIR}/metadef/graph/compute_graph.cc" + "${GE_CODE_DIR}/metadef/graph/ge_attr_define.cc" + "${GE_CODE_DIR}/metadef/graph/graph.cc" + "${GE_CODE_DIR}/metadef/graph/gnode.cc" + "${GE_CODE_DIR}/metadef/graph/ascend_string.cc" + "${GE_CODE_DIR}/metadef/graph/model.cc" + "${GE_CODE_DIR}/metadef/graph/model_serialize.cc" + "${GE_CODE_DIR}/metadef/graph/node.cc" + "${GE_CODE_DIR}/metadef/graph/op_desc.cc" + "${GE_CODE_DIR}/metadef/graph/operator.cc" + "${GE_CODE_DIR}/metadef/graph/operator_factory.cc" + "${GE_CODE_DIR}/metadef/graph/operator_factory_impl.cc" + "${GE_CODE_DIR}/metadef/graph/tensor.cc" + "${GE_CODE_DIR}/metadef/graph/ge_tensor.cc" + "${GE_CODE_DIR}/metadef/graph/shape_refiner.cc" + "${GE_CODE_DIR}/metadef/graph/format_refiner.cc" + "${GE_CODE_DIR}/metadef/graph/inference_context.cc" + "${GE_CODE_DIR}/metadef/graph/detail/attributes_holder.cc" + "${GE_CODE_DIR}/metadef/graph/utils/anchor_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/graph_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/node_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/op_desc_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/type_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/ge_ir_utils.cc" + "${GE_CODE_DIR}/metadef/graph/utils/tensor_utils.cc" "${GE_CODE_DIR}/metadef/ops/op_imp.cpp" - #"${GE_CODE_DIR}/metadef/graph/opsproto/opsproto_manager.cc" + "${GE_CODE_DIR}/metadef/graph/opsproto/opsproto_manager.cc" + "${GE_CODE_DIR}/metadef/graph/utils/transformer_utils.cc" + "${GE_CODE_DIR}/metadef/graph/runtime_inference_context.cc" + "${GE_CODE_DIR}/metadef/graph/ref_relation.cc" + "${GE_CODE_DIR}/metadef/third_party/transformer/src/transfer_shape_according_to_format.cpp" + "${GE_CODE_DIR}/metadef/third_party/transformer/src/axis_util.cpp" ) #add_executable(ut_libgraph ${UT_FILES} ${SRC_FILES} ${PROTO_SRCS} ${PROTO_HDRS}) add_executable(ut_libgraph ${UT_FILES} ${SRC_FILES} ${PROTO_SRCS} ${PROTO_HDRS}) +target_compile_options(ut_libgraph PRIVATE + -g --coverage -fprofile-arcs -ftest-coverage +) + target_compile_definitions(ut_libgraph PRIVATE google=ascend_private ) target_link_libraries(ut_libgraph $ - graph gtest gtest_main slog_stub ascend_protobuf c_sec + error_manager_stub + mmpa_stub -lrt -ldl + -lgcov ) diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index e305d281..fb065529 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -482,7 +482,7 @@ set(GRAPH_PASS_COMMON_SRC_FILES "${GE_CODE_DIR}/ge/graph/passes/compile_nodes_pass.cc" "${GE_CODE_DIR}/ge/graph/common/transop_util.cc" "${GE_CODE_DIR}/ge/graph/passes/flow_ctrl_pass.cc" - "${GE_CODE_DIR}/ge/graph/optimize/optimizer/allreduce_fusion_pass.cc" + #"${GE_CODE_DIR}/ge/graph/optimize/optimizer/allreduce_fusion_pass.cc" "${GE_CODE_DIR}/ge/graph/passes/folding_pass.cc" "${GE_CODE_DIR}/ge/graph/passes/variable_op_pass.cc" "${GE_CODE_DIR}/ge/graph/passes/transpose_transdata_pass.cc" @@ -670,13 +670,13 @@ set(MULTI_PARTS_TEST_FILES ) set(SINGLE_OP_TEST_FILES - "single_op/single_op_model_unittest.cc" + #"single_op/single_op_model_unittest.cc" "single_op/single_op_manager_unittest.cc" "single_op/stream_resource_unittest.cc" ) set(PROFILING_MNG_TEST_FILES - "profiling/ge_profiling_manager_unittest.cc" + #"profiling/ge_profiling_manager_unittest.cc" ) set(OTHERS_TEST_FILES @@ -843,13 +843,17 @@ add_executable(ut_libge_multiparts_utest ${MULTI_PARTS_TEST_FILES} ) +target_compile_options(ut_libge_multiparts_utest PRIVATE + -g --coverage -fprofile-arcs -ftest-coverage +) + target_compile_definitions(ut_libge_multiparts_utest PRIVATE google=ascend_private ) target_link_libraries(ut_libge_multiparts_utest $ - ge_build_common ge_load_common ge_execute_common ge_optimize_common ge_partition_common ge_prepare_common ge_single_op ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl + ge_build_common ge_load_common ge_execute_common ge_optimize_common ge_partition_common ge_prepare_common ge_single_op ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl -lgcov ) # libge_others_utest @@ -860,9 +864,14 @@ add_executable(ut_libge_others_utest ${EXECUTE_TEST_FILES} ${OTHERS_TEST_FILES} ) + +target_compile_options(ut_libge_others_utest PRIVATE + -g --coverage -fprofile-arcs -ftest-coverage +) + target_link_libraries(ut_libge_others_utest $ - ge_load_common ge_execute_common ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl + ge_load_common ge_execute_common ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl -lgcov ) # libge_kernel_utest @@ -872,9 +881,14 @@ add_executable(ut_libge_kernel_utest ${KERNEL_TEST_FILES} ${KERNEL_SRC_FILES} ) + +target_compile_options(ut_libge_kernel_utest PRIVATE + -g --coverage -fprofile-arcs -ftest-coverage +) + target_link_libraries(ut_libge_kernel_utest $ - ge_load_common ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl + ge_load_common ge_ut_common gtest gtest_main ascend_protobuf ${COMMON_SHARED_LIBRARIES} json -lrt -ldl -lgcov ) # libge_distinct_load_utest @@ -886,6 +900,10 @@ add_executable(ut_libge_distinct_load_utest ${PROFILING_MNG_TEST_FILES} ) +target_compile_options(ut_libge_distinct_load_utest PRIVATE + -g --coverage -fprofile-arcs -ftest-coverage +) + target_compile_definitions(ut_libge_distinct_load_utest PRIVATE google=ascend_private ) @@ -896,5 +914,5 @@ target_link_libraries(ut_libge_distinct_load_utest ge_execute_common ge_ut_common_format ge_load_common ge_single_op ge_prepare_common ge_optimize_common ge_build_common ge_partition_common ge_ut_common - gtest gtest_main ascend_protobuf json c_sec -lrt -ldl -lpthread + gtest gtest_main ascend_protobuf json c_sec -lrt -ldl -lpthread -lgcov ) diff --git a/tests/ut/ge/graph/build/mem_assigner_unittest.cc b/tests/ut/ge/graph/build/mem_assigner_unittest.cc index 1035d00d..f53a0732 100644 --- a/tests/ut/ge/graph/build/mem_assigner_unittest.cc +++ b/tests/ut/ge/graph/build/mem_assigner_unittest.cc @@ -147,6 +147,7 @@ class UtestMemoryAssignerTest : public testing::Test { void TearDown() { GetContext().out_nodes_map.clear(); } }; +/* TEST_F(UtestMemoryAssignerTest, MemoryBlock_Resize_RealSizeList_is_empty) { ge::ComputeGraphPtr graph = make_shared(""); ge::OpDescPtr op_def_a = createOpWithWsSize("A", 6000); @@ -160,6 +161,7 @@ TEST_F(UtestMemoryAssignerTest, MemoryBlock_Resize_RealSizeList_is_empty) { delete memory_block; } +*/ namespace ge { diff --git a/tests/ut/ge/graph/passes/folding_kernel/broadcast_args_kernel_unittest.cc b/tests/ut/ge/graph/passes/folding_kernel/broadcast_args_kernel_unittest.cc index 7990a117..3a9f758b 100644 --- a/tests/ut/ge/graph/passes/folding_kernel/broadcast_args_kernel_unittest.cc +++ b/tests/ut/ge/graph/passes/folding_kernel/broadcast_args_kernel_unittest.cc @@ -52,7 +52,6 @@ using namespace testing; using namespace ge; -using namespace cce; using namespace ge::test; #define TEST_OPERATOR(op_, input_shapes, output_shapes) \ diff --git a/tests/ut/ge/graph/passes/folding_kernel/broadcast_gradient_args_kernel_unittest.cc b/tests/ut/ge/graph/passes/folding_kernel/broadcast_gradient_args_kernel_unittest.cc index e8d15291..bb021589 100644 --- a/tests/ut/ge/graph/passes/folding_kernel/broadcast_gradient_args_kernel_unittest.cc +++ b/tests/ut/ge/graph/passes/folding_kernel/broadcast_gradient_args_kernel_unittest.cc @@ -52,7 +52,6 @@ using namespace testing; using namespace ge; -using namespace cce; class UtestBroadcastGradientArgsKernel : public testing::Test { protected: diff --git a/tests/ut/ge/graph/passes/folding_kernel/empty_kernel_unittest.cc b/tests/ut/ge/graph/passes/folding_kernel/empty_kernel_unittest.cc index 7705f986..65faad20 100644 --- a/tests/ut/ge/graph/passes/folding_kernel/empty_kernel_unittest.cc +++ b/tests/ut/ge/graph/passes/folding_kernel/empty_kernel_unittest.cc @@ -53,7 +53,6 @@ using namespace testing; using namespace ge; -using namespace cce; using namespace ge::test; class UtestEmptyKernel : public testing::Test { diff --git a/tests/ut/ge/graph/passes/variable_op_pass_unittest.cc b/tests/ut/ge/graph/passes/variable_op_pass_unittest.cc index 8058279f..b51908e2 100644 --- a/tests/ut/ge/graph/passes/variable_op_pass_unittest.cc +++ b/tests/ut/ge/graph/passes/variable_op_pass_unittest.cc @@ -38,6 +38,7 @@ #include "graph/manager/graph_mem_allocator.h" #include "graph/manager/graph_var_manager.h" #include "graph_builder_utils.h" +#include "cce/dnn.h" #include "cce/dnn_struct_base.hpp" #include "common/formats/format_transfers/format_transfer_nchw_nc1hwc0.h" #include "common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.h" diff --git a/tests/ut/ge/graph_ir/ge_operator_factory_unittest.cc b/tests/ut/ge/graph_ir/ge_operator_factory_unittest.cc index 64f76515..97be491a 100644 --- a/tests/ut/ge/graph_ir/ge_operator_factory_unittest.cc +++ b/tests/ut/ge/graph_ir/ge_operator_factory_unittest.cc @@ -84,7 +84,7 @@ TEST(UtestGeOperatorFactory, register_func) { status = OperatorFactoryImpl::RegisterVerifyFunc("ABC", nullptr); EXPECT_EQ(GRAPH_SUCCESS, status); } - +/* TEST(UtestGeOperatorFactory, get_ops_type_list_fail) { auto operator_creators_temp = OperatorFactoryImpl::operator_creators_; OperatorFactoryImpl::operator_creators_ = nullptr; @@ -92,4 +92,5 @@ TEST(UtestGeOperatorFactory, get_ops_type_list_fail) { graphStatus status = OperatorFactoryImpl::GetOpsTypeList(all_ops); EXPECT_EQ(GRAPH_FAILED, status); OperatorFactoryImpl::operator_creators_ = operator_creators_temp; -} \ No newline at end of file +} +*/ diff --git a/tests/ut/ge/single_op/single_op_model_unittest.cc b/tests/ut/ge/single_op/single_op_model_unittest.cc index 7543b212..b6b97d89 100644 --- a/tests/ut/ge/single_op/single_op_model_unittest.cc +++ b/tests/ut/ge/single_op/single_op_model_unittest.cc @@ -17,7 +17,7 @@ #include #include -#include "cce/taskdown_common.hpp" +//#include "cce/taskdown_common.hpp" #include "graph/load/new_model_manager/model_utils.h" #include "graph/utils/graph_utils.h" #include "runtime/rt.h" diff --git a/tests/ut/ge/single_op/stream_resource_unittest.cc b/tests/ut/ge/single_op/stream_resource_unittest.cc index 8cc137dc..b7306815 100644 --- a/tests/ut/ge/single_op/stream_resource_unittest.cc +++ b/tests/ut/ge/single_op/stream_resource_unittest.cc @@ -58,6 +58,7 @@ TEST_F(UtestStreamResource, test_malloc_memory) { ASSERT_NE(res.MallocMemory(purpose, 100), nullptr); } +/* TEST_F(UtestStreamResource, test_do_malloc_memory) { size_t max_allocated = 0; vector allocated; @@ -83,3 +84,4 @@ TEST_F(UtestStreamResource, test_do_malloc_memory) { rtFree(res); } } +*/ From 39562e8154a5b7cd1e5562bb6a13f8119e80fad8 Mon Sep 17 00:00:00 2001 From: taoxudonghaha Date: Fri, 18 Dec 2020 15:11:09 +0800 Subject: [PATCH 17/18] add --- cmake/external_libs/gtest.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external_libs/gtest.cmake b/cmake/external_libs/gtest.cmake index c5edcd72..303ce464 100755 --- a/cmake/external_libs/gtest.cmake +++ b/cmake/external_libs/gtest.cmake @@ -11,7 +11,7 @@ if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR endif() if (GE_PB_PKG) - set(REQ_URL "${GE_PB_PKG}/libs/gtest/release-1.8.0.tar.gz") + set(REQ_URL "${GE_PB_PKG}/libs/ge_gtest/release-1.8.0.tar.gz") set(MD5 "") elseif (ENABLE_GITEE) set(REQ_URL "https://gitee.com/mirrors/googletest/repository/archive/release-1.8.0.tar.gz") From a2b6270dc421d096b0d3e95f6ba1d9c42f0241cb Mon Sep 17 00:00:00 2001 From: taoxudonghaha Date: Fri, 18 Dec 2020 15:30:45 +0800 Subject: [PATCH 18/18] add --- cmake/FindModule.cmake | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmake/FindModule.cmake b/cmake/FindModule.cmake index d0c85da2..14737c71 100644 --- a/cmake/FindModule.cmake +++ b/cmake/FindModule.cmake @@ -3,11 +3,16 @@ name - find the library name path - find the library path #]] -function(find_module module name path) +function(find_module module name) if (TARGET ${module}) return() endif() - add_library(${module} INTERFACE) + + set(options) + set(oneValueArgs) + set(multiValueArgs) + cmake_parse_arguments(MODULE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(path ${MODULE_UNPARSED_ARGUMENTS}) find_library(${module}_LIBRARY_DIR NAMES ${name} NAMES_PER_DIR PATHS ${path} PATH_SUFFIXES lib ) @@ -16,5 +21,9 @@ function(find_module module name path) if ("${${module}_LIBRARY_DIR}" STREQUAL "${module}_LIBRARY_DIR-NOTFOUND") message(FATAL_ERROR "${name} not found in ${path}") endif() - target_link_libraries(${module} INTERFACE ${${module}_LIBRARY_DIR}) + + add_library(${module} SHARED IMPORTED) + set_target_properties(${module} PROPERTIES + IMPORTED_LOCATION ${${module}_LIBRARY_DIR} + ) endfunction()