From 5f105f35360eaffd83f67f68699e30d2b2e5c36f Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Fri, 2 Apr 2021 09:27:59 +0800 Subject: [PATCH] add error msg --- ge/graph/passes/atomic_addr_clean_pass.cc | 6 +- ge/graph/passes/cast_translate_pass.cc | 4 +- ge/graph/passes/constant_folding_pass.cc | 9 + ge/graph/passes/constant_fuse_same_pass.cc | 9 + ge/graph/passes/control_trigger_pass.cc | 54 +++++ ge/graph/passes/ctrl_edge_transfer_pass.cc | 4 +- ge/graph/passes/data_pass.cc | 12 + ge/graph/passes/dimension_adjust_pass.cc | 18 +- ge/graph/passes/dimension_compute_pass.cc | 4 + ge/graph/passes/dropout_pass.cc | 2 + .../end_of_sequence_add_control_pass.cc | 6 + ge/graph/passes/enter_pass.cc | 16 +- ge/graph/passes/flow_ctrl_pass.cc | 225 ++++++++++++++++-- ge/graph/passes/folding_pass.cc | 56 ++++- ge/graph/passes/for_pass.cc | 34 +++ .../fuse_data_nodes_with_common_input_pass.cc | 16 ++ ge/graph/passes/get_original_format_pass.cc | 21 ++ ge/graph/passes/global_step_insert_pass.cc | 15 +- ge/graph/passes/guarantee_const_pass.cc | 11 + .../passes/hccl_continuous_memcpy_pass.cc | 76 +++++- ge/graph/passes/hccl_memcpy_pass.cc | 63 ++++- ge/graph/passes/identity_pass.cc | 5 + ge/graph/passes/infershape_pass.cc | 52 ++++ ge/graph/passes/inplace_support_check_pass.cc | 3 + .../input_output_connection_identify_pass.cc | 16 ++ ge/graph/passes/iterator_op_pass.cc | 152 ++++++++++-- ge/graph/passes/link_gen_mask_nodes_pass.cc | 3 + ge/graph/passes/mark_agnostic_pass.cc | 8 +- .../passes/mark_graph_unknown_status_pass.cc | 2 +- ge/graph/passes/memcpy_addr_async_pass.cc | 55 ++++- ge/graph/passes/merge_input_memcpy_pass.cc | 15 +- ge/graph/passes/merge_pass.cc | 26 +- ge/graph/passes/merge_to_stream_merge_pass.cc | 76 ++++-- ge/graph/preprocess/graph_preprocess.cc | 22 +- .../insert_op/util_insert_aipp_op.cc | 2 +- ge/graph/preprocess/multi_batch_copy_graph.cc | 28 +-- inc/framework/common/debug/log.h | 13 +- 37 files changed, 1018 insertions(+), 121 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 98b19fbe..c2c2ef65 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -312,11 +312,11 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr graphStatus status = GraphUtils::AddEdge(out_ctrl_anchor, in_ctrl_anchor); if (status != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:%d) and op:%s(%s)(in_index:%d) failed " + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " "when AssignRemovePass %s", out_ctrl_anchor->GetOwnerNode()->GetName().c_str(), - out_ctrl_anchor->GetOwnerNode()->GetType().c_str(), out_ctrl_anchor->GetIdx(), + out_ctrl_anchor->GetOwnerNode()->GetType().c_str(), in_ctrl_anchor->GetOwnerNode()->GetName().c_str(), - in_ctrl_anchor->GetOwnerNode()->GetType().c_str(), in_ctrl_anchor->GetIdx(), __FUNCTION__); + in_ctrl_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Graph add cleanAddrNode op out ctrl edge fail, dst node: %s.", atomic_node->GetName().c_str()); diff --git a/ge/graph/passes/cast_translate_pass.cc b/ge/graph/passes/cast_translate_pass.cc index 4dd1e5cd..0807d657 100644 --- a/ge/graph/passes/cast_translate_pass.cc +++ b/ge/graph/passes/cast_translate_pass.cc @@ -249,14 +249,14 @@ Status CastTranslatePass::FuseDstNTranslates(NodePtr &node) { GE_CHECK_NOTNULL(out_data_node); AddRePassNodesWithInOut(out_data_node); // Has checked nodes only has one in data anchor one out data anchor - GE_CHK_STATUS_RET(NodeUtils::MoveOutputEdges(out_data_node, base_node), "move out put edge failed"); + GE_CHK_GRAPH_STATUS_RET(NodeUtils::MoveOutputEdges(out_data_node, base_node), "move out put edge failed"); // Relink in control anchor, delete in data anchor auto in_ctr_anchor = out_data_node->GetInControlAnchor(); GE_CHECK_NOTNULL(in_ctr_anchor); for (const auto &peer_anchor : in_ctr_anchor->GetPeerOutControlAnchors()) { GE_CHECK_NOTNULL(base_node->GetInControlAnchor()); - GE_CHK_STATUS_RET(base_node->GetInControlAnchor()->LinkFrom(peer_anchor), "link from peer anchor failed"); + GE_CHK_GRAPH_STATUS_RET(base_node->GetInControlAnchor()->LinkFrom(peer_anchor), "link from peer anchor failed"); } in_ctr_anchor->UnlinkAll(); out_data_node->GetAllInDataAnchors().at(0)->UnlinkAll(); diff --git a/ge/graph/passes/constant_folding_pass.cc b/ge/graph/passes/constant_folding_pass.cc index 66e076af..77371702 100644 --- a/ge/graph/passes/constant_folding_pass.cc +++ b/ge/graph/passes/constant_folding_pass.cc @@ -33,11 +33,14 @@ Status RunOpKernelWithCheck(NodePtr &node, std::vector &outputs) { std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if ((instance_ptr == nullptr) || (!instance_ptr->InitFlag())) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when %s", __FUNCTION__); GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized."); return UNSUPPORTED; } OpsKernelInfoStorePtr kernel_info = instance_ptr->OpsKernelManagerObj().GetOpsKernelInfoStore(kKernelLibName); if (kernel_info == nullptr) { + REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed for op:%s(%s), when %s", + kKernelLibName.c_str(), node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Get op kernel info store %s failed", kKernelLibName.c_str()); return UNSUPPORTED; } @@ -45,6 +48,8 @@ Status RunOpKernelWithCheck(NodePtr &node, std::string ops_flag; kernel_info->opsFlagCheck(*node, ops_flag); if (ops_flag == kOpsFlagClose) { + REPORT_CALL_ERROR("E19999", "Call opsFlagCheck faled, ops kernel name:%s, op:%s(%s) when %s", + kKernelLibName.c_str(), node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); return UNSUPPORTED; } return FoldingPass::RunOpKernel(node, inputs, outputs); @@ -108,6 +113,8 @@ Status ConstantFoldingPass::Run(ge::NodePtr &node) { node->GetType().c_str()); return SUCCESS; } + REPORT_CALL_ERROR("E19999", "Calculate for node %s(%s) failed when ConstantFoldingPass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Calculate for node %s failed in constant folding", node->GetName().c_str()); return ret; } @@ -125,6 +132,8 @@ Status ConstantFoldingPass::Run(ge::NodePtr &node) { } if (outputs.empty()) { + REPORT_INNER_ERROR("E19999", "After calculate for node %s(%s), output weight is empty, check invalid " + "when ConstantFoldingPass %s", node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to constant folding on node %s," " no output weight", diff --git a/ge/graph/passes/constant_fuse_same_pass.cc b/ge/graph/passes/constant_fuse_same_pass.cc index eb8b3470..d7909b2b 100644 --- a/ge/graph/passes/constant_fuse_same_pass.cc +++ b/ge/graph/passes/constant_fuse_same_pass.cc @@ -57,6 +57,7 @@ void GetOutDataNodeToIndexMap(NodePtr &node, std::map & Status ConstantFuseSamePass::Run(ge::ComputeGraphPtr graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when ConstantFuseSamePass %s", __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "Compute graph is null."); return GE_GRAPH_PARAM_NULLPTR; } @@ -159,6 +160,12 @@ Status ConstantFuseSamePass::MoveOutDataEdges(NodePtr &src_node, NodePtr &dst_no } auto ret = dst_out_data_anchor->LinkTo(it->second); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:0 link to op:%s(%s) in index:%d failed when ConstantFuseSamePass %s", + dst_node->GetName().c_str(), dst_node->GetType().c_str(), + it->second->GetOwnerNode()->GetName().c_str(), it->second->GetOwnerNode()->GetType().c_str(), + it->second->GetIdx(), + __FUNCTION__); GELOGE(FAILED, "Failed to move out data edge from %s to %s", src_node->GetName().c_str(), dst_node->GetName().c_str()); return FAILED; @@ -185,6 +192,8 @@ Status ConstantFuseSamePass::FuseConstNodes(ComputeGraphPtr &graph, return FAILED; } if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed when CastTranslatePass %s", + node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/control_trigger_pass.cc b/ge/graph/passes/control_trigger_pass.cc index e179c64e..ae012681 100644 --- a/ge/graph/passes/control_trigger_pass.cc +++ b/ge/graph/passes/control_trigger_pass.cc @@ -70,6 +70,12 @@ Status ControlTriggerPass::HandleDynamicCtrlEdges(ComputeGraphPtr &graph, NodePt NodePtr constant = (branch_flag ? iter2->second.second : iter2->second.first); if ((GraphUtils::RemoveEdge(in_ctrl_node->GetOutControlAnchor(), node->GetInControlAnchor()) != GRAPH_SUCCESS) || (GraphUtils::AddEdge(in_ctrl_node->GetOutControlAnchor(), constant->GetInControlAnchor()) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s), then " + "add control edge between op:%s(%s) and op:%s(%s) failed, when ControlTriggerPass %s", + in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), + in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), + constant->GetName().c_str(), constant->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Replace ctrl edge fail, %s->%s, %s->%s.", in_ctrl_node->GetName().c_str(), node->GetName().c_str(), in_ctrl_node->GetName().c_str(), constant->GetName().c_str()); return FAILED; @@ -185,6 +191,8 @@ ControlNodeType ControlTriggerPass::TransferNodeType(const NodePtr &node, uint32 } else if ((type == MERGE) || (type == REFMERGE)) { OpDescPtr merge_desc = node->GetOpDesc(); if (merge_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "op_desc in merge node is nullptr, check invalid when ControlTriggerPass %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "FindPredInput fail, merge_desc is null, merge_node: %s.", node->GetName().c_str()); return kInvalidType; } @@ -264,14 +272,24 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr } if (GraphUtils::AddEdge(in_ctrl_node->GetOutControlAnchor(), orig_const->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed when ControlTriggerPass %s", + in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), + orig_const->GetName().c_str(), orig_const->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add in ctrl edge fail, %s->%s.", in_ctrl_node->GetName().c_str(), orig_const->GetName().c_str()); return FAILED; } if (GraphUtils::AddEdge(switch_node->GetOutDataAnchor(new_idx), identity_node->GetInDataAnchor(0)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%u) and op:%s(%s)(index:0) failed " + "when ControlTriggerPass %s", + switch_node->GetName().c_str(), switch_node->GetType().c_str(), new_idx, + identity_node->GetName().c_str(), identity_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add in data edge fail, %s->%s.", switch_desc->GetName().c_str(), identity_node->GetName().c_str()); return FAILED; } if (GraphUtils::AddEdge(identity_node->GetOutControlAnchor(), new_const->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed when ControlTriggerPass %s", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + new_const->GetName().c_str(), new_const->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add in ctrl edge fail, %s->%s.", identity_node->GetName().c_str(), new_const->GetName().c_str()); return FAILED; } @@ -282,6 +300,7 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr control_trigger_map_[node] = {pred_const}; } else { if (!iter->second.insert(pred_const).second) { + REPORT_INNER_ERROR("E19999", "Insert to control_trigger_map_ failed when ControlTriggerPass %s", __FUNCTION__); GELOGE(FAILED, "control_trigger_map_ insert failed."); return FAILED; } @@ -303,12 +322,15 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod const std::string name = node->GetName() + "_" + MERGE; OpDescPtr op_desc = MakeShared(name, MERGE); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when ControlTriggerPass %s", __FUNCTION__); GELOGE(FAILED, "Create Merge op %s: create op_desc fail.", name.c_str()); return nullptr; } if ((op_desc->AddInputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddInputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add input or ouput desc to op:%s(%s) failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Merge op %s: add input/output desc fail.", name.c_str()); return nullptr; } @@ -316,12 +338,20 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod GELOGI("Create Merge op:%s.", name.c_str()); NodePtr merge_node = graph->AddNode(op_desc); if (merge_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Merge op %s fail.", name.c_str()); return nullptr; } if ((GraphUtils::RemoveEdge(in_ctrl_node->GetOutControlAnchor(), node->GetInControlAnchor()) != GRAPH_SUCCESS) || (GraphUtils::AddEdge(merge_node->GetOutControlAnchor(), node->GetInControlAnchor()) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s), then " + "add control edge between op:%s(%s) and op:%s(%s) failed, when ControlTriggerPass %s", + in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), + merge_node->GetName().c_str(), merge_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Replace ctrl edge fail, %s->%s, %s->%s", in_ctrl_node->GetName().c_str(), node->GetName().c_str(), merge_node->GetName().c_str(), node->GetName().c_str()); return nullptr; @@ -343,6 +373,7 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer const std::string name = merge_node->GetName() + "_" + CONSTANT + (flag ? "_t" : "_f"); OpDescPtr op_desc = MakeShared(name, CONSTANT); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when ControlTriggerPass %s", __FUNCTION__); GELOGE(FAILED, "Create Const op %s: create op_desc fail.", name.c_str()); return nullptr; } @@ -350,15 +381,20 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer int32_t value = 0; GeTensorPtr const_value = MakeShared(data_desc, reinterpret_cast(&value), sizeof(int32_t)); if (const_value == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when ControlTriggerPass %s", __FUNCTION__); GELOGE(FAILED, "Create tensor fail."); return nullptr; } if (!AttrUtils::SetTensor(op_desc, ATTR_NAME_WEIGHTS, const_value)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when ControlTriggerPass %s", ATTR_NAME_WEIGHTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Const op %s: set attr ATTR_NAME_WEIGHTS fail.", name.c_str()); return nullptr; } if (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Const op %s: add output desc fail.", name.c_str()); return nullptr; } @@ -366,12 +402,17 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer GELOGI("Create Const op: %s", name.c_str()); NodePtr const_node = graph->AddNode(op_desc); if (const_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Const op %s fail.", name.c_str()); return nullptr; } uint32_t out_idx = (flag ? SWITCH_TRUE_OUTPUT : SWITCH_FALSE_OUTPUT); if (GraphUtils::AddEdge(const_node->GetOutDataAnchor(0), merge_node->GetInDataAnchor(out_idx)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:%u) failed " + "when ControlTriggerPass %s", const_node->GetName().c_str(), const_node->GetType().c_str(), + merge_node->GetName().c_str(), merge_node->GetType().c_str(), out_idx, __FUNCTION__); GELOGE(FAILED, "Add in data edge fail, %s->%s", const_node->GetName().c_str(), merge_node->GetName().c_str()); return nullptr; } @@ -390,11 +431,14 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std const GeTensorDesc &data_desc) { OpDescPtr op_desc = MakeShared(name, IDENTITY); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when ControlTriggerPass %s", __FUNCTION__); GELOGE(FAILED, "Create Identity op %s: create op_desc fail.", name.c_str()); return nullptr; } if ((op_desc->AddInputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add input or output desc to op:%s(%s) failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op %s: add input/output desc fail.", name.c_str()); return nullptr; } @@ -402,6 +446,8 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std GELOGI("Create Identity op:%s.", name.c_str()); NodePtr identity_node = graph->AddNode(op_desc); if (identity_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op %s fail.", name.c_str()); return nullptr; } @@ -418,17 +464,25 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std /// Status ControlTriggerPass::FindPredInput(const NodePtr &switch_node) { if (switch_node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param switch_node is nullptr, check invalid when ControlTriggerPass %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "switch_node is null"); return INTERNAL_ERROR; } InDataAnchorPtr in_cond_anchor = switch_node->GetInDataAnchor(SWITCH_PRED_INPUT); if (in_cond_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Index:%d in anchor of switch_node:%s(%s) is nullptr, check invalid " + "when ControlTriggerPass %s", SWITCH_PRED_INPUT, + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "in_cond_anchor is nullptr, node: %s.", switch_node->GetName().c_str()); return INTERNAL_ERROR; } OutDataAnchorPtr pred_cond_anchor = in_cond_anchor->GetPeerOutAnchor(); if (pred_cond_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Index:%d in anchor of switch_node:%s(%s), it's peer anchor is nullptr, " + "check invalid when ControlTriggerPass %s", SWITCH_PRED_INPUT, + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "pred_cond_anchor is nullptr, node: %s.", switch_node->GetName().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/ctrl_edge_transfer_pass.cc b/ge/graph/passes/ctrl_edge_transfer_pass.cc index a538a10c..598d2e14 100755 --- a/ge/graph/passes/ctrl_edge_transfer_pass.cc +++ b/ge/graph/passes/ctrl_edge_transfer_pass.cc @@ -64,13 +64,13 @@ Status CtrlEdgeTransferPass::Run(ge::ComputeGraphPtr graph) { for (auto &in_control_node : n->GetInControlNodes()) { GE_CHECK_NOTNULL(in_control_node); - GE_CHK_STATUS_RET(ge::GraphUtils::RemoveEdge(in_control_node->GetOutControlAnchor(), + GE_CHK_GRAPH_STATUS_RET(ge::GraphUtils::RemoveEdge(in_control_node->GetOutControlAnchor(), n->GetInControlAnchor()), "remove edge failed"); for (auto &out_node : n->GetOutNodes()) { if (out_node == nullptr) { continue; } - GE_CHK_STATUS_RET(ge::GraphUtils::AddEdge(in_control_node->GetOutControlAnchor(), + GE_CHK_GRAPH_STATUS_RET(ge::GraphUtils::AddEdge(in_control_node->GetOutControlAnchor(), out_node->GetInControlAnchor()), "add edge failed."); } } diff --git a/ge/graph/passes/data_pass.cc b/ge/graph/passes/data_pass.cc index 5bbd2fb1..847f9c8f 100644 --- a/ge/graph/passes/data_pass.cc +++ b/ge/graph/passes/data_pass.cc @@ -30,6 +30,8 @@ Status MappingSubgraphInput(const ComputeGraphPtr &graph, const std::functionGetOpDesc(), "index", index)) { + REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed when %s", "index", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to get index from data[%s]", node->GetName().c_str()); return FAILED; } @@ -38,6 +40,8 @@ Status MappingSubgraphInput(const ComputeGraphPtr &graph, const std::functionGetName().c_str(), index, parent_index); if (!AttrUtils::SetInt(node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when %s", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to set parent index for node %s", node->GetName().c_str()); return FAILED; } @@ -65,6 +69,9 @@ Status MappingSubgraphOutput(const ComputeGraphPtr &graph, const std::functionMutableInputDesc(index); GE_CHECK_NOTNULL(tensor); if (!AttrUtils::SetInt(tensor, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to tensor of op:%s(%s) input:%zu failed when %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), + index, __FUNCTION__); GELOGE(FAILED, "Failed to set parent index for graph %s", graph->GetName().c_str()); return FAILED; } @@ -140,12 +147,17 @@ Status DataPass::PostParseSubgraph(const ComputeGraphPtr &graph, const string &i auto post_func_it = subgraph_handle.find(parent_node->GetType()); if (post_func_it == subgraph_handle.end()) { + REPORT_INNER_ERROR("E19999", "The subgraph post func for node %s type %s is null, check invalid when DataPass %s", + parent_node->GetName().c_str(), parent_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "The subgraph post func for node %s type %s is null.", parent_node->GetName().c_str(), parent_node->GetType().c_str()); return FAILED; } if (post_func_it->second(ir_name, graph) != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Post process subgraph %s on node %s type %s failed when DataPass %s", + graph->GetName().c_str(), parent_node->GetName().c_str(), parent_node->GetType().c_str(), + __FUNCTION__); GELOGE(FAILED, "Failed to post process subgraph %s on node %s type %s", graph->GetName().c_str(), parent_node->GetName().c_str(), parent_node->GetType().c_str()); return FAILED; diff --git a/ge/graph/passes/dimension_adjust_pass.cc b/ge/graph/passes/dimension_adjust_pass.cc index 9677fa5f..b9706c4c 100755 --- a/ge/graph/passes/dimension_adjust_pass.cc +++ b/ge/graph/passes/dimension_adjust_pass.cc @@ -29,12 +29,15 @@ const int kRemoveInputIndex = 1; Status DimensionAdjustPass::Run(ge::NodePtr &node) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when DimensionAdjustPass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "node is nullptr."); return PARAM_INVALID; } OpDescPtr op_desc_ptr = node->GetOpDesc(); if (op_desc_ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid when DimensionAdjustPass %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "GetOpDesc return nullptr."); return PARAM_INVALID; } @@ -42,6 +45,8 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) { string type; Status ret = GetOriginalType(node, type); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get OriginalType of op:%s(%s) failed when DimensionAdjustPass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(ret, "DimensionAdjustPass get originnal type fail."); return ret; } @@ -69,12 +74,17 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) { if (ret == NOT_CHANGED) { return SUCCESS; } + REPORT_CALL_ERROR("E19999", "kernel compute for op:%s(%s) failed when DimensionAdjustPass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(ret, "DimensionAdjustPass compute failed"); return ret; } if (node->GetAllInDataAnchors().size() > static_cast(kRemoveInputIndex)) { ret = PassUtils::UnlinkNodeWithControlCopy(node, kRemoveInputIndex); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Unlink op:%s(%s) data input:%u with control edge copy failed " + "when DimensionAdjustPass %s", + node->GetName().c_str(), node->GetType().c_str(), kRemoveInputIndex, __FUNCTION__); GELOGE(ret, "DimensionAdjustPass unlink node with control copy fail."); return ret; } @@ -111,12 +121,12 @@ Status DimensionAdjustPass::DealWithInNodes(NodePtr &node) { GE_CHECK_NOTNULL(identity); GELOGI("Create new identity node[%s] after node %s[type: %s] success.", identity->GetName().c_str(), in_node->GetName().c_str(), in_node->GetType().c_str()); - GE_CHK_STATUS_RET(GraphUtils::AddEdge(in_node_anchor, identity->GetInDataAnchor(0))) + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(in_node_anchor, identity->GetInDataAnchor(0))) GE_CHECK_NOTNULL(identity->GetOutControlAnchor()); if (identity->GetOutControlAnchor()->IsLinkedWith(node->GetInControlAnchor())) { continue; } - GE_CHK_STATUS_RET(GraphUtils::AddEdge(identity->GetOutControlAnchor(), node->GetInControlAnchor())) + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(identity->GetOutControlAnchor(), node->GetInControlAnchor())) } } @@ -126,12 +136,14 @@ Status DimensionAdjustPass::DealWithInNodes(NodePtr &node) { NodePtr DimensionAdjustPass::AddIdentityNodeToGraph(const string &name, const GeTensorDesc &tensor, ComputeGraphPtr &graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when DimensionAdjustPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Comput graph ptr is null in creating identity node."); return nullptr; } OpDescPtr desc = MakeShared("", ""); if (desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when DimensionAdjustPass %s", __FUNCTION__); GELOGE(MEMALLOC_FAILED, "Failed to create op desc."); return nullptr; } @@ -141,6 +153,8 @@ NodePtr DimensionAdjustPass::AddIdentityNodeToGraph(const string &name, const Ge auto ret = desc->AddInputDesc(tensor); auto ret2 = desc->AddOutputDesc(tensor); if ((ret != GRAPH_SUCCESS) || (ret2 != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add input or ouput desc to op:%s(%s) failed when DimensionAdjustPass %s", + desc->GetName().c_str(), desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add input/output desc in creating identity."); return nullptr; } diff --git a/ge/graph/passes/dimension_compute_pass.cc b/ge/graph/passes/dimension_compute_pass.cc index dfa2d404..a2119444 100755 --- a/ge/graph/passes/dimension_compute_pass.cc +++ b/ge/graph/passes/dimension_compute_pass.cc @@ -39,12 +39,16 @@ Status DimensionComputePass::Run(ge::NodePtr &node) { if (ret == NOT_CHANGED) { return SUCCESS; } else { + REPORT_CALL_ERROR("E19999", "kernel compute for op:%s(%s) failed when DimensionComputePass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(ret, "DimensionComputePass Compute failed"); return ret; } } if (outputs.empty()) { + REPORT_INNER_ERROR("E19999", "After compute for node %s(%s), output weight is empty, check invalid " + "when DimensionComputePass %s", node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to compute dims for node %s," " no output weight", diff --git a/ge/graph/passes/dropout_pass.cc b/ge/graph/passes/dropout_pass.cc index 09c297a6..192be4b2 100644 --- a/ge/graph/passes/dropout_pass.cc +++ b/ge/graph/passes/dropout_pass.cc @@ -31,10 +31,12 @@ namespace ge { Status DropOutPass::Run(NodePtr &node) { GELOGD("DropOutPass running"); if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when DropOutPass %s", __FUNCTION__); GELOGE(FAILED, "parameter is null."); return FAILED; } if (node->GetOpDesc() == nullptr) { + REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid when DropOutPass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "param [opDesc] must not be null."); return PARAM_INVALID; } diff --git a/ge/graph/passes/end_of_sequence_add_control_pass.cc b/ge/graph/passes/end_of_sequence_add_control_pass.cc index d6503d0d..1511eac3 100755 --- a/ge/graph/passes/end_of_sequence_add_control_pass.cc +++ b/ge/graph/passes/end_of_sequence_add_control_pass.cc @@ -26,6 +26,8 @@ namespace ge { Status EndOfSequenceAddControlPass::Run(ComputeGraphPtr graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when EndOfSequenceAddControlPass %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "param [graph] must not be null."); return PARAM_INVALID; } @@ -82,6 +84,10 @@ Status EndOfSequenceAddControlPass::AddControlEdge(NodePtr &end_of_sequence, std } Status status = GraphUtils::AddEdge(out_ctrl_anchor, in_ctrl_anchor); if (status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Add control edge between op:%s(%s) and op:%s(%s) failed when EndOfSequenceAddControlPass %s", + end_of_sequence->GetName().c_str(), end_of_sequence->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Graph add EndOfSequence op out ctrl edge fail, dst node: %s.", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/enter_pass.cc b/ge/graph/passes/enter_pass.cc index 066c97cf..995579dd 100644 --- a/ge/graph/passes/enter_pass.cc +++ b/ge/graph/passes/enter_pass.cc @@ -37,6 +37,7 @@ Status EnterPass::Run(NodePtr &node) { // enter node has only one input if (node->GetInDataNodes().empty()) { + REPORT_INNER_ERROR("E19999", "Param node in data nodes is empty, check invalid when EnterPass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "enter_node %s has no input", node->GetName().c_str()); return PARAM_INVALID; } @@ -58,6 +59,9 @@ Status EnterPass::Run(NodePtr &node) { } GELOGI("Remove control edge from %s to %s.", node->GetName().c_str(), out_ctrl_node->GetName().c_str()); if (GraphUtils::RemoveEdge(node->GetOutControlAnchor(), out_ctrl_node->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed when EnterPass %s", + node->GetName().c_str(), node->GetType().c_str(), + out_ctrl_node->GetName().c_str(), out_ctrl_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Remove Enter ctrl output fail, %s->%s", node->GetName().c_str(), out_ctrl_node->GetName().c_str()); return FAILED; @@ -89,14 +93,14 @@ Status EnterPass::OptimizeEnterWithOnlyDataOut(NodePtr &node, NodePtr &in_node) } GE_CHECK_NOTNULL(in_node->GetOutDataAnchor(0)); - GE_CHK_STATUS_RET(in_node->GetOutDataAnchor(0)->Unlink(node->GetInDataAnchor(0))) + GE_CHK_GRAPH_STATUS_RET(in_node->GetOutDataAnchor(0)->Unlink(node->GetInDataAnchor(0))) const auto &out_data_anchor = node->GetOutDataAnchor(0); GE_CHECK_NOTNULL(out_data_anchor); for (const auto &peer_in_data_anchor : out_data_anchor->GetPeerInDataAnchors()) { - GE_CHK_STATUS_RET(out_data_anchor->Unlink(peer_in_data_anchor)) - GE_CHK_STATUS_RET(in_node->GetOutDataAnchor(0)->LinkTo(peer_in_data_anchor)) + GE_CHK_GRAPH_STATUS_RET(out_data_anchor->Unlink(peer_in_data_anchor)) + GE_CHK_GRAPH_STATUS_RET(in_node->GetOutDataAnchor(0)->LinkTo(peer_in_data_anchor)) } - GE_CHK_STATUS_RET(GraphUtils::RemoveNodeWithoutRelink(node->GetOwnerComputeGraph(), node)) + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveNodeWithoutRelink(node->GetOwnerComputeGraph(), node)) AddNodeDeleted(node); AddRePassNodesWithInOut(in_node); @@ -136,11 +140,11 @@ Status EnterPass::UnlinkCtrlEdgeBeforeConst(NodePtr &node) { } GELOGI("Unlink control edge from %s to %s.", node->GetName().c_str(), out_ctrl_node->GetName().c_str()); - GE_CHK_STATUS_RET(out_ctrl_anchor->Unlink(out_ctrl_node->GetInControlAnchor())) + GE_CHK_GRAPH_STATUS_RET(out_ctrl_anchor->Unlink(out_ctrl_node->GetInControlAnchor())) for (auto &out_node_of_const : out_nodes_of_const) { if (!out_ctrl_anchor->IsLinkedWith(out_node_of_const->GetInControlAnchor())) { GELOGI("Link control edge from %s to %s.", node->GetName().c_str(), out_node_of_const->GetName().c_str()); - GE_CHK_STATUS_RET(out_ctrl_anchor->LinkTo(out_node_of_const->GetInControlAnchor())) + GE_CHK_GRAPH_STATUS_RET(out_ctrl_anchor->LinkTo(out_node_of_const->GetInControlAnchor())) } } } diff --git a/ge/graph/passes/flow_ctrl_pass.cc b/ge/graph/passes/flow_ctrl_pass.cc index 435130b3..ad9e2388 100755 --- a/ge/graph/passes/flow_ctrl_pass.cc +++ b/ge/graph/passes/flow_ctrl_pass.cc @@ -115,6 +115,7 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod const std::vector &output_list) { OpDescPtr op_desc = MakeShared(node_name, node_type); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when FlowCtrlPass %s", __FUNCTION__); GELOGE(FAILED, "Make OpDesc failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); return nullptr; } @@ -122,6 +123,8 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod for (auto &input_desc : input_list) { graphStatus graph_status = op_desc->AddInputDesc(input_desc); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when FlowCtrlPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add node:%s intput desc failed, error=%u.", node_name.c_str(), graph_status); return nullptr; } @@ -130,14 +133,23 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod for (auto &output_desc : output_list) { graphStatus graph_status = op_desc->AddOutputDesc(output_desc); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed when FlowCtrlPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add node:%s output desc failed, error=%u.", node_name.c_str(), graph_status); return nullptr; } } - GE_IF_BOOL_EXEC(compute_graph == nullptr, DOMI_LOGE("compute_graph is nullptr"); return nullptr); + GE_IF_BOOL_EXEC(compute_graph == nullptr, + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid when FlowCtrlPass %s", + __FUNCTION__); + DOMI_LOGE("compute_graph is nullptr"); + return nullptr); NodePtr node = compute_graph->AddNode(op_desc); if (node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when FlowCtrlPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str(), + __FUNCTION__); GELOGE(FAILED, "add node failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); return nullptr; } @@ -149,9 +161,15 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const string &switch_name, const NodePtr &loop_cond, const NodePtr &iter_per_loop) { GE_IF_BOOL_EXEC(loop_cond == nullptr || loop_cond->GetOpDesc() == nullptr, - GELOGE(FAILED, "loop_cond is null"); return nullptr); + REPORT_INNER_ERROR("E19999", "Param loop_cond or it's op_desc is nullptr, " + "check invalid when FlowCtrlPass %s", __FUNCTION__); + GELOGE(FAILED, "loop_cond is null"); + return nullptr); GE_IF_BOOL_EXEC(iter_per_loop == nullptr || iter_per_loop->GetOpDesc() == nullptr, - GELOGE(FAILED, "iter_per_loop is nullptr"); return nullptr); + REPORT_INNER_ERROR("E19999", "Param iter_per_loop or it's op_desc is nullptr, " + "check invalid when FlowCtrlPass %s", __FUNCTION__); + GELOGE(FAILED, "iter_per_loop is nullptr"); + return nullptr); std::vector input_desc_list = {loop_cond->GetOpDesc()->GetOutputDesc(0), iter_per_loop->GetOpDesc()->GetOutputDesc(0)}; std::vector output_desc_list; @@ -164,6 +182,10 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const // set input 0 graphStatus add_ret = GraphUtils::AddEdge(loop_cond->GetOutDataAnchor(0), stream_switch->GetInDataAnchor(0)); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed " + "when FlowCtrlPass %s", + loop_cond->GetName().c_str(), loop_cond->GetType().c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add loop_cond_node to switch_node:%s edge failed, ret = %u.", switch_name.c_str(), add_ret); return nullptr; } @@ -171,6 +193,10 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const // set input 1 add_ret = GraphUtils::AddEdge(iter_per_loop->GetOutDataAnchor(0), stream_switch->GetInDataAnchor(1)); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed " + "when FlowCtrlPass %s", + iter_per_loop->GetName().c_str(), iter_per_loop->GetType().c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add iter_per_loop_node to switch_node:%s edge failed, ret = %u.", switch_name.c_str(), add_ret); return nullptr; } @@ -178,13 +204,20 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const // stream switch op need switch cond by attr. GE_IF_BOOL_EXEC(!AttrUtils::SetInt(stream_switch->GetOpDesc(), ATTR_NAME_STREAM_SWITCH_COND, static_cast(RT_LESS)), - DOMI_LOGE("set ATTR_NAME_STREAM_SWITCH_COND failed"); return nullptr); + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when FlowCtrlPass %s", + ATTR_NAME_STREAM_SWITCH_COND.c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str(), __FUNCTION__); + DOMI_LOGE("set ATTR_NAME_STREAM_SWITCH_COND failed"); return nullptr); return stream_switch; } NodePtr FlowCtrlPass::AddVariableNode(ComputeGraphPtr &compute_graph, const string &name) { - GE_IF_BOOL_EXEC(compute_graph == nullptr, DOMI_LOGE("compute_graph is nullptr"); return nullptr); + GE_IF_BOOL_EXEC(compute_graph == nullptr, + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid when FlowCtrlPass %s", + __FUNCTION__); + DOMI_LOGE("compute_graph is nullptr"); + return nullptr); NodePtr exist_node = compute_graph->FindNode(name); if (exist_node != nullptr) { GELOGD("Node %s already exist, no need add.", name.c_str()); @@ -193,10 +226,14 @@ NodePtr FlowCtrlPass::AddVariableNode(ComputeGraphPtr &compute_graph, const stri // fetch and set tensor desc GeTensorDesc tensor_desc; if (ge::VarManager::Instance(compute_graph->GetSessionID()) == nullptr) { + REPORT_INNER_ERROR("E19999", "Get VarManager by session_id:%lu failed when FlowCtrlPass %s", + compute_graph->GetSessionID(), __FUNCTION__); return nullptr; } Status ret = ge::VarManager::Instance(compute_graph->GetSessionID())->GetCurVarDesc(name, tensor_desc); if (ret != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Get var tensor from VarManager by name:%s failed, session_id:%lu, " + "when FlowCtrlPass %s", name.c_str(), compute_graph->GetSessionID(), __FUNCTION__); GELOGE(FAILED, "Get var desc fail, name:%s", name.c_str()); return nullptr; } @@ -238,6 +275,9 @@ Status FlowCtrlPass::AddGlobalStepVariableNode(ComputeGraphPtr &compute_graph) { // add ctrl edges graphStatus add_ret = GraphUtils::AddEdge(global_step->GetOutControlAnchor(), output_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed when FlowCtrlPass %s", + global_step->GetName().c_str(), global_step->GetType().c_str(), + output_node->GetName().c_str(), output_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add global_step to netoutput edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -249,6 +289,8 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s const string &node_name, const NodePtr &ref_node, const NodePtr &value_node) { GE_IF_BOOL_EXEC(ref_node == nullptr || value_node == nullptr || ref_node->GetOpDesc() == nullptr || value_node->GetOpDesc() == nullptr, + REPORT_INNER_ERROR("E19999", "Param ref_node or value_node or their op_desc has nullptr, " + "check invalid when FlowCtrlPass %s", __FUNCTION__); GELOGE(FAILED, "ref node or value node is null"); return nullptr); GeTensorDesc ref_tensor_desc = ref_node->GetOpDesc()->GetOutputDesc(0); @@ -263,12 +305,20 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s // assign node input 0 = ref_node graphStatus add_ret = GraphUtils::AddEdge(ref_node->GetOutDataAnchor(0), assign_node->GetInDataAnchor(0)); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed " + "when FlowCtrlPass %s", + ref_node->GetName().c_str(), ref_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add ref_node to %s edge failed, add_ret=%u.", node_name.c_str(), add_ret); return nullptr; } // assign input 1 = value_node add_ret = GraphUtils::AddEdge(value_node->GetOutDataAnchor(0), assign_node->GetInDataAnchor(1)); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed " + "when FlowCtrlPass %s", + value_node->GetName().c_str(), value_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add value_node to %s edge failed, add_ret=%u.", node_name.c_str(), add_ret); return nullptr; } @@ -298,10 +348,24 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co string active_name = switch_node->GetName() + "_StreamActive"; // add attr for stream assign model to break branch. - GE_CHK_STATUS_RET(SetStreamLabel(assign_add_node_in_fpbp_loop_, active_name), "set stream label failed"); + auto status = SetStreamLabel(assign_add_node_in_fpbp_loop_, active_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when FlowCtrlPass %s", + active_name.c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str(), + assign_add_node_in_fpbp_loop_->GetType().c_str(), __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } // used for stream assign to find true branch - GE_CHK_STATUS_RET(SetActiveLabelList(switch_node, { active_name }), "set active label list failed"); + status = SetActiveLabelList(switch_node, { active_name }); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed when FlowCtrlPass %s", + active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set active_label_list failed."); + return status; + } // 2. Insert active node NodePtr active_node = InsertOp(compute_graph, STREAMACTIVE, active_name, {}, {}); @@ -309,14 +373,29 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co GELOGE(FAILED, "Insert stream active node:%s for IterCtrlTrueStream failed.", active_name.c_str()); return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(active_node, active_name), "set stream label failed"); + status = SetStreamLabel(active_node, active_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when FlowCtrlPass %s", + active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } GE_IF_BOOL_EXEC(!AttrUtils::SetBool(active_node->GetOpDesc(), ATTR_NAME_IS_LOOP_ACTIVE, true), - DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); return FAILED); + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when FlowCtrlPass %s", + ATTR_NAME_IS_LOOP_ACTIVE.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); + DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); + return FAILED); // add ctrl edges graphStatus add_ret = GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), assign_add_node_in_fpbp_loop_->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", switch_node->GetName().c_str(), switch_node->GetType().c_str(), + assign_add_node_in_fpbp_loop_->GetName().c_str(), + assign_add_node_in_fpbp_loop_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add switch_node to assign_add_node ctrl edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -324,6 +403,10 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co add_ret = GraphUtils::AddEdge(assign_add_node_in_fpbp_loop_->GetOutControlAnchor(), active_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", assign_add_node_in_fpbp_loop_->GetName().c_str(), + assign_add_node_in_fpbp_loop_->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add assign_add_node to active_node ctrl edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -351,10 +434,20 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(assign_node, switch_node->GetName()), "set stream label failed."); + auto status = SetStreamLabel(assign_node, switch_node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when FlowCtrlPass %s", + switch_node->GetName().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } graphStatus add_ret = GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), assign_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", switch_node->GetName().c_str(), switch_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add switch_node to assign_node ctrl edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -368,15 +461,32 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c GELOGE(FAILED, "Insert stream active node:%s for IterCtrlTrueStream failed.", active_name.c_str()); return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(active_node, switch_node->GetName()), "set stream label failed"); + status = SetStreamLabel(active_node, switch_node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when FlowCtrlPass %s", + switch_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } GE_CHK_STATUS_RET(SetSwitchBranchNodeLabel(active_node, switch_node->GetName()), "set switch branch node label failed."); string model_exit_name = switch_node->GetName() + "_ModelExit"; GE_CHK_STATUS_RET(SetActiveLabelList(active_node, { model_exit_name }), "set active label list failed"); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed when FlowCtrlPass %s", + model_exit_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set active_label_list failed."); + return status; + } add_ret = GraphUtils::AddEdge(assign_node->GetOutControlAnchor(), active_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", assign_node->GetName().c_str(), assign_node->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add assign_node to active_node ctrl edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -387,10 +497,20 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c GELOGE(FAILED, "Insert model_exit node:%s for IterCtrlTrueStream failed.", model_exit_name.c_str()); return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(model_exit_node, model_exit_name), "set stream label failed"); + status = SetStreamLabel(model_exit_node, model_exit_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when FlowCtrlPass %s", + model_exit_name.c_str(), model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } add_ret = GraphUtils::AddEdge(active_node->GetOutControlAnchor(), model_exit_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", active_node->GetName().c_str(), assign_node->GetType().c_str(), + model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add active_node to model_exit_node ctrl edge failed, add_ret=%u.", add_ret); return FAILED; } @@ -433,10 +553,20 @@ Status FlowCtrlPass::AddFpBpIteratorCtrl(ComputeGraphPtr &compute_graph, NodePtr GELOGE(FAILED, "InsertStreamSwitchOp:%s failed.", switch_name.c_str()); return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(switch_node, switch_name), "set stream label failed"); + auto status = SetStreamLabel(switch_node, switch_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when FlowCtrlPass %s", + switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } graphStatus add_ret = GraphUtils::AddEdge(pre_node->GetOutControlAnchor(), switch_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", pre_node->GetName().c_str(), pre_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add pre node:%s to switch_node:%s ctrl edge failed, ret = %u.", pre_node_name.c_str(), switch_name.c_str(), add_ret); return FAILED; @@ -477,9 +607,14 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, * itersPerLoop loopCond */ GE_IF_BOOL_EXEC(loop_after_node == nullptr || compute_graph == nullptr, - DOMI_LOGE("loop after node or compute graph is null."); return FAILED); + REPORT_INNER_ERROR("E19999", "Param loop_after_node or compute_graph is nullptr, " + "check invalid when FlowCtrlPass %s", __FUNCTION__); + DOMI_LOGE("loop after node or compute graph is null."); + return FAILED); InDataAnchorPtr in_anchor = loop_after_node->GetInDataAnchor(0); if (in_anchor == nullptr || in_anchor->GetPeerOutAnchor() == nullptr) { + REPORT_INNER_ERROR("E19999", "Param loop_after_node:%s(%s) no in data node, check invalid when FlowCtrlPass %s", + loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Find %s in data anchor failed.", loop_after_node->GetName().c_str()); return FAILED; } @@ -488,17 +623,26 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, // 1. Get variables NodePtr loop_cond_node = compute_graph->FindNode(NODE_NAME_FLOWCTRL_LOOP_COND); if (loop_cond_node == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s not found in graph:%s, check invalid when FlowCtrlPass %s", + NODE_NAME_FLOWCTRL_LOOP_COND.c_str(), compute_graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Find node :%s failed.", NODE_NAME_FLOWCTRL_LOOP_COND.c_str()); return FAILED; } NodePtr iter_per_loop_node = compute_graph->FindNode(NODE_NAME_FLOWCTRL_LOOP_PER_ITER); if (iter_per_loop_node == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s not found in graph:%s, check invalid when FlowCtrlPass %s", + NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str(), compute_graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Find node :%s failed.", NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str()); return FAILED; } // 2. Add StreamSwitch and edges to switch_node. - GE_IF_BOOL_EXEC(loop_pre_node == nullptr, DOMI_LOGE("loop pre node is null."); return FAILED); + GE_IF_BOOL_EXEC(loop_pre_node == nullptr, + REPORT_INNER_ERROR("E19999", "Param loop_after_node:%s(%s) no in data node, " + "check invalid when FlowCtrlPass %s", loop_after_node->GetName().c_str(), + loop_after_node->GetType().c_str(), __FUNCTION__); + DOMI_LOGE("loop pre node is null."); + return FAILED); string switch_name = loop_pre_node->GetName() + "_" + NODE_NAME_STREAM_SWITCH; NodePtr switch_node = InsertStreamSwitchOp(compute_graph, switch_name, loop_cond_node, iter_per_loop_node); if (switch_node == nullptr) { @@ -506,16 +650,29 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(switch_node, switch_name), "set stream label failed."); + auto status = SetStreamLabel(switch_node, switch_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when FlowCtrlPass %s", + switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } graphStatus add_ret = GraphUtils::AddEdge(loop_pre_node->GetOutControlAnchor(), switch_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", loop_pre_node->GetName().c_str(), loop_pre_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add loop_pre_node:%s to switch_node:%s ctrl edge failed, ret = %u.", loop_pre_node->GetName().c_str(), switch_name.c_str(), add_ret); return FAILED; } add_ret = GraphUtils::AddEdge(loop_after_node->GetOutControlAnchor(), switch_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add node:%s to switch_node:%s ctrl edge failed, ret = %u.", loop_after_node->GetName().c_str(), switch_name.c_str(), add_ret); return FAILED; @@ -529,22 +686,50 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, return FAILED; } - GE_CHK_STATUS_RET(SetStreamLabel(active_node, active_name), "set stream label failed."); + status = SetStreamLabel(active_node, active_name); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when FlowCtrlPass %s", + active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } GE_IF_BOOL_EXEC(!AttrUtils::SetBool(active_node->GetOpDesc(), ATTR_NAME_IS_LOOP_ACTIVE, true), - DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); return FAILED); + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when ControlTriggerPass %s", + ATTR_NAME_IS_LOOP_ACTIVE.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); + DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); + return FAILED); add_ret = GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), active_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FlowCtrlPass %s", switch_node->GetName().c_str(), switch_node->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add switch_node:%s to active_node:%s ctrl edge failed, ret = %u.", switch_name.c_str(), active_name.c_str(), add_ret); return FAILED; } // used for stream assign to find true branch - GE_CHK_STATUS_RET(SetActiveLabelList(switch_node, { active_name }), "set active label list failed."); + status = SetActiveLabelList(switch_node, { active_name }); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed when FlowCtrlPass %s", + active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set active_label_list failed."); + return status; + } // used for stream assign to find active stream - GE_CHK_STATUS_RET(SetActiveLabelList(active_node, { loop_pre_node->GetName() }), "set active label list failed"); + status = SetActiveLabelList(active_node, { loop_pre_node->GetName() }); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed when FlowCtrlPass %s", + loop_pre_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "set active_label_list failed."); + return status; + } active_nodes_in_iter_loop_.push_back(active_node); return SUCCESS; } diff --git a/ge/graph/passes/folding_pass.cc b/ge/graph/passes/folding_pass.cc index 227a0f61..c898d2a9 100755 --- a/ge/graph/passes/folding_pass.cc +++ b/ge/graph/passes/folding_pass.cc @@ -35,6 +35,7 @@ namespace ge { namespace folding_pass { shared_ptr GetKernelByType(const NodePtr &node) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when %s", __FUNCTION__); GELOGE(FAILED, "parameter is null."); return nullptr; } @@ -42,6 +43,9 @@ shared_ptr GetKernelByType(const NodePtr &node) { string type = node->GetType(); if (type == FRAMEWORKOP) { if (!ge::AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, type)) { + REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed when %s", + ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); return nullptr; } } @@ -79,6 +83,7 @@ IndexsToAnchors GetIndexAndPeerInDataAnchors(NodePtr &node) { NodePtr AddConstNodeToGraph(GeTensorPtr &tensor, ComputeGraphPtr &graph) { auto const_desc = OpDescUtils::CreateConstOp(tensor); if (const_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "Create Const op failed when %s", __FUNCTION__); GELOGE(OUT_OF_MEMORY, "Failed to get const desc from tensor"); return nullptr; } @@ -90,12 +95,14 @@ NodePtr AddConstNodeToGraph(GeTensorPtr &tensor, ComputeGraphPtr &graph) { NodePtr AddIdentityNodeToGraph(const std::string &name, const GeTensorDesc &tensor, ComputeGraphPtr &graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Compute graph ptr is null in creating identity node."); return nullptr; } OpDescPtr desc = MakeShared("", ""); if (desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when %s", __FUNCTION__); GELOGE(MEMALLOC_FAILED, "Failed to create op desc."); return nullptr; } @@ -105,6 +112,8 @@ NodePtr AddIdentityNodeToGraph(const std::string &name, const GeTensorDesc &tens auto ret = desc->AddInputDesc(tensor); auto ret2 = desc->AddOutputDesc(tensor); if ((ret != GRAPH_SUCCESS) || (ret2 != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add input or output desc to op:%s(%s) failed when %s", + desc->GetName().c_str(), desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add input/output desc in creating Identity."); return nullptr; } @@ -116,7 +125,12 @@ NodePtr AddIdentityNodeToGraph(const std::string &name, const GeTensorDesc &tens Status FoldingPass::RunOpKernel(NodePtr &node, const vector &inputs, std::vector &outputs) { - return HostCpuEngine::GetInstance().Run(node, inputs, outputs); + auto status = HostCpuEngine::GetInstance().Run(node, inputs, outputs); + if (status != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Run OpKernel for op:%s(%s) in HostCpuEngine failed when FoldingPass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); + } + return status; } Status FoldingPass::Folding(NodePtr &node, vector &outputs) { @@ -137,6 +151,8 @@ Status FoldingPass::Folding(NodePtr &node, vector &outputs) { auto in_data_nodes = node->GetInDataNodes(); std::unordered_set in_data_nodes_set(in_data_nodes.begin(), in_data_nodes.end()); if (IsolateAndDeleteNode(node, {}) != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) faild when %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to isolate and delete node %s, type %s.", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -149,6 +165,8 @@ Status FoldingPass::Folding(NodePtr &node, vector &outputs) { continue; } if (IsolateAndDeleteNode(pre_node, {}) != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) faild when FoldingPass %s", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to isolate and delete in data node %s, type %s.", pre_node->GetName().c_str(), pre_node->GetType().c_str()); return INTERNAL_ERROR; @@ -177,6 +195,10 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { GELOGI("The in_node name is %s, and node type is %s.", in_node->GetName().c_str(), in_node->GetType().c_str()); auto ret = in_node_anchor->Unlink(in_data_anchor); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed when FoldingPass %s", + in_node->GetName().c_str(), in_node->GetType().c_str(), in_node_anchor->GetIdx(), + node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to unlink anchor between const node %s to constant-folding-node %s, type %s.", in_node->GetName().c_str(), node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -192,6 +214,10 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { } ret = GraphUtils::AddEdge(in_node_anchor, identity->GetInDataAnchor(0)); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(inde:0) failed " + "when FoldingPass %s", + in_node->GetName().c_str(), in_node->GetType().c_str(), in_node_anchor->GetIdx(), + identity->GetName().c_str(), identity->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add edge, from node %s to node %s.", in_node->GetName().c_str(), identity->GetName().c_str()); return INTERNAL_ERROR; @@ -199,6 +225,9 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { GELOGI("Create new identity node success."); ret = GraphUtils::AddEdge(identity->GetOutControlAnchor(), node->GetInControlAnchor()); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FoldingPass %s", identity->GetName().c_str(), identity->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add edge, from node %s to node %s.", in_node->GetName().c_str(), node->GetName().c_str()); return INTERNAL_ERROR; @@ -212,6 +241,7 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_anchors, std::vector &v_weight) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when FoldingPass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "node is null"); return FAILED; } @@ -219,6 +249,8 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho for (auto &index_to_anchors : indexes_to_anchors) { auto index = static_cast(index_to_anchors.first); if (index >= v_weight.size()) { + REPORT_INNER_ERROR("E19999", "Index:%lu in param index_to_anchors >= param v_weight.size:%zu, " + "check invalid when FoldingPass %s", index, v_weight.size(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to constant fold on node %s type %s, " "the out nodes num %lu calculated is less than the node out anchor index %zu", @@ -227,6 +259,8 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho } GeTensorPtr weight = v_weight[index]; if (weight == nullptr) { + REPORT_INNER_ERROR("E19999", "Index:%lu in param v_weight is nullptr check invalid when FoldingPass %s", + index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to constant fold on node %s type %s, the %lust node calculated is null", node->GetName().c_str(), node->GetType().c_str(), index); return INTERNAL_ERROR; @@ -243,6 +277,8 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho // add new const to re-pass node for (auto &in_anchor : index_to_anchors.second) { if (in_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Index:%lu in param index_to_anchors has nullptr member in_anchor, " + "check invalid when FoldingPass %s", index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "In anchor is nullptr."); return INTERNAL_ERROR; } @@ -254,6 +290,9 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho } Status ret = GraphUtils::AddEdge(node->GetOutControlAnchor(), const_node->GetInControlAnchor()); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when FoldingPass %s", node->GetName().c_str(), node->GetType().c_str(), + const_node->GetName().c_str(), const_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add control edge, from node %s to const node %s.", node->GetName().c_str(), const_node->GetName().c_str()); return INTERNAL_ERROR; @@ -263,6 +302,9 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho if (AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_STREAM_LABEL, stream_label)) { GE_CHECK_NOTNULL(const_node->GetOpDesc()); if (!AttrUtils::SetStr(const_node->GetOpDesc(), ATTR_NAME_STREAM_LABEL, stream_label)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when FoldingPass %s", + ATTR_NAME_STREAM_LABEL.c_str(), + const_node->GetName().c_str(), const_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to set stream label on dynamic const node %s, with stream label:%s.", const_node->GetName().c_str(), stream_label.c_str()); return INTERNAL_ERROR; @@ -279,6 +321,8 @@ Status FoldingPass::RemoveNodeKeepingCtrlEdges(NodePtr &node) { GE_IF_BOOL_EXEC(node == nullptr, GELOGE(PARAM_INVALID, "node is null"); return PARAM_INVALID); auto ret = GraphUtils::IsolateNode(node, {}); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) in graph failed when FoldingPass %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to isolate the folding-node %s type %s", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -287,6 +331,8 @@ Status FoldingPass::RemoveNodeKeepingCtrlEdges(NodePtr &node) { auto graph = node->GetOwnerComputeGraph(); ret = GraphUtils::RemoveNodeWithoutRelink(graph, node); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed when FoldingPass %s", + node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to remove node %s from graph", node->GetName().c_str()); return INTERNAL_ERROR; } @@ -297,6 +343,7 @@ Status FoldingPass::RemoveNodeKeepingCtrlEdges(NodePtr &node) { Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &node, int node_index) { // the origin edge must be removed before add if (in_anchor == nullptr || node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node or in_anchor is nullptr, check invalid when FoldingPass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "in anchor or node is null"); return PARAM_INVALID; } @@ -309,6 +356,9 @@ Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &n auto new_out_anchor = node->GetOutDataAnchor(node_index); if (new_out_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Param out index:%d data anchor of node:%s(%s) is nullptr, check invalid " + "when FoldingPass %s", node_index, node->GetName().c_str(), node->GetType().c_str(), + __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add node to in anchor," " the index %d for node %s, type %s is invalid", @@ -316,6 +366,10 @@ Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &n return INTERNAL_ERROR; } if (GraphUtils::AddEdge(new_out_anchor, in_anchor) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed " + "when FoldingPass %s", node->GetName().c_str(), node->GetType().c_str(), node_index, + in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), + in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add edge between anchors," " new node %s, type %s", diff --git a/ge/graph/passes/for_pass.cc b/ge/graph/passes/for_pass.cc index 3b7a0886..3f9e3746 100644 --- a/ge/graph/passes/for_pass.cc +++ b/ge/graph/passes/for_pass.cc @@ -64,12 +64,16 @@ Status ForPass::Run(NodePtr &node) { ComputeGraphPtr cond_graph = BuildCondGraph(while_info); if ((cond_graph == nullptr) || (root_graph->AddSubgraph(cond_graph) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Build cond graph failed or add cond subgraph to root_graph:%s failed when ForPass %s", + root_graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Add while_cond_graph failed, node:%s.", node->GetName().c_str()); return FAILED; } ComputeGraphPtr body_graph = BuildBodyGraph(while_info); if ((body_graph == nullptr) || (root_graph->AddSubgraph(body_graph) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Build body graph failed or add body subgraph to root_graph:%s failed when ForPass %s", + root_graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Add while_body_graph failed, node:%s.", node->GetName().c_str()); return FAILED; } @@ -99,6 +103,10 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n OutDataAnchorPtr limit = FindInputWithIndex(node, FOR_LIMIT_INPUT); OutDataAnchorPtr delta = FindInputWithIndex(node, FOR_DELTA_INPUT); if ((start == nullptr) || (limit == nullptr) || (delta == nullptr)) { + REPORT_INNER_ERROR("E19999", "FOR_START_INPUT index:%d or FOR_LIMIT_INPUT index:%d or FOR_DELTA_INPUT index:%d " + "in data anchor of op:%s(%s) lack, check invalid when ForPass %s", + FOR_START_INPUT, FOR_LIMIT_INPUT, FOR_DELTA_INPUT, + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "BuildForInfo for %s failed: start/limit/delta is NULL.", node->GetName().c_str()); return FAILED; } @@ -118,11 +126,15 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n // For node has and only has one sub_graph std::string for_body_name = op_desc->GetSubgraphInstanceName(0); if (for_body_name.empty()) { + REPORT_INNER_ERROR("E19999", "Get subgraph name from op:%s(%s) by index 0 failed when ForPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "BuildForInfo for %s failed: sub_graph_name is empty.", node->GetName().c_str()); return FAILED; } ComputeGraphPtr for_body = root_graph->GetSubgraph(for_body_name); if (for_body == nullptr) { + REPORT_INNER_ERROR("E19999", "Get subgraph from graph:%s by name:%s failed when ForPass %s", + root_graph->GetName().c_str(), for_body_name.c_str(), __FUNCTION__); GELOGE(FAILED, "BuildForInfo for %s failed: for_body_graph is NULL.", node->GetName().c_str()); return FAILED; } @@ -222,6 +234,8 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i std::string i_name = for_name + "_i"; NodePtr i_node = graph->AddNode(CreateConstDesc(i_name, 0)); if (i_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(Const) to graph:%s failed when ForPass %s", + i_name.c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "TranWhileInfo failed: create i_node failed."); return FAILED; } @@ -232,6 +246,9 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i // Const node has and only has one output, Identity node has and only has one input if ((identity_node == nullptr) || (GraphUtils::AddEdge(i_node->GetOutDataAnchor(0), identity_node->GetInDataAnchor(0)) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed " + "when ControlTriggerPass %s", i_node->GetName().c_str(), i_node->GetType().c_str(), + identity_node->GetName().c_str(), identity_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "TranWhileInfo failed: Add data-edge %s:0->%s:0 failed.", i_name.c_str(), identity_name.c_str()); return FAILED; } @@ -240,6 +257,8 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i // Identity node has and only has one output OutDataAnchorPtr i_input = identity_node->GetOutDataAnchor(0); if (i_input == nullptr) { + REPORT_INNER_ERROR("E19999", "Out data anchor index:0 in op:%s(%s) is nullptr, check invalid when ForPass %s", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "TranWhileInfo failed: i_input is NULL."); return FAILED; } @@ -272,6 +291,7 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i OpDescPtr ForPass::CreateConstDesc(const std::string &name, int32_t value) { OpDescPtr const_op_desc = MakeShared(name, CONSTANT); if (const_op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when ForPass %s", __FUNCTION__); GELOGE(FAILED, "Create op_desc failed, const:%s.", name.c_str()); return nullptr; } @@ -279,16 +299,21 @@ OpDescPtr ForPass::CreateConstDesc(const std::string &name, int32_t value) { GeTensorDesc data_desc(GeShape(), FORMAT_ND, DT_INT32); GeTensorPtr const_value = MakeShared(data_desc, reinterpret_cast(&value), sizeof(int32_t)); if (const_value == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when ForPass %s", __FUNCTION__); GELOGE(FAILED, "Create tensor failed, const:%s.", name.c_str()); return nullptr; } if (!AttrUtils::SetTensor(const_op_desc, ATTR_NAME_WEIGHTS, const_value)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when ForPass %s", ATTR_NAME_WEIGHTS.c_str(), + const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Set ATTR_NAME_WEIGHTS failed, const:%s.", name.c_str()); return nullptr; } if (const_op_desc->AddOutputDesc("y", data_desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed, name:y, when ForPass %s", + const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add output desc failed, const:%s.", name.c_str()); return nullptr; } @@ -334,6 +359,7 @@ Status ForPass::CreateLoopInput(const ComputeGraphPtr &graph, const ForInfo &for graphStatus error_code = GRAPH_SUCCESS; std::string error_msg; if ((graph_builder.Build(error_code, error_msg) == nullptr) || (error_code != GRAPH_SUCCESS)) { + GELOGE(FAILED, "Create loop_count node failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); return FAILED; } @@ -346,6 +372,7 @@ Status ForPass::CreateLoopInput(const ComputeGraphPtr &graph, const ForInfo &for NodePtr abs_delta_node = graph_builder.GetNode(abs_name_0); NodePtr loop_count_node = graph_builder.GetNode(abs_name_1); if ((abs_delta_node == nullptr) || (loop_count_node == nullptr)) { + GELOGE(FAILED, "Create loop node failed: node is NULL."); return FAILED; } @@ -431,11 +458,15 @@ Status ForPass::InsertWhileNode(const ComputeGraphPtr &graph, const std::string OpDescBuilder op_desc_builder(name, WHILE); OpDescPtr op_desc = op_desc_builder.AddDynamicInput("input", arg_num).AddDynamicOutput("output", arg_num).Build(); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "Add dynamic input or output to op:%s(%s) failed when FoldingPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Create while op_desc failed, name:%s.", name.c_str()); return FAILED; } NodePtr while_node = graph->AddNode(op_desc); if (while_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when ControlTriggerPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Create while node failed, name:%s.", name.c_str()); return FAILED; } @@ -553,6 +584,7 @@ ComputeGraphPtr ForPass::BuildCondGraph(WhileInfo &while_info) { std::string error_msg; ComputeGraphPtr cond_graph = graph_builder.Build(error_code, error_msg); if (cond_graph == nullptr) { + REPORT_CALL_ERROR("E19999", "Build graph:%s failed when ForPass %s", cond_name.c_str(), __FUNCTION__); GELOGE(FAILED, "Build cond_graph failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); return nullptr; } @@ -667,6 +699,8 @@ OpDescPtr ForPass::CreateSubgraphOpDesc(const std::string &name, uint32_t input_ OpDescPtr op_desc = op_desc_builder.Build(); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "Build op_desc:%s(%s) failed when ForPass %s", + name.c_str(), PARTITIONEDCALL, __FUNCTION__); GELOGE(FAILED, "Create op_desc for subgraph node failed, name:%s.", name.c_str()); return nullptr; } diff --git a/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc b/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc index ab8fc39b..641bba1e 100644 --- a/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc +++ b/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc @@ -34,6 +34,8 @@ using std::string; namespace ge { Status FuseDataNodesWithCommonInputPass::Run(ge::ComputeGraphPtr graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when FuseDataNodesWithCommonInputPass %s", + __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "Compute graph is null."); return GE_GRAPH_PARAM_NULLPTR; } @@ -101,12 +103,26 @@ Status FuseDataNodesWithCommonInputPass::FuseDataNodes( first_node->GetName().c_str(), subgraph->GetName().c_str()); // the data node which can be fused has none input(both data and control in) if (GraphUtils::MoveOutCtrlEdges(node, first_node) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Move out control edge from node:%s(%s) to node:%s(%s) failed " + "when FuseDataNodesWithCommonInputPass %s", + node->GetName().c_str(), node->GetType().c_str(), + first_node->GetName().c_str(), first_node->GetType().c_str(), + __FUNCTION__); return FAILED; } if (GraphUtils::ReplaceNodeDataAnchors(first_node, node, {}, {0}) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Replace data edge from node:%s(%s) to node:%s(%s) failed " + "when FuseDataNodesWithCommonInputPass %s", + node->GetName().c_str(), node->GetType().c_str(), + first_node->GetName().c_str(), first_node->GetType().c_str(), + __FUNCTION__); return FAILED; } if (GraphUtils::RemoveNodeWithoutRelink(subgraph, node) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed " + "when FuseDataNodesWithCommonInputPass %s", + node->GetName().c_str(), node->GetType().c_str(), subgraph->GetName().c_str(), + __FUNCTION__); GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/get_original_format_pass.cc b/ge/graph/passes/get_original_format_pass.cc index e743f190..3f51d748 100644 --- a/ge/graph/passes/get_original_format_pass.cc +++ b/ge/graph/passes/get_original_format_pass.cc @@ -51,6 +51,9 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph GE_CHECK_NOTNULL(node_ptr); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(node_ptr->GetOpDesc(), ATTR_NAME_INFERRED_FORMAT, DOMI_TENSOR_RESERVED), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_INFERRED_FORMAT.c_str(), + node_ptr->GetName().c_str(), node_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); return FAILED); } @@ -64,9 +67,15 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph GELOGI("Data node: %s,format :%d", node_ptr->GetName().c_str(), GetLocalOmgContext().format); ori_format = static_cast(GetLocalOmgContext().format); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(desc_ptr, ATTR_NAME_FORMAT, ori_format), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_FORMAT.c_str(), + desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_FORMAT failed"); return FAILED); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(desc_ptr, ATTR_NAME_INFERRED_FORMAT, ori_format), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_INFERRED_FORMAT.c_str(), + desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); return FAILED); continue; @@ -130,6 +139,9 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph if (ignore_pred_format) { GE_IF_BOOL_EXEC(!AttrUtils::SetBool(tmp_op_ptr, ATTR_NAME_IGNORE_PRED_FORMAT, true), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_IGNORE_PRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "remove edge failed"); return FAILED); } @@ -137,9 +149,15 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph // Do not reset ATTR_NAME_FORMAT if it is set in the OpParser. if (!tmp_op_ptr->HasAttr(ATTR_NAME_FORMAT)) { GE_IF_BOOL_EXEC(!AttrUtils::SetInt(tmp_op_ptr, ATTR_NAME_FORMAT, ori_format), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_FORMAT failed"); return FAILED); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(tmp_op_ptr, ATTR_NAME_INFERRED_FORMAT, ori_format), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_INFERRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); return FAILED); } else { @@ -147,6 +165,9 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph GE_RETURN_WITH_LOG_IF_FALSE(AttrUtils::GetInt(tmp_op_ptr, ATTR_NAME_FORMAT, existingFormat), "Get existing_format attr failed"); if (!AttrUtils::SetInt(tmp_op_ptr, ATTR_NAME_INFERRED_FORMAT, existingFormat)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when GetOriginalFormatPass %s", + ATTR_NAME_INFERRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); return FAILED; } diff --git a/ge/graph/passes/global_step_insert_pass.cc b/ge/graph/passes/global_step_insert_pass.cc index 4431fc3d..07541ca0 100755 --- a/ge/graph/passes/global_step_insert_pass.cc +++ b/ge/graph/passes/global_step_insert_pass.cc @@ -34,11 +34,16 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, const std::vector &input_list, const std::vector &output_list) { OpDescPtr op_desc = MakeShared(node_name, node_type); - GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGE(FAILED,"Make OpDesc failed"); return nullptr); + GE_IF_BOOL_EXEC(op_desc == nullptr, + REPORT_CALL_ERROR("E19999", "New OpDesc failed when GlobalStepInsertPass %s", __FUNCTION__); + GELOGE(FAILED,"Make OpDesc failed"); + return nullptr); for (auto &input_desc : input_list) { graphStatus graph_status = op_desc->AddInputDesc(input_desc); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when GlobalStepInsertPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add node:%s intput desc failed, error=%u.", node_name.c_str(), graph_status); return nullptr; } @@ -47,6 +52,8 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, for (auto &output_desc : output_list) { graphStatus graph_status = op_desc->AddOutputDesc(output_desc); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed when GlobalStepInsertPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add node:%s output desc failed, error=%u.", node_name.c_str(), graph_status); return nullptr; } @@ -55,6 +62,9 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, GE_IF_BOOL_EXEC(compute_graph == nullptr, GELOGE(FAILED,"compute_graph is nullptr"); return nullptr); NodePtr node = compute_graph->AddNode(op_desc); GE_IF_BOOL_EXEC(node == nullptr, + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when GlobalStepInsertPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str(), + __FUNCTION__); GELOGE(FAILED, "add node failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); return nullptr); @@ -93,6 +103,9 @@ Status GlobalStepInsertPass::Run(ComputeGraphPtr compute_graph) { // add ctrl edges graphStatus add_ret = GraphUtils::AddEdge(global_step->GetOutControlAnchor(), output_node->GetInControlAnchor()); if (add_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed when ControlTriggerPass %s", + global_step->GetName().c_str(), global_step->GetType().c_str(), + output_node->GetName().c_str(), output_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Add global_step to netoutput edge failed, add_ret=%u.", add_ret); return FAILED; } diff --git a/ge/graph/passes/guarantee_const_pass.cc b/ge/graph/passes/guarantee_const_pass.cc index a2d8f262..ffccb0d5 100644 --- a/ge/graph/passes/guarantee_const_pass.cc +++ b/ge/graph/passes/guarantee_const_pass.cc @@ -24,6 +24,7 @@ #include "graph/common/omg_util.h" #include "graph/utils/attr_utils.h" #include "graph/utils/graph_utils.h" +#include "graph/utils/type_utils.h" namespace ge { namespace { @@ -35,6 +36,8 @@ Status GuaranteeConstPass::Run(NodePtr &node) { string type; Status status_ret = GetOriginalType(node, type); if (status_ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get original type for node:%s failed when GuaranteeConstPass %s", + node->GetName().c_str(), __FUNCTION__); GELOGE(status_ret, "GuaranteeConstPass get original type fail."); return status_ret; } @@ -42,6 +45,9 @@ Status GuaranteeConstPass::Run(NodePtr &node) { return SUCCESS; } if (node->GetOpDesc()->GetAllInputsDesc().size() != kGuaranteeConstInputsSize) { + REPORT_CALL_ERROR("E19999", "Num:%zu of input desc node:%s(%s) not equal to %u, " + "check invalid when GuaranteeConstPass %s", node->GetOpDesc()->GetAllInputsDesc().size(), + node->GetName().c_str(), node->GetType().c_str(), kGuaranteeConstInputsSize, __FUNCTION__); GELOGE(PARAM_INVALID, "input size error. Input size:%zu", node->GetOpDesc()->GetAllInputsDesc().size()); return PARAM_INVALID; } @@ -51,6 +57,11 @@ Status GuaranteeConstPass::Run(NodePtr &node) { // Input tensor cannot be a resource variable handle. const DataType &input_dtype = in_desc->GetDataType(); if (input_dtype == DT_RESOURCE) { + REPORT_CALL_ERROR("E19999", + "Data type:%s of op:%s(%s) input 0 not equal to %s, check invalid when GuaranteeConstPass %s", + TypeUtils::DataTypeToSerialString(input_dtype).c_str(), + node->GetName().c_str(), node->GetType().c_str(), + TypeUtils::DataTypeToSerialString(DT_RESOURCE).c_str(), __FUNCTION__); GELOGE(FAILED, "Input tensor cannot be a resource variable handle in [%s].", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/hccl_continuous_memcpy_pass.cc b/ge/graph/passes/hccl_continuous_memcpy_pass.cc index 1931baf0..26de838e 100644 --- a/ge/graph/passes/hccl_continuous_memcpy_pass.cc +++ b/ge/graph/passes/hccl_continuous_memcpy_pass.cc @@ -36,6 +36,8 @@ Status HcclContinuousMemcpyPass::Run(ge::ComputeGraphPtr graph) { for (const auto &node : graph->GetDirectNode()) { auto op_desc = node->GetOpDesc(); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Node with nullptr op_desc exist in Param graph:%s, check invalid " + "when HcclContinuousMemcpyPass %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "node has no op_desc, node_name : %s.", node->GetName().c_str()); return INTERNAL_ERROR; } @@ -76,6 +78,9 @@ Status HcclContinuousMemcpyPass::ContinuousInputProcess(const ComputeGraphPtr &g } auto src_out_anchor = hccl_in_anchor->GetPeerOutAnchor(); if (src_out_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) input:%d anchor, peer anchor is nullptr, check invalid " + "when HcclContinuousMemcpyPass %s", node->GetName().c_str(), node->GetType().c_str(), + hccl_in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "hcom op input has no peer anchor, node_name:%s", node->GetName().c_str()); return INTERNAL_ERROR; } @@ -115,6 +120,9 @@ Status HcclContinuousMemcpyPass::P2pmemInputProcess(const ComputeGraphPtr &graph } auto src_out_anchor = hccl_in_anchor->GetPeerOutAnchor(); if (src_out_anchor == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) input:%u anchor, peer anchor is nullptr, check invalid " + "when HcclContinuousMemcpyPass %s", node->GetName().c_str(), node->GetType().c_str(), + index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "hcom op input has no peer anchor, node_name:%s", node->GetName().c_str()); return INTERNAL_ERROR; } @@ -146,6 +154,8 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap NodePtr pre_node = out_data_anchor->GetOwnerNode(); OpDescPtr pre_op_desc = pre_node->GetOpDesc(); if (pre_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when HcclContinuousMemcpyPass %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "OpDesc of pre node is invalid."); return nullptr; } @@ -154,6 +164,7 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap node_name = CheckDuplicateName(node_name); OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when HcclContinuousMemcpyPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); return nullptr; } @@ -161,12 +172,16 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); return nullptr; } ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); return nullptr; } @@ -175,6 +190,8 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap NodePtr memcpy_node = graph->AddNode(op_desc); if (memcpy_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } @@ -247,6 +264,14 @@ Status HcclContinuousMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); if (ret1 != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed " + "when HcclContinuousMemcpyPass %s", + src_out_anchor->GetOwnerNode()->GetName().c_str(), + src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; @@ -255,6 +280,13 @@ Status HcclContinuousMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr GE_CHECK_NOTNULL(out_data_anchor_0); ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); if (ret1 != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed when HcclContinuousMemcpyPass %s", + out_data_anchor_0->GetOwnerNode()->GetName().c_str(), + out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; @@ -262,6 +294,12 @@ Status HcclContinuousMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclContinuousMemcpyPass %s", + src_out_anchor->GetOwnerNode()->GetName().c_str(), + src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), + memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), + kAnchorNum, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), memcpy_node->GetName().c_str()); return FAILED; @@ -307,6 +345,12 @@ Status HcclContinuousMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeG Status ret = hccl_out_anchor->LinkTo(assign_node->GetInDataAnchor(kAnchorAssignValueIndex)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclContinuousMemcpyPass %s", + hccl_out_anchor->GetOwnerNode()->GetName().c_str(), + hccl_out_anchor->GetOwnerNode()->GetType().c_str(), hccl_out_anchor->GetIdx(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), + kAnchorAssignValueIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", hccl_out_anchor->GetOwnerNode()->GetName().c_str(), assign_node->GetName().c_str()); return FAILED; @@ -314,6 +358,12 @@ Status HcclContinuousMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeG ret = var_out_anchor->LinkTo(assign_node->GetInDataAnchor(kAnchorAssignRefIndex)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclContinuousMemcpyPass %s", + var_out_anchor->GetOwnerNode()->GetName().c_str(), + var_out_anchor->GetOwnerNode()->GetType().c_str(), var_out_anchor->GetIdx(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), + kAnchorAssignRefIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", var_out_anchor->GetOwnerNode()->GetName().c_str(), assign_node->GetName().c_str()); return FAILED; @@ -328,7 +378,12 @@ Status HcclContinuousMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeG continue; } ret = assign_out_control_anchor->LinkTo(in_data_anchor->GetOwnerNode()->GetInControlAnchor()); - if (ret != SUCCESS) { + if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Op:%s(%s) link control to op:%s(%s) failed when HcclContinuousMemcpyPass %s", + assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), + assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), + in_data_anchor->GetOwnerNode()->GetName().c_str(), + in_data_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link control anchor %s fail.", assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), in_data_anchor->GetOwnerNode()->GetName().c_str()); @@ -342,6 +397,11 @@ Status HcclContinuousMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeG } ret = assign_out_control_anchor->LinkTo(in_control_anchor); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Op:%s(%s) link control to op:%s(%s) failed when HcclContinuousMemcpyPass %s", + assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), + assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), + in_control_anchor->GetOwnerNode()->GetName().c_str(), + in_control_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link control anchor %s fail.", assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), in_control_anchor->GetOwnerNode()->GetName().c_str()); @@ -363,6 +423,8 @@ NodePtr HcclContinuousMemcpyPass::CreateAssignNode(const ComputeGraphPtr &graph, NodePtr pre_node = out_data_anchor->GetOwnerNode(); OpDescPtr pre_op_desc = pre_node->GetOpDesc(); if (pre_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when HcclContinuousMemcpyPass %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "OpDesc of pre node is invalid."); return nullptr; } @@ -371,36 +433,48 @@ NodePtr HcclContinuousMemcpyPass::CreateAssignNode(const ComputeGraphPtr &graph, node_name = CheckDuplicateName(node_name); OpDescPtr op_desc = MakeShared(node_name.c_str(), ASSIGN); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when HcclContinuousMemcpyPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: MakeShared op_desc fail."); return nullptr; } GELOGI("Create Assign op:%s.", op_desc->GetName().c_str()); if (!AttrUtils::SetBool(op_desc, ATTR_NEED_COMPILE, true)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when HcclContinuousMemcpyPass %s", + ATTR_NEED_COMPILE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Set ATTR_NEED_COMPILE Attr for node:%s fail.", op_desc->GetName().c_str()); return nullptr; } graphStatus ret = op_desc->AddInputDesc("ref", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:ref, when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add ref input desc fail."); return nullptr; } ret = op_desc->AddInputDesc("value", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:value, when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add value input desc fail."); return nullptr; } ret = op_desc->AddOutputDesc("ref", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, name:ref, when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add output desc fail."); return nullptr; } NodePtr assign_node = graph->AddNode(op_desc); if (assign_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when HcclContinuousMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } diff --git a/ge/graph/passes/hccl_memcpy_pass.cc b/ge/graph/passes/hccl_memcpy_pass.cc index 537920b7..1f3ba786 100755 --- a/ge/graph/passes/hccl_memcpy_pass.cc +++ b/ge/graph/passes/hccl_memcpy_pass.cc @@ -38,6 +38,8 @@ Status HcclMemcpyPass::Run(ge::ComputeGraphPtr graph) { for (const auto &node : graph->GetDirectNode()) { auto op_desc = node->GetOpDesc(); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Node with nullptr op_desc exist in Param graph:%s, check invalid " + "when HcclMemcpyPass %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "node has no op_desc, node_name : %s.", node->GetName().c_str()); return INTERNAL_ERROR; } @@ -64,6 +66,8 @@ Status HcclMemcpyPass::MutableInputProcess(const ComputeGraphPtr &graph, const N } if (!AttrUtils::GetBool(op_desc, kInputMutable, node_input_mutable)) { + REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed when HcclMemcpyPass %s", kInputMutable, + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "node:%s get attr:_input_mutable failed.", node->GetName().c_str()); return FAILED; } @@ -116,6 +120,7 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O NodePtr pre_node = out_data_anchor->GetOwnerNode(); OpDescPtr pre_op_desc = pre_node->GetOpDesc(); if (pre_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when HcclMemcpyPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "OpDesc of pre node is invalid."); return nullptr; } @@ -124,6 +129,7 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O node_name = CheckDuplicateName(node_name); OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when HcclMemcpyPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); return nullptr; } @@ -131,12 +137,16 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:x, when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); return nullptr; } ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, name:y, when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); return nullptr; } @@ -145,6 +155,8 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O NodePtr memcpy_node = graph->AddNode(op_desc); if (memcpy_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } @@ -215,6 +227,7 @@ Status HcclMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, co Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); if (ret1 != SUCCESS) { + GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; @@ -223,6 +236,13 @@ Status HcclMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, co GE_CHECK_NOTNULL(out_data_anchor_0); ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); if (ret1 != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed when HcclMemcpyPass %s", + out_data_anchor_0->GetOwnerNode()->GetName().c_str(), + out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; @@ -230,6 +250,12 @@ Status HcclMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, co Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclMemcpyPass %s", + src_out_anchor->GetOwnerNode()->GetName().c_str(), + src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), + memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), + kAnchorNum, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), memcpy_node->GetName().c_str()); return FAILED; @@ -275,6 +301,12 @@ Status HcclMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeGraphPtr &g Status ret = hccl_out_anchor->LinkTo(assign_node->GetInDataAnchor(kAnchorAssignValueIndex)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclMemcpyPass %s", + hccl_out_anchor->GetOwnerNode()->GetName().c_str(), + hccl_out_anchor->GetOwnerNode()->GetType().c_str(), hccl_out_anchor->GetIdx(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), + kAnchorAssignValueIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", hccl_out_anchor->GetOwnerNode()->GetName().c_str(), assign_node->GetName().c_str()); return FAILED; @@ -282,6 +314,12 @@ Status HcclMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeGraphPtr &g ret = var_out_anchor->LinkTo(assign_node->GetInDataAnchor(kAnchorAssignRefIndex)); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed when HcclMemcpyPass %s", + var_out_anchor->GetOwnerNode()->GetName().c_str(), + var_out_anchor->GetOwnerNode()->GetType().c_str(), var_out_anchor->GetIdx(), + assign_node->GetName().c_str(), assign_node->GetType().c_str(), + kAnchorAssignRefIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", var_out_anchor->GetOwnerNode()->GetName().c_str(), assign_node->GetName().c_str()); return FAILED; @@ -296,7 +334,14 @@ Status HcclMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeGraphPtr &g continue; } ret = assign_out_control_anchor->LinkTo(in_data_anchor->GetOwnerNode()->GetInControlAnchor()); - if (ret != SUCCESS) { + if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", + "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed when HcclMemcpyPass %s", + assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), + assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), assign_out_control_anchor->GetIdx(), + in_data_anchor->GetOwnerNode()->GetName().c_str(), + in_data_anchor->GetOwnerNode()->GetType().c_str(), + in_data_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link control anchor %s fail.", assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), in_data_anchor->GetOwnerNode()->GetName().c_str()); @@ -310,6 +355,11 @@ Status HcclMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeGraphPtr &g } ret = assign_out_control_anchor->LinkTo(in_control_anchor); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Op:%s(%s) link control to op:%s(%s) failed when HcclMemcpyPass %s", + assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), + assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), + in_control_anchor->GetOwnerNode()->GetName().c_str(), + in_control_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "The op %s link control anchor %s fail.", assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), in_control_anchor->GetOwnerNode()->GetName().c_str()); @@ -330,6 +380,8 @@ NodePtr HcclMemcpyPass::CreateAssignNode(const ComputeGraphPtr &graph, const Out NodePtr pre_node = out_data_anchor->GetOwnerNode(); OpDescPtr pre_op_desc = pre_node->GetOpDesc(); if (pre_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when HcclMemcpyPass %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "OpDesc of pre node is invalid."); return nullptr; } @@ -338,6 +390,7 @@ NodePtr HcclMemcpyPass::CreateAssignNode(const ComputeGraphPtr &graph, const Out node_name = CheckDuplicateName(node_name); OpDescPtr op_desc = MakeShared(node_name.c_str(), ASSIGN); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when HcclMemcpyPass %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: MakeShared op_desc fail."); return nullptr; } @@ -345,24 +398,32 @@ NodePtr HcclMemcpyPass::CreateAssignNode(const ComputeGraphPtr &graph, const Out graphStatus ret = op_desc->AddInputDesc("ref", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:ref, when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add ref input desc fail."); return nullptr; } ret = op_desc->AddInputDesc("value", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:value, when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add value input desc fail."); return nullptr; } ret = op_desc->AddOutputDesc("ref", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, name:ref, when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create Assign op: add output desc fail."); return nullptr; } NodePtr assign_node = graph->AddNode(op_desc); if (assign_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when HcclMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } diff --git a/ge/graph/passes/identity_pass.cc b/ge/graph/passes/identity_pass.cc index 5a54e391..269f2993 100755 --- a/ge/graph/passes/identity_pass.cc +++ b/ge/graph/passes/identity_pass.cc @@ -99,6 +99,8 @@ Status IdentityPass::Run(NodePtr &node) { string type; Status status_ret = GetOriginalType(node, type); if (status_ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get original type for node:%s failed when IdentityPass %s", + node->GetName().c_str(), __FUNCTION__); GELOGE(status_ret, "Identity pass get original type fail."); return status_ret; } @@ -118,6 +120,9 @@ Status IdentityPass::Run(NodePtr &node) { } size_t n = node->GetOpDesc()->GetOutputsSize(); if (node->GetOpDesc()->GetInputsSize() != n) { + REPORT_CALL_ERROR("E19999", "Num:%zu of input desc node:%s(%s) not equal to it's output desc num:%zu, " + "check invalid when GuaranteeConstPass %s", node->GetOpDesc()->GetInputsSize(), + node->GetName().c_str(), node->GetType().c_str(), n, __FUNCTION__); GELOGE(PARAM_INVALID, "Identity input / output size must be equal. in size:%lu, out size:%lu", node->GetOpDesc()->GetInputsSize(), n); return PARAM_INVALID; diff --git a/ge/graph/passes/infershape_pass.cc b/ge/graph/passes/infershape_pass.cc index b9a98f62..728f5512 100755 --- a/ge/graph/passes/infershape_pass.cc +++ b/ge/graph/passes/infershape_pass.cc @@ -22,8 +22,58 @@ #include "framework/common/util.h" #include "graph/shape_refiner.h" #include "graph/utils/graph_utils.h" +#include "utils/tensor_utils.h" +#include "utils/type_utils.h" namespace ge { + +void SerialShapeRange(const GeTensorDescPtr &desc, std::string &desc_str) { + desc_str += "["; + std::vector> shape_range; + (void)desc->GetShapeRange(shape_range); + for (const auto &pair : shape_range) { + desc_str += "{"; + desc_str += std::to_string(pair.first) + "," + std::to_string(pair.second); + desc_str += "},"; + } + desc_str += "]"; + shape_range.clear(); + (void)desc->GetOriginShapeRange(shape_range); + for (const auto &pair : shape_range) { + desc_str += ",{"; + desc_str += std::to_string(pair.first) + "," + std::to_string(pair.second); + desc_str += "},"; + } +} + +std::string GetInTensorInfoWithString(const ge::NodePtr &node) { + ge::OpDescPtr op_desc = node->GetOpDesc(); + std::stringstream ss; + ss << "{"; + int32_t in_idx = 0; + for (const auto &input_desc : op_desc->GetAllInputsDescPtr()) { + if (input_desc == nullptr) { + in_idx++; + continue; + } + if (in_idx > 0) { + ss << " "; + } + ss << "input_" << in_idx << " " << "tensor: ["; + ss << "(shape:[" << input_desc->MutableShape().ToString() << "]),"; + ss << "(format:" << TypeUtils::FormatToSerialString(input_desc->GetFormat()) << "),"; + ss << "(dtype:" << TypeUtils::DataTypeToSerialString(input_desc->GetDataType()) << "),"; + ss << "(origin_shape:" << input_desc->GetOriginShape().ToString() << "),"; + ss << "(origin_format:" << TypeUtils::FormatToSerialString(input_desc->GetOriginFormat()) << "),"; + ss << "(origin_dtype:" << TypeUtils::DataTypeToSerialString(input_desc->GetOriginDataType()) << "),"; + string range_str; + SerialShapeRange(input_desc, range_str); + ss << "(shape_range:" << range_str << ")]"; + in_idx++; + } + return ss.str(); +} + Status InferShapePass::Run(NodePtr &node) { // kOptimizeAfterSubGraph exist means after subgraph auto ret = ShapeRefiner::InferShapeAndType(node, !OptionExists(kOptimizeAfterSubGraph)); @@ -39,6 +89,8 @@ Status InferShapePass::Run(NodePtr &node) { (void)Analyzer::GetInstance()->SaveAnalyzerDataToFile(root_graph->GetSessionID(), root_graph->GetGraphID()); + REPORT_CALL_ERROR("E19999", "Call InferShapeAndType for node:%s(%s) failed, input_tensor:%s", + node->GetName().c_str(), node->GetType().c_str(), GetInTensorInfoWithString(node).c_str()); GELOGE(GE_GRAPH_INFERSHAPE_FAILED, "infershape failed. node: %s", node->GetName().c_str()); return GE_GRAPH_INFERSHAPE_FAILED; } diff --git a/ge/graph/passes/inplace_support_check_pass.cc b/ge/graph/passes/inplace_support_check_pass.cc index 44a0b3ef..432695fa 100644 --- a/ge/graph/passes/inplace_support_check_pass.cc +++ b/ge/graph/passes/inplace_support_check_pass.cc @@ -69,6 +69,9 @@ Status InplaceSupportCheckPass::Run(NodePtr &node) { GELOGD("add attr INPLACE_SUPPORT_INPUT_INDEX on node %s, input_idx=%d", node->GetName().c_str(), inplace_input_idx); if (!AttrUtils::SetInt(node->GetOpDesc()->MutableOutputDesc(kInplaceSupportOutputIndex), INPLACE_SUPPORT_INPUT_INDEX, inplace_input_idx)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to output:%u tensor of op:%s(%s) failed when InplaceSupportCheckPass %s", + INPLACE_SUPPORT_INPUT_INDEX.c_str(), kInplaceSupportOutputIndex, + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Set attr INPLACE_SUPPORT_INPUT_INDEX on node %s failed.", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/input_output_connection_identify_pass.cc b/ge/graph/passes/input_output_connection_identify_pass.cc index 0d198dfb..b9bc047e 100644 --- a/ge/graph/passes/input_output_connection_identify_pass.cc +++ b/ge/graph/passes/input_output_connection_identify_pass.cc @@ -42,11 +42,15 @@ inline bool IsDataOp(const std::string &node_type) { Status InputOutputConnectionIdentifyPass::Run(ComputeGraphPtr graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when InputOutputConnectionIdentifyPass %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "Input param graph is null, skip identification of nodes that connect to input and output."); return PARAM_INVALID; } if (graph->GetParentGraph() != nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph's parent graph is nullptr, " + "check invalid when InputOutputConnectionIdentifyPass %s", __FUNCTION__); GELOGD("Current graph %s is a subgraph, skip identification of nodes that connect to input and output.", graph->GetName().c_str()); return SUCCESS; @@ -54,11 +58,15 @@ Status InputOutputConnectionIdentifyPass::Run(ComputeGraphPtr graph) { GELOGD("Start to identify nodes that connect to input and output."); if (graph->TopologicalSorting() != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Topological Sorting graph:%s failed when InputOutputConnectionIdentifyPass %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Graph topological sort failed."); return INTERNAL_ERROR; } if (GraphUtils::GetRefMapping(graph, symbol_to_anchors_, anchor_to_symbol_) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get ref mapping from graph:%s failed when InputOutputConnectionIdentifyPass %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get ref-mapping for graph %s failed.", graph->GetName().c_str()); return INTERNAL_ERROR; } @@ -125,6 +133,8 @@ Status InputOutputConnectionIdentifyPass::UpdateNodeIdxMap(const string &symbol_ map> &connect_output_node_idx) { auto symbol_iter = symbol_to_anchors_.find(symbol_string); if (symbol_iter == symbol_to_anchors_.end()) { + REPORT_CALL_ERROR("E19999", "Can't find symbol:%s in symbol_to_anchors map, check invalid " + "when InputOutputConnectionIdentifyPass %s", symbol_string.c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "Input param symbol string: %s is invalid.", symbol_string.c_str()); return PARAM_INVALID; } @@ -171,6 +181,9 @@ Status InputOutputConnectionIdentifyPass::SetNodeAttrOfConnectingInputOutput( GE_CHECK_NOTNULL(iter.first); if (iter.first->GetOpDesc() != nullptr) { if (!AttrUtils::SetListInt(iter.first->GetOpDesc(), ATTR_NAME_NODE_CONNECT_INPUT, iter.second)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when InputOutputConnectionIdentifyPass %s", + ATTR_NAME_NODE_CONNECT_INPUT.c_str(), + iter.first->GetName().c_str(), iter.first->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to set attr %s for node %s.", ATTR_NAME_NODE_CONNECT_INPUT.c_str(), iter.first->GetName().c_str()); return INTERNAL_ERROR; @@ -182,6 +195,9 @@ Status InputOutputConnectionIdentifyPass::SetNodeAttrOfConnectingInputOutput( GE_CHECK_NOTNULL(iter.first); if (iter.first->GetOpDesc() != nullptr) { if (!AttrUtils::SetListInt(iter.first->GetOpDesc(), ATTR_NAME_NODE_CONNECT_OUTPUT, iter.second)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when InputOutputConnectionIdentifyPass %s", + ATTR_NAME_NODE_CONNECT_OUTPUT.c_str(), + iter.first->GetName().c_str(), iter.first->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to set attr %s for node %s.", ATTR_NAME_NODE_CONNECT_OUTPUT.c_str(), iter.first->GetName().c_str()); return INTERNAL_ERROR; diff --git a/ge/graph/passes/iterator_op_pass.cc b/ge/graph/passes/iterator_op_pass.cc index 1ec2bba9..b1c62168 100644 --- a/ge/graph/passes/iterator_op_pass.cc +++ b/ge/graph/passes/iterator_op_pass.cc @@ -55,20 +55,36 @@ Status IteratorOpPass::Run(ge::ComputeGraphPtr graph) { if (type == "IteratorV2" || type == "Iterator" || op_type == kGetNext) { ge::NodePtr memcpy_async_node = InsertMemcpyAsyncNode(node, graph); GE_CHECK_NOTNULL(memcpy_async_node); - GE_CHK_STATUS_RET(SetCycleEvent(memcpy_async_node), "Set cycle event fail, node:%s", - memcpy_async_node->GetName().c_str()); + auto status = SetCycleEvent(memcpy_async_node); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set cycle event to op:%s(%s) failed when IteratorOpPass %s", + memcpy_async_node->GetName().c_str(), memcpy_async_node->GetType().c_str(), __FUNCTION__); + GELOGE(status, "Set cycle event failed."); + return status; + } - GE_CHK_STATUS_RET(SetStreamLabel(memcpy_async_node, memcpy_async_node->GetName()), - "Set stream label fail, node:%s", node->GetName().c_str()); + status = SetStreamLabel(memcpy_async_node, memcpy_async_node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when IteratorOpPass %s", + memcpy_async_node->GetName().c_str(), memcpy_async_node->GetName().c_str(), + memcpy_async_node->GetType().c_str(), __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } - GE_CHK_STATUS_RET(SetStreamLabel(node, node->GetName()), "Set stream label fail, node:%s", - node->GetName().c_str()); + status = SetStreamLabel(node, node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when IteratorOpPass %s", + node->GetName().c_str(), node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } GELOGI("Set independent loop for iterator node success"); int64_t loop_per_iter = 0; ge::GeTensorDesc ge_tensor_desc; - Status status = VarManager::Instance(graph->GetSessionID())->GetCurVarDesc(NODE_NAME_FLOWCTRL_LOOP_PER_ITER, + status = VarManager::Instance(graph->GetSessionID())->GetCurVarDesc(NODE_NAME_FLOWCTRL_LOOP_PER_ITER, ge_tensor_desc); GE_IF_BOOL_EXEC(status != SUCCESS, GELOGW("Fail to Get var_desc of NODE_NAME_FLOWCTRL_LOOP_PER_ITER failed."); continue); @@ -92,8 +108,14 @@ Status IteratorOpPass::Run(ge::ComputeGraphPtr graph) { if (loop_per_iter == kMaxIterationsPerLoop) { ge::NodePtr end_of_sequence_node = InsertEndOfSequenceNode(node, memcpy_async_node, graph); GE_CHECK_NOTNULL(end_of_sequence_node); - GE_CHK_STATUS_RET(SetStreamLabel(end_of_sequence_node, end_of_sequence_node->GetName()), - "Set stream label fail, node:%s", node->GetName().c_str()); + status = SetStreamLabel(end_of_sequence_node, end_of_sequence_node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed when IteratorOpPass %s", + end_of_sequence_node->GetName().c_str(), end_of_sequence_node->GetName().c_str(), + end_of_sequence_node->GetType().c_str(), __FUNCTION__); + GELOGE(status, "set stream label failed."); + return status; + } GELOGI("Insert EndOfSequence node success."); } } @@ -110,8 +132,13 @@ Status IteratorOpPass::GetVariableValue(uint64_t session_id, const ge::GeTensorD GE_CHECK_NOTNULL(var_mem_base); // offset + logic_base uint8_t *dev_ptr = nullptr; - GE_CHK_STATUS_RET(VarManager::Instance(session_id)->GetVarAddr(var_name, tensor_desc, &dev_ptr), - "Get variable %s address failed.", var_name.c_str()); + auto status = VarManager::Instance(session_id)->GetVarAddr(var_name, tensor_desc, &dev_ptr); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get Var add by name:%s failed, session_id:%lu, when IteratorOpPass %s", + var_name.c_str(), session_id, __FUNCTION__); + GELOGE(status, "Get variable %s address failed.", var_name.c_str()); + return status; + } int64_t offset = static_cast(reinterpret_cast(dev_ptr)); // logic_base_addr auto logic_var_base = VarManager::Instance(session_id)->GetVarMemLogicBase(); @@ -144,7 +171,13 @@ ge::NodePtr IteratorOpPass::InsertEndOfSequenceNode(const ge::NodePtr &pre_node, auto out_anchor = pre_node->GetOutDataAnchor(0); ge::graphStatus status; status = GraphUtils::AddEdge(out_anchor, end_of_seq_node->GetInDataAnchor(0)); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed " + "when IteratorOpPass %s", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), + end_of_seq_node->GetName().c_str(), end_of_seq_node->GetType().c_str(), + __FUNCTION__); + return nullptr, "Graph add EndOfSequence op input edge fail, dst node: %s.", end_of_seq_node->GetName().c_str()); // EOS(control) --> subsequent of memcpy @@ -157,7 +190,12 @@ ge::NodePtr IteratorOpPass::InsertEndOfSequenceNode(const ge::NodePtr &pre_node, continue; } status = GraphUtils::AddEdge(out_ctrl_anchor, in_ctrl_anchor); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when IteratorOpPass %s", + end_of_seq_node->GetName().c_str(), end_of_seq_node->GetType().c_str(), + out_node->GetName().c_str(), out_node->GetType().c_str(), __FUNCTION__); + return nullptr, "Graph add EndOfSequence op out ctrl edge fail, dst node: %s.", out_node->GetName().c_str()); GELOGI("Graph add EndOfSequence op out ctrl edge, dst node: %s.", @@ -175,19 +213,29 @@ ge::NodePtr IteratorOpPass::InsertEndOfSequenceNode(const ge::NodePtr &pre_node, /// ge::OpDescPtr IteratorOpPass::CreateEndOfSequenceOp(const ge::NodePtr &pre_node) { GELOGI("Start to create endOfSequence op."); - GE_CHK_BOOL_EXEC(pre_node != nullptr, return nullptr, "Input param invalid."); + GE_CHK_BOOL_EXEC(pre_node != nullptr, + REPORT_INNER_ERROR("E19999", "Param pre_node is nullptr, check invalid when IteratorOpPass %s", + __FUNCTION__); + return nullptr, "Input param invalid."); string node_name = pre_node->GetName() + "_EndOfSequence"; ge::OpDescPtr op_desc = MakeShared(node_name, ENDOFSEQUENCE); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when IteratorOpPass %s", __FUNCTION__); GELOGE(FAILED, "MakeShared fail."); return op_desc; } ge::OpDescPtr pre_node_op_desc = pre_node->GetOpDesc(); - GE_CHK_BOOL_EXEC(pre_node_op_desc != nullptr, return nullptr, "OpDesc of pre_node is invalid."); + GE_CHK_BOOL_EXEC(pre_node_op_desc != nullptr, + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when IteratorOpPass %s", + __FUNCTION__); + return nullptr, "OpDesc of pre_node is invalid."); GELOGI("Create EndOfSequence op:%s.", op_desc->GetName().c_str()); - GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_node_op_desc->GetOutputDesc(0)) == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_node_op_desc->GetOutputDesc(0)) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when IteratorOpPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); + return nullptr, "Create EndOfSequence op:add input desc fail."); return op_desc; } @@ -205,7 +253,11 @@ ge::NodePtr IteratorOpPass::InsertMemcpyAsyncNode(const ge::NodePtr &pre_node, c ge::OpDescPtr memcpy_async_op_desc = CreateMemcpyAsyncOp(pre_node); GE_CHK_BOOL_EXEC(memcpy_async_op_desc != nullptr, GELOGW("Create memcpyAsync op fail."); return nullptr); ge::NodePtr memcpy_async_node = graph->AddNode(memcpy_async_op_desc); - GE_CHK_BOOL_EXEC(memcpy_async_node != nullptr, return nullptr, "Insert mencpy node fail."); + GE_CHK_BOOL_EXEC(memcpy_async_node != nullptr, + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when IteratorOpPass %s", + memcpy_async_op_desc->GetName().c_str(), memcpy_async_op_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); + return nullptr, "Insert mencpy node fail."); // Data out for (auto &out_anchor : pre_node->GetAllOutDataAnchors()) { @@ -217,9 +269,26 @@ ge::NodePtr IteratorOpPass::InsertMemcpyAsyncNode(const ge::NodePtr &pre_node, c for (auto &peer_in_anchor : out_anchor->GetPeerInDataAnchors()) { GE_IF_BOOL_EXEC(peer_in_anchor == nullptr, GELOGW("peer_in_anchor is nullptr"); return nullptr); status = GraphUtils::RemoveEdge(out_anchor, peer_in_anchor); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, "Remove edge failed, index:%d.", out_anchor->GetIdx()); + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR( + "E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed " + "when IteratorOpPass %s", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), out_anchor->GetIdx(), + peer_in_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetType().c_str(), + peer_in_anchor->GetIdx(), __FUNCTION__); + return nullptr, "Remove edge failed, index:%d.", out_anchor->GetIdx()); status = GraphUtils::AddEdge(memcpy_async_node->GetOutDataAnchor(out_anchor->GetIdx()), peer_in_anchor); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR( + "E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed " + "when IteratorOpPass %s", + memcpy_async_node->GetName().c_str(), memcpy_async_node->GetType().c_str(), + out_anchor->GetIdx(), + peer_in_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetType().c_str(), + peer_in_anchor->GetIdx(), __FUNCTION__); + return nullptr, "Graph add memcpyAsync op out edge fail, src index:%d, dst index:%d, dst node: %s.", out_anchor->GetIdx(), peer_in_anchor->GetIdx(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); @@ -227,7 +296,14 @@ ge::NodePtr IteratorOpPass::InsertMemcpyAsyncNode(const ge::NodePtr &pre_node, c peer_in_anchor->GetIdx(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); } status = GraphUtils::AddEdge(out_anchor, memcpy_async_node->GetInDataAnchor(out_anchor->GetIdx())); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, "Graph add memcpyAsync op in edge fail, index:%d.", + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR( + "E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed " + "when IteratorOpPass %s", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), out_anchor->GetIdx(), + memcpy_async_node->GetName().c_str(), memcpy_async_node->GetType().c_str(), + out_anchor->GetIdx(), __FUNCTION__); + return nullptr, "Graph add memcpyAsync op in edge fail, index:%d.", out_anchor->GetIdx()); } // Control out @@ -235,10 +311,24 @@ ge::NodePtr IteratorOpPass::InsertMemcpyAsyncNode(const ge::NodePtr &pre_node, c GE_IF_BOOL_EXEC(out_ctrl_anchor != nullptr, for (auto &peer_in_ctrl_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { ge::graphStatus status = GraphUtils::RemoveEdge(out_ctrl_anchor, peer_in_ctrl_anchor); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, "Remove edge failed, dst node: %s.", + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR( + "E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed " + "when IteratorOpPass %s", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), + peer_in_ctrl_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_ctrl_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); + return nullptr, "Remove edge failed, dst node: %s.", peer_in_ctrl_anchor->GetOwnerNode()->GetName().c_str()); status = GraphUtils::AddEdge(memcpy_async_node->GetOutControlAnchor(), peer_in_ctrl_anchor); - GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(status == GRAPH_SUCCESS, + REPORT_CALL_ERROR( + "E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when IteratorOpPass %s", + memcpy_async_node->GetName().c_str(), memcpy_async_node->GetType().c_str(), + peer_in_ctrl_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_ctrl_anchor->GetOwnerNode()->GetType().c_str(), __FUNCTION__); + return nullptr, "Graph add memcpyAsync op out ctrl edge fail, dst node: %s.", peer_in_ctrl_anchor->GetOwnerNode()->GetName().c_str()); GELOGI("Graph add memcpyAsync op out ctrl edge, dst node: %s.", @@ -261,20 +351,32 @@ ge::OpDescPtr IteratorOpPass::CreateMemcpyAsyncOp(const ge::NodePtr &pre_node) { string node_name = pre_node->GetName() + "_MemcpyAsync"; ge::OpDescPtr op_desc = MakeShared(node_name.c_str(), MEMCPYASYNC); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when IteratorOpPass %s", __FUNCTION__); GELOGE(FAILED, "MakeShared fail."); return op_desc; } GELOGI("Create memcpyAsync op:%s.", op_desc->GetName().c_str()); ge::OpDescPtr pre_node_op_desc = pre_node->GetOpDesc(); - GE_CHK_BOOL_EXEC(pre_node_op_desc != nullptr, return nullptr, "OpDesc of pre_node is invalid."); + GE_CHK_BOOL_EXEC(pre_node_op_desc != nullptr, + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when IteratorOpPass %s", + __FUNCTION__); + return nullptr, "OpDesc of pre_node is invalid."); size_t out_size = pre_node_op_desc->GetOutputsSize(); GELOGI("Create memcpyAsync op, pre_node out_size: %zu.", out_size); for (size_t i = 0; i < out_size; i++) { - GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_node_op_desc->GetOutputDesc(i)) == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_node_op_desc->GetOutputDesc(i)) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when IteratorOpPass %s", + pre_node_op_desc->GetName().c_str(), pre_node_op_desc->GetType().c_str(), + __FUNCTION__); + return nullptr, "Create memcpyAsync op:add input desc fail."); - GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(pre_node_op_desc->GetOutputDesc(i)) == GRAPH_SUCCESS, return nullptr, + GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(pre_node_op_desc->GetOutputDesc(i)) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed when IteratorOpPass %s", + pre_node_op_desc->GetName().c_str(), pre_node_op_desc->GetType().c_str(), + __FUNCTION__); + return nullptr, "Create memcpyAsync op:add output desc fail."); } diff --git a/ge/graph/passes/link_gen_mask_nodes_pass.cc b/ge/graph/passes/link_gen_mask_nodes_pass.cc index 5c4df98b..f737d947 100755 --- a/ge/graph/passes/link_gen_mask_nodes_pass.cc +++ b/ge/graph/passes/link_gen_mask_nodes_pass.cc @@ -70,6 +70,9 @@ Status LinkGenMaskNodesPass::Run(ComputeGraphPtr graph) { graphStatus status_link_to = src_anchor->LinkTo(dest_anchor); if (status_link_to != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Op:%s(%s) link control to op:%s(%s) failed when LinkGenMaskNodesPass %s", + src_node->GetName().c_str(), src_node->GetType().c_str(), + dest_node->GetName().c_str(), dest_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Link from %s to %s failed.", src_node->GetName().c_str(), dest_node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/mark_agnostic_pass.cc b/ge/graph/passes/mark_agnostic_pass.cc index 80b4bd7a..5b1abddc 100644 --- a/ge/graph/passes/mark_agnostic_pass.cc +++ b/ge/graph/passes/mark_agnostic_pass.cc @@ -132,7 +132,13 @@ Status MarkAgnosticPass::SetContinuousAttr(const NodePtr &node, const std::vecto (void)AttrUtils::SetBool(op_desc, ATTR_NAME_REFRESH_CONTINUOUS_FLAG, true); for (auto index : indexes) { auto out = op_desc->MutableOutputDesc(index); - GE_CHECK_NOTNULL(out); + if (out == nullptr) { + REPORT_INNER_ERROR("E19999", "Op:%s(%s) output:%u desc is nullptr, check invalid when MarkAgnosticPass :%s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, __FUNCTION__); + GELOGE(FAILED, "[Check][Param]Op:%s(%s) output:%u desc is nullptr", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), index); + return FAILED; + } // This attr is for out's dtype and format continuous with it's peer input (void)AttrUtils::SetInt(out, ATTR_NAME_FORMAT_CONTINUOUS, 1); } diff --git a/ge/graph/passes/mark_graph_unknown_status_pass.cc b/ge/graph/passes/mark_graph_unknown_status_pass.cc index ae0919fe..bf69480a 100644 --- a/ge/graph/passes/mark_graph_unknown_status_pass.cc +++ b/ge/graph/passes/mark_graph_unknown_status_pass.cc @@ -28,7 +28,7 @@ Status MarkGraphUnknownStatusPass::Run(ComputeGraphPtr graph) { bool is_unknown_shape = false; bool forced_unknown = false; for (const auto &node : graph->GetDirectNode()) { - GE_CHK_STATUS_RET(ge::NodeUtils::GetNodeUnknownShapeStatus(*node, is_unknown_shape), + GE_CHK_GRAPH_STATUS_RET(ge::NodeUtils::GetNodeUnknownShapeStatus(*node, is_unknown_shape), "Get node[%s] shape status failed!", node->GetName().c_str()); if (is_unknown_shape) { break; diff --git a/ge/graph/passes/memcpy_addr_async_pass.cc b/ge/graph/passes/memcpy_addr_async_pass.cc index b930f7cb..15ddd16c 100755 --- a/ge/graph/passes/memcpy_addr_async_pass.cc +++ b/ge/graph/passes/memcpy_addr_async_pass.cc @@ -33,6 +33,8 @@ Status MemcpyAddrAsyncPass::Run(ComputeGraphPtr graph) { int64_t value = 0; rtError_t rt_ret = rtGetRtCapability(FEATURE_TYPE_MEMCPY, MEMCPY_INFO_SUPPORT_ZEROCOPY, &value); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtGetRtCapability failed, ret = 0x%X, when MemcpyAddrAsyncPass %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "rtGetRtCapability failed, error=0x%x.", rt_ret); return RT_FAILED; } @@ -85,6 +87,9 @@ Status MemcpyAddrAsyncPass::AddMemcpyAddrAsyncNode(const ComputeGraphPtr &graph, } else { uint32_t parent_index = 0; if (!AttrUtils::GetInt(in_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed when MemcpyAddrAsyncPass %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), + in_node->GetName().c_str(), in_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to get parent index of %s", in_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -147,6 +152,9 @@ void MemcpyAddrAsyncPass::FindUserDataForNonDynamic(const ge::NodePtr &parent_no InDataAnchorPtr in_data_anchor = parent_node->GetInDataAnchor(parent_index); OutDataAnchorPtr out_anchor = in_data_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(out_anchor == nullptr, + REPORT_INNER_ERROR("E19999", "Index:%u in data node of op:%s(%s) not exist, check invalid " + "when MemcpyAddrAsyncPass :%s", parent_index, + parent_node->GetName().c_str(), parent_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Cannot find out_anchor of %s.", parent_node->GetName().c_str()); return); NodePtr in_node = out_anchor->GetOwnerNode(); @@ -160,6 +168,9 @@ void MemcpyAddrAsyncPass::FindUserDataForNonDynamic(const ge::NodePtr &parent_no in_anchor_for_known_ = in_data_anchor; NodePtr pre_in_node = in_node->GetOwnerComputeGraph()->GetParentNode(); if (!AttrUtils::GetInt(in_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), + in_node->GetName().c_str(), in_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to refresh parent index of %s", in_node->GetName().c_str()); return; } @@ -176,6 +187,9 @@ void MemcpyAddrAsyncPass::FindUserDataForNonDynamic(const ge::NodePtr &parent_no } } else if (in_node->GetType() == IF || in_node->GetType() == WHILE || in_node->GetType() == CASE) { if (!AttrUtils::GetInt(parent_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_CALL_ERROR("E19999", "Get Attr:%s to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), + parent_node->GetName().c_str(), parent_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to refresh parent index of %s", in_node->GetName().c_str()); return; } @@ -207,18 +221,27 @@ NodePtr MemcpyAddrAsyncPass::CreateMemcpyAddrAsyncNode(const ComputeGraphPtr &gr GELOGD("Start CreateMemcpyAddrAsyncNode."); static uint32_t new_node_index = 0; OpDescPtr pre_op_desc = out_data_anchor->GetOwnerNode()->GetOpDesc(); - GE_CHK_BOOL_EXEC(pre_op_desc != nullptr, return nullptr, "Op_desc of pre node is invalid."); + GE_CHK_BOOL_EXEC(pre_op_desc != nullptr, + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when MemcpyAddrAsyncPass :%s", + __FUNCTION__); + return nullptr, "Op_desc of pre node is invalid."); std::string node_name = pre_op_desc->GetName() + "_" + MEMCPYADDRASYNC + "_" + std::to_string(new_node_index++); OpDescPtr op_desc = MakeShared(node_name, MEMCPYADDRASYNC); - GE_CHECK_NOTNULL_EXEC(op_desc, return nullptr); + GE_CHECK_NOTNULL_EXEC(op_desc, + REPORT_CALL_ERROR("E19999", "New OpDesc failed when MemcpyAddrAsyncPass %s", __FUNCTION__); + return nullptr); if (op_desc->AddInputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + pre_op_desc->GetName().c_str(), pre_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add memcpy_addr_async input desc failed."); return nullptr; } if (op_desc->AddOutputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + pre_op_desc->GetName().c_str(), pre_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add memcpy_addr_async output desc failed."); return nullptr; } @@ -239,11 +262,17 @@ NodePtr MemcpyAddrAsyncPass::CreateMemcpyAddrAsyncNode(const ComputeGraphPtr &gr (void)ge::AttrUtils::GetBool(out_of_user_data->GetOpDesc(), ATTR_NAME_NODE_CONNECT_INPUT, labeled_input); if (labeled_input) { if (!ge::AttrUtils::SetBool(out_of_user_data->GetOpDesc(), ATTR_NAME_NODE_CONNECT_INPUT, false)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + ATTR_NAME_NODE_CONNECT_INPUT.c_str(), + out_of_user_data->GetName().c_str(), out_of_user_data->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to unset attr %s for node %s.", ATTR_NAME_NODE_CONNECT_INPUT.c_str(), out_of_user_data->GetName().c_str()); return nullptr; } if (!ge::AttrUtils::SetBool(op_desc, ATTR_NAME_NODE_CONNECT_INPUT, true)) { + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when MemcpyAddrAsyncPass %s", + ATTR_NAME_NODE_CONNECT_INPUT.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to set attr %s for node %s.", ATTR_NAME_NODE_CONNECT_INPUT.c_str(), op_desc->GetName().c_str()); return nullptr; @@ -251,7 +280,11 @@ NodePtr MemcpyAddrAsyncPass::CreateMemcpyAddrAsyncNode(const ComputeGraphPtr &gr } NodePtr memcpy_addr_async_node = graph->AddNode(op_desc); - GE_CHECK_NOTNULL_EXEC(memcpy_addr_async_node, return nullptr); + GE_CHECK_NOTNULL_EXEC(memcpy_addr_async_node, + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when MemcpyAddrAsyncPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); + return nullptr); return memcpy_addr_async_node; } @@ -260,16 +293,32 @@ Status MemcpyAddrAsyncPass::InsertMemcpyAddrAsyncNode(const OutDataAnchorPtr &ou const InDataAnchorPtr &in_anchor, const NodePtr &node) { // insert memcpy_addr of each user_data and out_of_user_data if (GraphUtils::RemoveEdge(out_anchor, in_anchor) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed " + "when MemcpyAddrAsyncPass %s", + out_anchor->GetOwnerNode()->GetName().c_str(), out_anchor->GetOwnerNode()->GetType().c_str(), + out_anchor->GetIdx(), + in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), + in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Remove edge of %s and %s failed.", out_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } if (GraphUtils::AddEdge(out_anchor, node->GetInDataAnchor(0)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed " + "when MemcpyAddrAsyncPass %s", + out_anchor->GetOwnerNode()->GetName().c_str(), out_anchor->GetOwnerNode()->GetType().c_str(), + out_anchor->GetIdx(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add edge of %s and %s failed.", out_anchor->GetOwnerNode()->GetName().c_str(), node->GetName().c_str()); return INTERNAL_ERROR; } if (GraphUtils::AddEdge(node->GetOutDataAnchor(0), in_anchor) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:0) and op:%s(%s)(index:%d) failed " + "when MemcpyAddrAsyncPass %s", + node->GetName().c_str(), node->GetType().c_str(), + in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), + in_anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add edge of %s and %s failed.", node->GetName().c_str(), in_anchor->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; diff --git a/ge/graph/passes/merge_input_memcpy_pass.cc b/ge/graph/passes/merge_input_memcpy_pass.cc index e8bf0377..6ad42e59 100644 --- a/ge/graph/passes/merge_input_memcpy_pass.cc +++ b/ge/graph/passes/merge_input_memcpy_pass.cc @@ -55,11 +55,12 @@ Status MergeInputMemcpyPass::AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, c const std::string &memcpy_name = node->GetName() + "_input_" + std::to_string(in_data_anchor->GetIdx()); NodePtr memcpy_node = CreateMemcpyAsyncNode(graph, memcpy_name, peer_out_anchor, multi_batch_flag); GE_CHK_BOOL_EXEC(memcpy_node != nullptr, return FAILED, "Create MemcpyAsync node failed."); - GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), "MemcpyAsync node remove edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(peer_out_anchor, memcpy_node->GetInDataAnchor(0)), - "MemcpyAsync node add edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(memcpy_node->GetOutDataAnchor(0), in_data_anchor), - "MemcpyAsync node add edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), + "MemcpyAsync node remove edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(peer_out_anchor, memcpy_node->GetInDataAnchor(0)), + "MemcpyAsync node add edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(memcpy_node->GetOutDataAnchor(0), in_data_anchor), + "MemcpyAsync node add edge failed."); } return SUCCESS; @@ -88,8 +89,12 @@ NodePtr MergeInputMemcpyPass::CreateMemcpyAsyncNode(const ComputeGraphPtr &graph } GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add input to op:%s(%s) failed when MergeInputMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); return nullptr, "Create MemcpyAsync op: add input desc failed."); GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add output to op:%s(%s) failed when MergeInputMemcpyPass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); return nullptr, "Create MemcpyAsync op: add output desc failed."); return graph->AddNode(op_desc); diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index 26d82820..d0b9cc8a 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -35,6 +35,7 @@ const size_t kCaseOneInput = 1; Status MergePass::Run(NodePtr &node) { GELOGD("MergePass running"); if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when MergePass %s", __FUNCTION__); GELOGE(PARAM_INVALID, "param [node] must not be null."); return PARAM_INVALID; } @@ -46,6 +47,8 @@ Status MergePass::Run(NodePtr &node) { } if (node->GetAllOutDataAnchors().empty()) { + REPORT_INNER_ERROR("E19999", "Param node:%s(%s) all data anchor size is 0, check invalid when MergePass %s", + node->GetName().c_str(), node->GetType().c_str() ,__FUNCTION__); GELOGE(PARAM_INVALID, "[%s] Merge node output anchor is empty", node->GetName().c_str()); return PARAM_INVALID; } @@ -79,6 +82,8 @@ Status MergePass::Run(NodePtr &node) { auto in_node = in_data_nodes.at(0); if (IsMergeInputNeedOptimized(in_node)) { if (IsolateAndDeleteNode(in_node, {0}) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed when MergePass %s", + in_node->GetName().c_str(), in_node->GetType().c_str() ,__FUNCTION__); GELOGE(FAILED, "Isolate and delete node %s failed.", in_node->GetName().c_str()); return FAILED; } @@ -115,6 +120,8 @@ Status MergePass::ChangeIndexToConstant(NodePtr &node, int &value_index) { GE_CHECK_NOTNULL(node); ComputeGraphPtr graph = node->GetOwnerComputeGraph(); if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Owner graph of node:%s(%s) is nullptr, check invalid when MergePass %s", + node->GetName().c_str(), node->GetType().c_str() ,__FUNCTION__); GELOGE(FAILED, "[%s] The owner graph must not be null.", node->GetName().c_str()); return FAILED; } @@ -125,11 +132,17 @@ Status MergePass::ChangeIndexToConstant(NodePtr &node, int &value_index) { } NodePtr const_node = graph->AddNode(constant_op_desc); if (const_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when MergePass %s", + constant_op_desc->GetName().c_str(), constant_op_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); return FAILED; } // Change peer in anchors from value_index to new Constant node if (GraphUtils::ReplaceNodeAnchors(const_node, node, {}, {1}) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Replace node:%s(%s) by node:%s(%s) failed when MergePass %s", + node->GetName().c_str(), node->GetType().c_str(), + const_node->GetName().c_str(), const_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "[%s] ReplaceNodeAnchors failed.", node->GetName().c_str()); return FAILED; } @@ -137,6 +150,9 @@ Status MergePass::ChangeIndexToConstant(NodePtr &node, int &value_index) { GE_CHECK_NOTNULL(out_control_anchor); // Add control anchor between Merge and Constant if (out_control_anchor->LinkTo(const_node->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Op:%s(%s) link control to op:%s(%s) failed when MergePass %s", + node->GetName().c_str(), node->GetType().c_str(), + const_node->GetName().c_str(), const_node->GetType().c_str(), __FUNCTION__); return FAILED; } @@ -148,6 +164,7 @@ Status MergePass::CreateConstByValue(NodePtr &node, int value_index, OpDescPtr & // 1. create Constant OpDesc op_desc = MakeShared(constant_name, CONSTANT); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when MergePass %s", __FUNCTION__); GELOGE(FAILED, "[%s] Make shared of Constant op desc failed.", constant_name.c_str()); return FAILED; } @@ -155,6 +172,7 @@ Status MergePass::CreateConstByValue(NodePtr &node, int value_index, OpDescPtr & // 2. get OpDesc of output number one of Merge(value_index) OpDescPtr original_op_desc = node->GetOpDesc(); if (original_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid when MemcpyAddrAsyncPass :%s", __FUNCTION__); GELOGE(FAILED, "[%s] Op desc must not be null.", constant_name.c_str()); return FAILED; } @@ -165,15 +183,19 @@ Status MergePass::CreateConstByValue(NodePtr &node, int value_index, OpDescPtr & GeTensorPtr const_tensor_ptr = MakeShared(original_out_tensor_desc, reinterpret_cast(&value_index), sizeof(int)); if (const_tensor_ptr == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when MergePass %s", __FUNCTION__); GELOGE(FAILED, "[%s] Make shared of Constant tensor failed.", constant_name.c_str()); return FAILED; } GE_IF_BOOL_EXEC(!AttrUtils::SetTensor(op_desc, ATTR_NAME_WEIGHTS, const_tensor_ptr), - GELOGE(FAILED, "get ATTR_NAME_WEIGHTS failed"); return FAILED); + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when MergePass %s", + ATTR_NAME_WEIGHTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); + GELOGE(FAILED, "get ATTR_NAME_WEIGHTS failed"); return FAILED); // 4. set Constant output desc - GE_CHK_STATUS_RET(op_desc->AddOutputDesc(original_out_tensor_desc), "add out put desc failed"); + GE_CHK_GRAPH_STATUS_RET(op_desc->AddOutputDesc(original_out_tensor_desc), "add out put desc failed"); return SUCCESS; } diff --git a/ge/graph/passes/merge_to_stream_merge_pass.cc b/ge/graph/passes/merge_to_stream_merge_pass.cc index c1a57a61..7ee040af 100644 --- a/ge/graph/passes/merge_to_stream_merge_pass.cc +++ b/ge/graph/passes/merge_to_stream_merge_pass.cc @@ -33,15 +33,25 @@ Status MergeToStreamMergePass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(merge_op_desc); if (merge_op_desc->HasAttr(ATTR_INSERT_BY_MBATCH)) { GE_CHK_STATUS_RET(AddActiveNodes(graph, node), "Merge add active node failed."); - GE_CHK_STATUS_RET(SetStreamLabel(node, node->GetName()), "Set stream label failed"); + auto status = SetStreamLabel(node, node->GetName()); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when MergeToStreamMergePass %s", + node->GetName().c_str(), node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } } else { GE_CHK_STATUS_RET(ReplaceMergeNode(graph, node), "Add StreamMerge node failed."); } } for (const auto &node : bypass_nodes_) { - GE_CHK_BOOL_EXEC(GraphUtils::RemoveNodeWithoutRelink(graph, node) == GRAPH_SUCCESS, return FAILED, - "Remove merge node failed."); + GE_CHK_STATUS_EXEC(GraphUtils::RemoveNodeWithoutRelink(graph, node) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed " + "when MergeToStreamMergePass %s", node->GetName().c_str(), + node->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); + return FAILED, + "Remove merge node failed."); } GELOGD("MergeToStreamMergePass Leave"); @@ -62,28 +72,40 @@ Status MergeToStreamMergePass::ReplaceMergeNode(const ComputeGraphPtr &graph, co GELOGI("Create StreamMerge Op, name=%s.", node_name.c_str()); OpDescPtr op_desc = MakeShared(node_name, STREAMMERGE); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when MergeToStreamMergePass %s", __FUNCTION__); GELOGE(FAILED, "Create op_desc failed, StreamMerge:%s.", node_name.c_str()); return FAILED; } for (const InDataAnchorPtr &in_anchor : merge_node->GetAllInDataAnchors()) { GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(merge_op_desc->GetInputDesc(in_anchor->GetIdx())) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed when MergeToStreamMergePass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); return FAILED, "Create StreamMerge op: add input desc failed."); } for (const OutDataAnchorPtr &out_anchor : merge_node->GetAllOutDataAnchors()) { GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(merge_op_desc->GetOutputDesc(out_anchor->GetIdx())) == GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed when MergeToStreamMergePass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); return FAILED, "Create StreamMerge op: add output desc failed."); } NodePtr stream_merge = graph->AddNode(op_desc); - GE_CHK_BOOL_EXEC(stream_merge != nullptr, return FAILED, "Insert StreamMerge node failed."); + GE_CHK_BOOL_EXEC(stream_merge != nullptr, + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when MergeToStreamMergePass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); + return FAILED, "Insert StreamMerge node failed."); GE_CHK_STATUS_RET(MoveEdges(merge_node, stream_merge), "Move edges failed."); bypass_nodes_.insert(merge_node); if (merge_op_desc->HasAttr(ATTR_NAME_NEXT_ITERATION)) { std::string next_iteration_name; GE_IF_BOOL_EXEC(!AttrUtils::GetStr(merge_op_desc, ATTR_NAME_NEXT_ITERATION, next_iteration_name), + REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed when MergeToStreamMergePass %s", + ATTR_NAME_NEXT_ITERATION.c_str(), + merge_op_desc->GetName().c_str(), merge_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get ATTR_NAME_NEXT_ITERATION failed"); return INTERNAL_ERROR); GE_CHK_STATUS_RET(SetNextIteration(stream_merge, next_iteration_name), "Set next iteration failed"); @@ -99,7 +121,9 @@ Status MergeToStreamMergePass::ReplaceMergeNode(const ComputeGraphPtr &graph, co /// @return Status /// Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, const NodePtr &node) { - GE_CHK_BOOL_EXEC(node != nullptr, return FAILED, "Param of pre node is null."); + GE_CHK_BOOL_EXEC(node != nullptr, + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when MergeToStreamMergePass %s", __FUNCTION__); + return FAILED, "Param of pre node is null."); for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); @@ -132,13 +156,21 @@ NodePtr MergeToStreamMergePass::CreateActiveNode(const ComputeGraphPtr &graph, c GELOGI("Create StreamActive op:%s.", node_name.c_str()); OpDescPtr op_desc = MakeShared(node_name, STREAMACTIVE); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when MergeToStreamMergePass %s", __FUNCTION__); GELOGE(FAILED, "Create op_desc failed, StreamActive:%s.", node_name.c_str()); return nullptr; } NodePtr active_node = graph->AddNode(op_desc); - GE_CHK_BOOL_EXEC(active_node != nullptr, return nullptr, "Create StreamActive node failed."); + GE_CHK_BOOL_EXEC(active_node != nullptr, + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when MergeToStreamMergePass %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), + __FUNCTION__); + return nullptr, "Create StreamActive node failed."); GE_IF_BOOL_EXEC(GraphUtils::AddEdge(node->GetOutControlAnchor(), active_node->GetInControlAnchor()) != SUCCESS, + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed " + "when MergeToStreamMergePass %s", node->GetName().c_str(), node->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "add edge failed"); return nullptr); GE_IF_BOOL_EXEC(SetSwitchBranchNodeLabel(active_node, node_name) != SUCCESS, @@ -159,31 +191,35 @@ Status MergeToStreamMergePass::MoveEdges(const NodePtr &old_node, const NodePtr OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); - GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), "Merge remove in data edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(peer_out_anchor, new_node->GetInDataAnchor(in_data_anchor->GetIdx())), - "StreamMerge add in data edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), + "Merge remove in data edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(peer_out_anchor, new_node->GetInDataAnchor(in_data_anchor->GetIdx())), + "StreamMerge add in data edge failed."); } for (const OutDataAnchorPtr &out_data_anchor : old_node->GetAllOutDataAnchors()) { for (const InDataAnchorPtr &peer_in_anchor : out_data_anchor->GetPeerInDataAnchors()) { - GE_CHK_STATUS(GraphUtils::RemoveEdge(out_data_anchor, peer_in_anchor), "Merge remove out data edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(new_node->GetOutDataAnchor(out_data_anchor->GetIdx()), peer_in_anchor), - "StreamMerge add out data edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveEdge(out_data_anchor, peer_in_anchor), + "Merge remove out data edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(new_node->GetOutDataAnchor(out_data_anchor->GetIdx()), + peer_in_anchor), + "StreamMerge add out data edge failed."); } } for (const NodePtr &in_ctrl_node : old_node->GetInControlNodes()) { - GE_CHK_STATUS(GraphUtils::RemoveEdge(in_ctrl_node->GetOutControlAnchor(), old_node->GetInControlAnchor()), - "Merge remove in ctrl edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(in_ctrl_node->GetOutControlAnchor(), new_node->GetInControlAnchor()), - "StreamMerge add in ctrl edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveEdge(in_ctrl_node->GetOutControlAnchor(), old_node->GetInControlAnchor()), + "Merge remove in ctrl edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(in_ctrl_node->GetOutControlAnchor(), new_node->GetInControlAnchor()), + "StreamMerge add in ctrl edge failed."); } for (const NodePtr &out_ctrl_node : old_node->GetOutControlNodes()) { - GE_CHK_STATUS(GraphUtils::RemoveEdge(old_node->GetOutControlAnchor(), out_ctrl_node->GetInControlAnchor()), - "Merge remove out ctrl edge failed."); - GE_CHK_STATUS(GraphUtils::AddEdge(new_node->GetOutControlAnchor(), out_ctrl_node->GetInControlAnchor()), - "StreamMerge add out ctrl edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveEdge(old_node->GetOutControlAnchor(), + out_ctrl_node->GetInControlAnchor()), + "Merge remove out ctrl edge failed."); + GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(new_node->GetOutControlAnchor(), out_ctrl_node->GetInControlAnchor()), + "StreamMerge add out ctrl edge failed."); } return SUCCESS; diff --git a/ge/graph/preprocess/graph_preprocess.cc b/ge/graph/preprocess/graph_preprocess.cc index c6b8810b..f3e7a87d 100644 --- a/ge/graph/preprocess/graph_preprocess.cc +++ b/ge/graph/preprocess/graph_preprocess.cc @@ -413,7 +413,14 @@ Status RecoverTransRoadForVar(const NodePtr &var, const VarTransRoad &road) { std::string stream_label; (void)AttrUtils::GetStr(var_desc, ATTR_NAME_STREAM_LABEL, stream_label); if (!stream_label.empty()) { - GE_CHK_STATUS_RET(SetStreamLabel(last_node, stream_label), "set stream label failed"); + auto status = SetStreamLabel(last_node, stream_label); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when %s", + stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } } GE_CHK_BOOL_EXEC((ge::AttrUtils::SetBool(last_node->GetOpDesc(), ge::ATTR_INSERTED_BY_GE, true)), REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed when %s", @@ -450,7 +457,14 @@ Status RecoverTransRoadForVarRef(const std::set &nodes, const VarTransR std::string stream_label; (void)AttrUtils::GetStr(var_desc, ATTR_NAME_STREAM_LABEL, stream_label); if (!stream_label.empty()) { - GE_CHK_STATUS_RET(SetStreamLabel(last_node, stream_label), "set stream label failed"); + auto status = SetStreamLabel(last_node, stream_label); + if (status != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed when %s", + stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str(), + __FUNCTION__); + GELOGE(status, "Set stream label failed."); + return status; + } } GE_CHK_BOOL_EXEC((ge::AttrUtils::SetBool(last_node->GetOpDesc(), ge::ATTR_INSERTED_BY_GE, true)), @@ -1159,9 +1173,9 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, desc.SetShapeRange(current_shape_range_vec); graphStatus graph_ret = op->UpdateInputDesc(0, desc); - GE_CHK_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); + GE_CHK_GRAPH_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); graph_ret = op->UpdateOutputDesc(0, desc); - GE_CHK_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); + GE_CHK_GRAPH_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); return SUCCESS; } } // namespace diff --git a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc index 486c78e2..721f9db1 100755 --- a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc +++ b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc @@ -99,7 +99,7 @@ Status InsertNewOpUtil::InsertAippOps(ComputeGraphPtr &graph, std::string &aippC GE_CHK_STATUS_RET(CheckGraph(graph), "after inserting all ops, check graph failed"); - GE_CHK_STATUS_RET(graph->TopologicalSorting(), "after insert dynamic op, sort graph failed"); + GE_CHK_GRAPH_STATUS_RET(graph->TopologicalSorting(), "after insert dynamic op, sort graph failed"); ClearNewOps(); diff --git a/ge/graph/preprocess/multi_batch_copy_graph.cc b/ge/graph/preprocess/multi_batch_copy_graph.cc index d4ccf060..04d75260 100644 --- a/ge/graph/preprocess/multi_batch_copy_graph.cc +++ b/ge/graph/preprocess/multi_batch_copy_graph.cc @@ -325,7 +325,7 @@ Status MultiBatchGraphCopyer::RelinkConstCtrlEdge() { continue; } if (!out_ctrl_anchor_of_in_ctrl_node->IsLinkedWith(out_node->GetInControlAnchor())) { - GE_CHK_STATUS_RET(out_ctrl_anchor_of_in_ctrl_node->LinkTo(out_node->GetInControlAnchor())) + GE_CHK_GRAPH_STATUS_RET(out_ctrl_anchor_of_in_ctrl_node->LinkTo(out_node->GetInControlAnchor())) } } } @@ -466,7 +466,7 @@ Status MultiBatchGraphCopyer::MoveInEntersInDataAnchorDown(NodePtr &node, OpDesc GE_CHECK_NOTNULL(peer_out_data_anchor); auto peer_in_data_node = peer_out_data_anchor->GetOwnerNode(); if (IsEnterType(peer_in_data_node->GetType())) { - GE_CHK_STATUS_RET(peer_out_data_anchor->Unlink(in_data_anchor)) + GE_CHK_GRAPH_STATUS_RET(peer_out_data_anchor->Unlink(in_data_anchor)) GELOGD("Unlink data edge from %s to %s.", peer_in_data_node->GetName().c_str(), node->GetName().c_str()); auto enter_in_data_anchors = peer_in_data_node->GetAllInDataAnchors(); for (auto &enter_in_data_anchor : enter_in_data_anchors) { @@ -475,7 +475,7 @@ Status MultiBatchGraphCopyer::MoveInEntersInDataAnchorDown(NodePtr &node, OpDesc if (peer_out_data_anchor_of_enter->IsLinkedWith(in_data_anchor)) { continue; } - GE_CHK_STATUS_RET(peer_out_data_anchor_of_enter->LinkTo(in_data_anchor)) + GE_CHK_GRAPH_STATUS_RET(peer_out_data_anchor_of_enter->LinkTo(in_data_anchor)) GELOGD("Relink data edge from %s to %s.", peer_out_data_anchor_of_enter->GetOwnerNode()->GetName().c_str(), node->GetName().c_str()); } @@ -512,17 +512,17 @@ Status MultiBatchGraphCopyer::InsertEnterAfterNode(NodePtr &node, const OpDescPt GELOGD("Create Enter op %s after %s.", name.c_str(), node->GetName().c_str()); auto enter_desc = AttrUtils::CopyOpDesc(copy_desc); enter_desc->SetName(name); - GE_CHK_STATUS_RET( + GE_CHK_GRAPH_STATUS_RET( enter_desc->UpdateInputDesc("x", node_desc->GetOutputDesc(outanchor_inanchors_nodes.first->GetIdx()))) - GE_CHK_STATUS_RET( + GE_CHK_GRAPH_STATUS_RET( enter_desc->UpdateOutputDesc("y", node_desc->GetOutputDesc(outanchor_inanchors_nodes.first->GetIdx()))) auto enter_node = graph_->AddNode(enter_desc); GE_CHECK_NOTNULL(enter_node); - GE_CHK_STATUS_RET(outanchor_inanchors_nodes.first->LinkTo(enter_node->GetInDataAnchor(kDataInIndex))) + GE_CHK_GRAPH_STATUS_RET(outanchor_inanchors_nodes.first->LinkTo(enter_node->GetInDataAnchor(kDataInIndex))) GE_CHECK_NOTNULL(enter_node->GetOutDataAnchor(kDataInIndex)); for (auto &inanchor_node : outanchor_inanchors_nodes.second) { - GE_CHK_STATUS_RET(outanchor_inanchors_nodes.first->Unlink(inanchor_node.first)) - GE_CHK_STATUS_RET(enter_node->GetOutDataAnchor(kDataInIndex)->LinkTo(inanchor_node.first)) + GE_CHK_GRAPH_STATUS_RET(outanchor_inanchors_nodes.first->Unlink(inanchor_node.first)) + GE_CHK_GRAPH_STATUS_RET(enter_node->GetOutDataAnchor(kDataInIndex)->LinkTo(inanchor_node.first)) GELOGD("Unlink from %s to %s, link from %s to %s then to %s.", node->GetName().c_str(), inanchor_node.second->GetName().c_str(), node->GetName().c_str(), enter_node->GetName().c_str(), inanchor_node.second->GetName().c_str()); @@ -538,14 +538,14 @@ Status MultiBatchGraphCopyer::MoveCtrlEdgeToOutNodes(NodePtr &node, set GE_CHECK_NOTNULL(in_ctrl_anchor); auto peer_out_ctrl_anchors = in_ctrl_anchor->GetPeerOutControlAnchors(); for (auto &peer_out_ctrl_anchor : peer_out_ctrl_anchors) { - GE_CHK_STATUS_RET(peer_out_ctrl_anchor->Unlink(in_ctrl_anchor)) + GE_CHK_GRAPH_STATUS_RET(peer_out_ctrl_anchor->Unlink(in_ctrl_anchor)) GELOGD("Unlink control edge from %s to %s.", peer_out_ctrl_anchor->GetOwnerNode()->GetName().c_str(), node->GetName().c_str()); for (auto &out_node : out_nodes) { auto in_ctrl_anchor_of_out_node = out_node->GetInControlAnchor(); GE_CHECK_NOTNULL(in_ctrl_anchor_of_out_node); if (!peer_out_ctrl_anchor->IsLinkedWith(in_ctrl_anchor_of_out_node)) { - GE_CHK_STATUS_RET(peer_out_ctrl_anchor->LinkTo(in_ctrl_anchor_of_out_node)) + GE_CHK_GRAPH_STATUS_RET(peer_out_ctrl_anchor->LinkTo(in_ctrl_anchor_of_out_node)) GELOGD("Link control edge from %s to %s.", peer_out_ctrl_anchor->GetOwnerNode()->GetName().c_str(), out_node->GetName().c_str()); } @@ -562,8 +562,8 @@ Status MultiBatchGraphCopyer::DeleteEnterWithoutDataOut() { auto out_nodes = node->GetOutAllNodes(); if (out_nodes.empty()) { GELOGD("Delete enter node: %s which has no output.", node->GetName().c_str()); - GE_CHK_STATUS_RET(GraphUtils::IsolateNode(node, {})) - GE_CHK_STATUS_RET(GraphUtils::RemoveNodeWithoutRelink(graph_, node)) + GE_CHK_GRAPH_STATUS_RET(GraphUtils::IsolateNode(node, {})) + GE_CHK_GRAPH_STATUS_RET(GraphUtils::RemoveNodeWithoutRelink(graph_, node)) } } } @@ -1387,11 +1387,11 @@ Status MultiBatchGraphCopyer::UpdateMaxShapeToData(const NodePtr &node, size_t o // must not be error, the calc result has been checked in function InsertSwitchNForData (void)CalcShape(data_to_dynamic_info_.at(data_name).at(max_shape_index), data_shape); auto ret = NodeUtils::UpdateOutputShape(*node, out_anchor_index, data_shape); - GE_CHK_STATUS_RET(ret, "Failed to update output shape for data %s", node->GetName().c_str()); + GE_CHK_GRAPH_STATUS_RET(ret, "Failed to update output shape for data %s", node->GetName().c_str()); // getnext_sink not has input if (!getnext_sink_dynamic_dims_) { ret = NodeUtils::UpdateInputShape(*node, kDataInIndex, data_shape); - GE_CHK_STATUS_RET(ret, "Failed to update input shape for data %s", node->GetName().c_str()); + GE_CHK_GRAPH_STATUS_RET(ret, "Failed to update input shape for data %s", node->GetName().c_str()); } else { // need to update shape of Shape_node when getnext_sink_dynamic GE_CHK_STATUS_RET(UpdateShapeOfShapeNode(node, out_anchor_index), "Failed to update shape of shape node"); diff --git a/inc/framework/common/debug/log.h b/inc/framework/common/debug/log.h index 422bd020..f8df82e5 100644 --- a/inc/framework/common/debug/log.h +++ b/inc/framework/common/debug/log.h @@ -86,12 +86,13 @@ } while (0); // If expr is not GRAPH_SUCCESS, print the log and return FAILED -#define GE_CHK_GRAPH_STATUS_RET(expr, ...) \ - do { \ - if ((expr) != ge::GRAPH_SUCCESS) { \ - DOMI_LOGE(__VA_ARGS__); \ - return FAILED; \ - } \ +#define GE_CHK_GRAPH_STATUS_RET(expr, ...) \ + do { \ + if ((expr) != ge::GRAPH_SUCCESS) { \ + REPORT_CALL_ERROR("E19999", "Operator graph failed when %s", __FUNCTION__); \ + DOMI_LOGE(__VA_ARGS__); \ + return FAILED; \ + } \ } while (0); // If expr is not SUCCESS, print the log and execute a custom statement