From 27257b990141b2d96bc01240e34316a59effe2ea Mon Sep 17 00:00:00 2001 From: jjfeing Date: Tue, 24 Nov 2020 21:48:00 +0800 Subject: [PATCH] add trace when mindspore error --- .../backend/optimizer/ascend/ascend_helper.cc | 6 +- .../insert_memcpy_async_for_cascade.cc | 7 +- .../insert_memcpy_async_for_hccl_op.cc | 11 +- .../split_unsupported_transdata.cc | 3 +- .../ir_fission/batch_norm_bert_fission.cc | 13 +- .../batch_norm_grad_infer_fission.cc | 31 ++-- .../ir_fission/batch_norm_grad_split.cc | 19 ++- .../ascend/ir_fission/bn_grad_split.cc | 13 +- .../optimizer/ascend/ir_fission/bn_split.cc | 10 +- .../ascend/ir_fission/concat_fission.cc | 4 +- .../ir_fission/dynamic_rnn_grad_fission_v2.cc | 4 +- .../ascend/ir_fission/lars_v2_fission.cc | 10 +- .../ir_fission/layer_norm_grad_split.cc | 7 +- .../ascend/ir_fission/pack_fission.cc | 4 +- .../ir_fission/single_batch_norm_fission.cc | 10 +- .../ascend/ir_fission/split_fission.cc | 3 +- .../ascend/ir_fission/transdata_split.cc | 4 +- .../ascend/ir_fusion/adam_apply_one_fusion.cc | 12 +- .../adam_apply_one_with_decay_rule.cc | 10 +- .../ascend/ir_fusion/add_input_to_output.cc | 4 +- .../ir_fusion/confusion_mul_grad_fusion.cc | 5 +- .../ir_fusion/fused_batch_norm_fusion.cc | 29 ++-- .../ir_fusion/lamb_next_mv_with_decay_rule.cc | 5 +- .../lamb_next_mv_with_decay_v1_rule.cc | 26 ++-- .../ascend/ir_fusion/lamb_next_right_rule.cc | 5 +- .../layer_norm_beta_gamma_backprop_fusion.cc | 10 +- .../ascend/ir_fusion/matmul_biasadd_fusion.cc | 5 +- .../ascend/ir_fusion/square_sum_fusion.cc | 17 ++- .../backend/session/anf_runtime_algorithm.cc | 138 ++++++++++++------ .../device/ascend/ascend_kernel_runtime.cc | 20 ++- .../device/ascend/ascend_kernel_runtime.h | 2 +- .../device/ascend/kernel_select_ascend.cc | 10 +- mindspore/core/utils/trace_base.cc | 4 +- mindspore/core/utils/trace_base.h | 1 + 34 files changed, 303 insertions(+), 159 deletions(-) diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc b/mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc index 5a0a42ab74..e9e85024c4 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc @@ -27,7 +27,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "backend/session/kernel_graph.h" #include "utils/ms_context.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; @@ -133,7 +133,7 @@ AnfNodePtr InsertTransOpForSingleOutput(const FuncGraphPtr &func_graph, const An std::vector origin_shape = AnfAlgo::GetOutputInferShape(node, 0); if (output_format == kOpFormat_NC1KHKWHWC0) { MS_LOG(EXCEPTION) << "got the hw format " << output_format << "when insert the transdata node " - << node->DebugString(); + << node->DebugString() << " trace: " << trace::DumpSourceLines(node); } if (kCommonFormatSet.find(output_format) == kCommonFormatSet.end() && origin_shape.size() > 1) { MS_LOG(DEBUG) << "Inserted Transdata " << output_format << " To default , index :0"; @@ -166,7 +166,7 @@ AnfNodePtr InsertTransOpForMultipleOutput(const FuncGraphPtr &func_graph, const std::string output_format = AnfAlgo::GetOutputFormat(node, output_idx); if (output_format == kOpFormat_NC1KHKWHWC0) { MS_LOG(EXCEPTION) << "Got the special format" << output_format << " when insert the transdata node " - << node->DebugString(); + << node->DebugString() << " trace: " << trace::DumpSourceLines(node); } auto tuple_getitem = CreatTupleGetItemNode(func_graph, node, output_idx); std::vector origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc b/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc index ba5f235feb..b4b148bc5b 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc @@ -19,6 +19,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "frontend/optimizer/opt.h" #include "backend/optimizer/ascend/ascend_helper.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -45,7 +46,8 @@ bool IsPartOutputsOfHcclOp(const AnfNodePtr &node, const CNodePtr &cur_hccl, con auto &node_users = manager->node_users(); auto iter = node_users.find(prev_hccl_op); if (iter == node_users.end()) { - MS_LOG(EXCEPTION) << "node has no output in manager"; + MS_LOG(EXCEPTION) << "node has no output in manager" + << " trace: " << trace::DumpSourceLines(cur_hccl); } for (const auto &node_index : iter->second) { AnfNodePtr output = node_index.first; @@ -79,7 +81,8 @@ AnfNodePtr InsertMemcpyAsyncForCascade::InsertMemcpyAsync(const FuncGraphPtr &gr if (IsPartOutputsOfHcclOp(input, hccl_node, graph)) { auto memcpy_async = CreateMemcpyAsyncOp(graph, input); if (memcpy_async == nullptr) { - MS_LOG(EXCEPTION) << "Create memcpy_async op failed."; + MS_LOG(EXCEPTION) << "Create memcpy_async op failed." + << " trace: " << trace::DumpSourceLines(hccl_node); } if (AnfAlgo::IsNodeDynamicShape(input)) { AnfAlgo::SetNodeAttr(kAttrIsDynamicShape, MakeValue(true), memcpy_async); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc b/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc index 9d676bad76..a7d3626b72 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc @@ -21,7 +21,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "frontend/optimizer/opt.h" #include "backend/optimizer/ascend/ascend_helper.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { namespace { @@ -61,7 +61,8 @@ void DealControlForGetitem(const CNodePtr &tuple_getitem, const FuncGraphPtr &gr auto &node_users = manager->node_users(); auto iter = node_users.find(tuple_getitem); if (iter == node_users.end()) { - MS_LOG(EXCEPTION) << "node has no output in manager"; + MS_LOG(EXCEPTION) << "node has no output in manager" + << " trace: " << trace::DumpSourceLines(hccl_node); } for (const auto &node_index : iter->second) { AnfNodePtr output = node_index.first; @@ -81,7 +82,8 @@ void TransferControl(const CNodePtr &hccl_node, const std::vector &m auto &node_users = manager->node_users(); auto iter = node_users.find(hccl_node); if (iter == node_users.end()) { - MS_LOG(EXCEPTION) << "node has no output in manager"; + MS_LOG(EXCEPTION) << "node has no output in manager" + << " trace: " << trace::DumpSourceLines(hccl_node); } // find hccl_node's output which is a control depend for (const auto &node_index : iter->second) { @@ -140,7 +142,8 @@ bool InsertMemcpyAsyncForHcclOp::NeedInsertMemcpy(const FuncGraphPtr &graph, con // when input is used by others auto iter = node_users.find(input); if (iter == node_users.end()) { - MS_LOG(EXCEPTION) << "node has no output in manager"; + MS_LOG(EXCEPTION) << "node has no output in manager" + << " trace: " << trace::DumpSourceLines(input); } if (IsNodeOutPutUsedByOtherRealKernel(iter->second)) { return true; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc b/mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc index 92f6c5799b..40be5856f8 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc @@ -18,6 +18,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -39,7 +40,7 @@ const AnfNodePtr SplitUnsupportedTransData::Process(const FuncGraphPtr &func_gra MS_EXCEPTION_IF_NULL(kernel_info); if (kernel_info->GetInputNum() != 1 || kernel_info->GetOutputNum() != 1) { MS_LOG(EXCEPTION) << "Transdata node's kernel info's input and output format size is not 1" - << ori_trans_data->DebugString(); + << ori_trans_data->DebugString() << trace::DumpSourceLines(node); } return SplitTransData(func_graph, ori_trans_data); } diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc index 53c1df5bee..f03e562c35 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc @@ -19,6 +19,7 @@ #include #include "backend/session/anf_runtime_algorithm.h" #include "backend/optimizer/common/helper.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -65,7 +66,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP MS_EXCEPTION_IF_NULL(bn_cnode); if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) { MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than " - << kBatchNormRealInputNum + 1; + << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn); } std::vector bn_training_reduce_inputs = { NewValueNode(std::make_shared(kBNTrainingReduceOpName)), bn_cnode->input(1)}; @@ -91,11 +92,12 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod MS_EXCEPTION_IF_NULL(bn_cnode); if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) { MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than " - << kBatchNormRealInputNum + 1; + << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn); } if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) { MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum - << ", but it is " << bn_training_reduce_outputs.size(); + << ", but it is " << bn_training_reduce_outputs.size() + << " trace: " << trace::DumpSourceLines(bn); } std::vector bn_training_update_v2_inputs = { NewValueNode(std::make_shared(kBNTrainingUpdateV2OpName)), @@ -111,7 +113,7 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod MS_EXCEPTION_IF_NULL(bn_abstract_tuple); if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) { MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is " - << bn_abstract_tuple->elements().size(); + << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn); } std::vector abstract_list{bn_abstract_tuple->elements()[0], bn_abstract_tuple->elements()[3], bn_abstract_tuple->elements()[4]}; @@ -155,7 +157,8 @@ const AnfNodePtr BatchNormBertFission::Process(const FuncGraphPtr &func_graph, c &bn_training_update_v2_outputs); if (bn_training_update_v2_outputs.size() != kBNTrainingUpdateV2OutputNum) { MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingUpdateV2OutputNum - << ", but it is " << bn_training_update_v2_outputs.size(); + << ", but it is " << bn_training_update_v2_outputs.size() + << " trace: " << trace::DumpSourceLines(node); } auto manager = func_graph->manager(); MS_EXCEPTION_IF_NULL(manager); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc index 28997a093a..83e0f647dd 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc @@ -17,6 +17,7 @@ #include #include "backend/optimizer/common/helper.h" #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -61,15 +62,18 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func // Set inputs auto iter_input0 = (*equiv).find(input0_var_); if (iter_input0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } auto iter_input2 = (*equiv).find(input2_var_); if (iter_input2 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } auto iter_input4 = (*equiv).find(input4_var_); if (iter_input4 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } std::vector bn_infer_grad_inputs = { NewValueNode(std::make_shared(kBNInferGradOpName)), utils::cast(iter_input0->second), @@ -80,7 +84,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func auto bn_grad_abstract_tuple = dyn_cast(bn_grad->abstract()); MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple); if (bn_grad_abstract_tuple->elements().empty()) { - MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty"; + MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty" + << " trace: " << trace::DumpSourceLines(bn_grad); } bn_infer_grad->set_abstract(bn_grad_abstract_tuple->elements()[0]); AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_grad, bn_infer_grad); @@ -97,19 +102,23 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph // Set inputs auto iter_input0 = (*equiv).find(input0_var_); if (iter_input0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } auto iter_input1 = (*equiv).find(input1_var_); if (iter_input1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } auto iter_input3 = (*equiv).find(input3_var_); if (iter_input3 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } auto iter_input4 = (*equiv).find(input4_var_); if (iter_input4 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched." + << " trace: " << trace::DumpSourceLines(bn_grad); } std::vector bn_training_update_grad_inputs = { NewValueNode(std::make_shared(kBNTrainingUpdateGradOpName)), @@ -121,7 +130,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph auto bn_grad_abstract_tuple = dyn_cast(bn_grad->abstract()); MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple); if (bn_grad_abstract_tuple->elements().size() < kBatchNormGradInferOutputNum) { - MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3"; + MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3" + << trace::DumpSourceLines(bn_grad); } std::vector abstract_list{bn_grad_abstract_tuple->elements()[1], bn_grad_abstract_tuple->elements()[2]}; @@ -160,7 +170,8 @@ const AnfNodePtr BatchNormGradInferFission::Process(const FuncGraphPtr &func_gra &bn_training_update_grad_outputs); if (bn_training_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { MS_LOG(EXCEPTION) << "The output size of " << bn_training_update_grad << " should be " - << kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size(); + << kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size() + << trace::DumpSourceLines(node); } std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_infer_grad, bn_training_update_grad_outputs[0], bn_training_update_grad_outputs[1]}; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc index e63249ca46..b6d74c487d 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc @@ -23,6 +23,7 @@ #include "backend/optimizer/common/helper.h" #include "runtime/device/kernel_info.h" #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra MS_EXCEPTION_IF_NULL(bn_grad_node); const auto &bn_grad_inputs = bn_grad_node->inputs(); if (bn_grad_inputs.size() < kBNGradInputNum) { - MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size." + << " trace: " << trace::DumpSourceLines(bn_grad_node); } std::vector bn_update_grad_inputs = { NewValueNode(std::make_shared(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2], @@ -59,10 +61,12 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra MS_EXCEPTION_IF_NULL(bn_reduce_grad_outputs); const auto &bn_grad_inputs = bn_grad_node->inputs(); if (bn_grad_inputs.size() < kBNGradInputNum) { - MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size" + << " trace: " << trace::DumpSourceLines(bn_grad_node); } if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { - MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(bn_grad_node); } std::vector bn_reduce_grad_inputs = { NewValueNode(std::make_shared(kBNTrainingReduceGradOpName)), @@ -101,7 +105,8 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con auto primitive = AnfAlgo::GetCNodePrimitive(cnode); MS_EXCEPTION_IF_NULL(primitive); if (!primitive->HasAttr(kAttrIsTraining)) { - MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training"; + MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training" + << " trace: " << trace::DumpSourceLines(node); return nullptr; } if (!AnfAlgo::GetNodeAttr(cnode, kAttrIsTraining)) { @@ -112,13 +117,15 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con std::vector bn_update_grad_outputs; CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs); if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { - MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(node); } std::vector bn_reduce_grad_outputs; CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs); if (bn_reduce_grad_outputs.size() != kSingleOutputNum) { - MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(node); } std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0], diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc index 72ad8d4366..159e832b3b 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc @@ -23,6 +23,7 @@ #include "backend/optimizer/common/helper.h" #include "runtime/device/kernel_info.h" #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra MS_EXCEPTION_IF_NULL(bn_grad_node); auto bn_grad_inputs = bn_grad_node->inputs(); if (bn_grad_inputs.size() != kBNGradInputNum) { - MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size" + << " trace: " << trace::DumpSourceLines(bn_grad_node); } std::vector bn_update_grad_inputs = { NewValueNode(std::make_shared(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2], @@ -58,7 +60,8 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra MS_EXCEPTION_IF_NULL(bn_grad_node); auto bn_grad_inputs = bn_grad_node->inputs(); if (bn_grad_inputs.size() != kBNGradInputNum) { - MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size" + << " trace: " << trace::DumpSourceLines(bn_grad_node); } if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"; @@ -90,13 +93,15 @@ CNodePtr BNGradSplitForTBE(const FuncGraphPtr &func_graph, const CNodePtr &cnode std::vector bn_update_grad_outputs; CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs); if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { - MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(cnode); } std::vector bn_reduce_grad_outputs; CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs); if (bn_reduce_grad_outputs.size() != 1) { - MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(cnode); } std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0], diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc index 36dc2a30da..044b3d925a 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc @@ -23,6 +23,7 @@ #include "backend/optimizer/common/helper.h" #include "runtime/device/kernel_info.h" #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -64,10 +65,12 @@ AnfNodePtr CreateOutputsOfBNTrainingUpdate(const FuncGraphPtr &graph, const CNod MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(bn_cnode); if (bn_cnode->inputs().size() != kBnInputNum) { - MS_LOG(EXCEPTION) << "BN node has wrong input size"; + MS_LOG(EXCEPTION) << "BN node has wrong input size" + << " trace: " << trace::DumpSourceLines(bn_cnode); } if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) { - MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"; + MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size" + << " trace: " << trace::DumpSourceLines(bn_cnode); } // the inputs of BNTrainingUpdate are from the outputs of BNTrainingReduce and the inputs of BN std::vector bn_training_update_inputs = { @@ -110,7 +113,8 @@ AnfNodePtr SplitFusedBatchNormForTBE(const FuncGraphPtr &func_graph, const AnfNo return nullptr; } if (bn_training_reduce_outputs.size() != kBN1OutputNum) { - MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail"; + MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail" + << " trace: " << trace::DumpSourceLines(node); } // Create BNTrainingUpdate node diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc index fb5a8d623b..c2c5355fcf 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc @@ -17,6 +17,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -47,7 +48,8 @@ AnfNodePtr CreateNewConcat(const FuncGraphPtr &func_graph, const CNodePtr &origi } auto output_shape = AnfAlgo::GetOutputInferShape(origin_concat_cnode, 0); if (axis < 0 || axis >= SizeToLong(output_shape.size()) || axis >= SizeToLong(input_shape.size())) { - MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"; + MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range" + << " trace: " << trace::DumpSourceLines(origin_concat_cnode); } output_shape[axis] = input_shape[axis] * offset; AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(origin_concat_cnode, 0)}, {output_shape}, diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc index 29bd17b1db..5ab380c715 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc @@ -17,6 +17,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -290,7 +291,8 @@ AnfNodePtr CreateHConcat(const FuncGraphPtr &func_graph, const CNodePtr &dynamic std::vector splitv_outputs; CreateMultipleOutputsOfAnfNode(func_graph, splitv, kSplitVOutputNum, &splitv_outputs); if (splitv_outputs.size() != kSplitVOutputNum) { - MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed"; + MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed" + << " trace: " << trace::DumpSourceLines(dynamic_rnn_grad_cnode); } auto origin_input4 = dynamic_rnn_grad_cnode->input(5); auto origin_input4_shape = AnfAlgo::GetOutputInferShape(origin_input4, 0); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc index 563e44b501..5fba8692db 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc @@ -19,6 +19,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "backend/optimizer/common/helper.h" #include "utils/utils.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -28,7 +29,8 @@ void CreateOutputsOfSquareSumAll(const FuncGraphPtr &graph, const CNodePtr &lars MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(lars_v2); if (lars_v2->size() != kLarsV2InputNum) { - MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum; + MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum + << " trace: " << trace::DumpSourceLines(lars_v2); } std::vector inputs = {NewValueNode(std::make_shared(kSquareSumAllOpName)), lars_v2->input(1), @@ -50,10 +52,12 @@ CNodePtr CreateLarsV2Update(const FuncGraphPtr &graph, const CNodePtr &lars_v2, MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(lars_v2); if (square_sum_all_outputs.size() != 2) { - MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2"; + MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2" + << " trace: " << trace::DumpSourceLines(lars_v2); } if (lars_v2->size() != kLarsV2InputNum) { - MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum; + MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum + << " trace: " << trace::DumpSourceLines(lars_v2); } std::vector inputs = {NewValueNode(std::make_shared(kLarsV2UpdateOpName)), lars_v2->input(1), diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc index 8753fa20af..e4858d7a79 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc @@ -22,6 +22,7 @@ #include "runtime/device/kernel_info.h" #include "ir/primitive.h" #include "utils/utils.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -98,14 +99,16 @@ const AnfNodePtr LayerNormGradSplit::Process(const FuncGraphPtr &graph, const An std::vector layer_norm_x_backprop_outputs; CreateOutputsOfLayerNormXBackprop(graph, cnode, &layer_norm_x_backprop_outputs); if (layer_norm_x_backprop_outputs.size() != kSingleOutputNum) { - MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size"; + MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(node); } // create layer_norm_beta_gamma_backprop std::vector layer_norm_beta_gamma_backprop_outputs; CreateOutputsOfLayerNormBetaGammaBackprop(graph, cnode, &layer_norm_beta_gamma_backprop_outputs); if (layer_norm_beta_gamma_backprop_outputs.size() != kLayerNormBetaGammaBackpropOutputNum) { - MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size"; + MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size" + << " trace: " << trace::DumpSourceLines(node); } std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), layer_norm_x_backprop_outputs[0], diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc index c78ddd10f4..441c2762ae 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc @@ -17,6 +17,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -45,7 +46,8 @@ AnfNodePtr CreateNewPack(const FuncGraphPtr &func_graph, const CNodePtr &origin_ axis += output_shape.size(); } if (axis < 0) { - MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"; + MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range" + << " trace: " << trace::DumpSourceLines(origin_pack_cnode); } std::vector new_shape; for (size_t i = 0; i < output_shape.size() + 1; ++i) { diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc index 9df2de87ce..d58644d74c 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc @@ -18,6 +18,7 @@ #include #include "backend/session/anf_runtime_algorithm.h" #include "backend/optimizer/common/helper.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -31,7 +32,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP MS_EXCEPTION_IF_NULL(bn_cnode); if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) { MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than " - << kBatchNormRealInputNum + 1; + << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn); } std::vector bn_training_reduce_inputs = { NewValueNode(std::make_shared(kBNTrainingReduceOpName)), bn_cnode->input(1)}; @@ -56,11 +57,12 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod MS_EXCEPTION_IF_NULL(bn_cnode); if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) { MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than " - << kBatchNormRealInputNum + 1; + << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn); } if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) { MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum - << ", but it is " << bn_training_reduce_outputs.size(); + << ", but it is " << bn_training_reduce_outputs.size() + << " trace: " << trace::DumpSourceLines(bn); } std::vector bn_training_update_v3_inputs = { NewValueNode(std::make_shared(kBNTrainingUpdateV3OpName)), @@ -76,7 +78,7 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod MS_EXCEPTION_IF_NULL(bn_abstract_tuple); if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) { MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is " - << bn_abstract_tuple->elements().size(); + << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn); } bn_training_update_v3->set_abstract(bn->abstract()); bn_training_update_v3->set_scope(bn->scope()); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc index 47ded45024..dc8b497303 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc @@ -17,6 +17,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -35,7 +36,7 @@ CNodePtr CreateBaseSplitVNode(const FuncGraphPtr &func_graph, const CNodePtr &or MS_EXCEPTION_IF_NULL(origin_cnode); if (origin_cnode->inputs().size() < kSplitInputNum) { MS_LOG(EXCEPTION) << "The input number of split: " << origin_cnode->DebugString() << " should be " - << kSplitInputNum - 1; + << kSplitInputNum - 1 << " trace: " << trace::DumpSourceLines(origin_cnode); } return CreateSplitVNode(func_graph, origin_cnode->input(1)); } diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc index 7a80dfacf2..ef0fe087c7 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc @@ -18,6 +18,7 @@ #include "backend/optimizer/ascend/ascend_helper.h" #include "backend/session/anf_runtime_algorithm.h" #include "debug/anf_ir_dump.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -99,7 +100,8 @@ bool TransDataSplit::DoSplit(const FuncGraphPtr &func_graph, const AnfNodePtr &n manager->AddFuncGraph(func_graph); if (!manager->Replace(node, new_replace_node)) { - MS_LOG(EXCEPTION) << "Manager replace node failed"; + MS_LOG(EXCEPTION) << "Manager replace node failed" + << " trace: " << trace::DumpSourceLines(node); } MS_LOG(INFO) << "Transdata node:" << cnode->DebugString() << "split success."; return true; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc index 9164484b30..8ab01e132d 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc @@ -16,6 +16,7 @@ #include "backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.h" #include "backend/optimizer/common/helper.h" #include "backend/session/anf_runtime_algorithm.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { const BaseRef AdamApplyOneFusion::DefinePattern() const { @@ -218,7 +219,8 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) { auto iter_sub0 = (*equiv).find(sub0_var_); if (iter_sub0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched." + << " trace: " << trace::DumpSourceLines(node); } sub0 = utils::cast(iter_sub0->second); } @@ -233,11 +235,13 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con AbstractBasePtrList new_node_abstract_list; auto iter_add0 = (*equiv).find(add0_var_); if (iter_add0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto iter_add1 = (*equiv).find(add1_var_); if (iter_add1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto add0 = utils::cast(iter_add0->second); MS_EXCEPTION_IF_NULL(add0); @@ -253,7 +257,7 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con CreateMultipleOutputsOfAnfNode(func_graph, new_node, kAdamApplyOneOutputNum, &new_node_outputs); if (new_node_outputs.size() != kAdamApplyOneOutputNum) { MS_LOG(EXCEPTION) << "The output size of node " << new_node->DebugString() << " should be " - << kAdamApplyOneOutputNum; + << kAdamApplyOneOutputNum << " trace: " << trace::DumpSourceLines(node); } auto manager = func_graph->manager(); MS_EXCEPTION_IF_NULL(manager); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc index f89c833d06..6c60342461 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc @@ -21,6 +21,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "ir/primitive.h" #include "backend/optimizer/common/helper.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -281,7 +282,8 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) { auto iter_sub0 = (*equiv).find(sub0_var_); if (iter_sub0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched." + << " trace: " << trace::DumpSourceLines(node); } sub0 = utils::cast(iter_sub0->second); } @@ -296,11 +298,13 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c auto iter_add0 = (*equiv).find(add0_var_); if (iter_add0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto iter_add1 = (*equiv).find(add1_var_); if (iter_add1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto add0 = utils::cast(iter_add0->second); MS_EXCEPTION_IF_NULL(add0); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc index b055847908..63cbd5aaba 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc @@ -19,6 +19,7 @@ #include "backend/optimizer/ascend/ir_fusion/input_to_output_registry.h" #include "backend/session/anf_runtime_algorithm.h" #include "backend/kernel_compiler/oplib/oplib.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -107,7 +108,8 @@ const AnfNodePtr AddInputToOutput::Process(const FuncGraphPtr &func_graph, const MS_EXCEPTION_IF_NULL(new_abstract_tuple); CreateMultipleOutputsOfAnfNode(func_graph, cnode, new_abstract_tuple->size(), &new_outputs); if (new_outputs.size() != new_abstract_tuple->size()) { - MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString(); + MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString() + << " trace: " << trace::DumpSourceLines(node); } return new_outputs[0]; } diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc index f051c7594a..fecdc6af4e 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc @@ -24,7 +24,7 @@ #include "utils/utils.h" #include "abstract/abstract_value.h" #include "backend/optimizer/common/helper.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { namespace { @@ -58,7 +58,8 @@ AnfNodePtr GetMul0(const FuncGraphPtr &graph, const AnfNodePtr &input2, const An auto manager = graph->manager(); MS_EXCEPTION_IF_NULL(manager); if (manager->node_users().find(input2) == manager->node_users().end()) { - MS_LOG(EXCEPTION) << "node has no output in manager"; + MS_LOG(EXCEPTION) << "node has no output in manager" + << " trace: " << trace::DumpSourceLines(input2); } AnfNodePtr mul0 = nullptr; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc index 8d4a5743fe..4bb5d1159c 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc @@ -19,6 +19,7 @@ #include "backend/optimizer/common/helper.h" #include "backend/session/anf_runtime_algorithm.h" #include "utils/utils.h" +#include "utils/trace_base.h" namespace mindspore { namespace opt { @@ -41,7 +42,8 @@ void GetBNOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &bn, std::vect auto manager = func_graph->manager(); MS_EXCEPTION_IF_NULL(manager); if (manager->node_users().find(bn) == manager->node_users().end()) { - MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs"; + MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs" + << " trace: " << trace::DumpSourceLines(bn); } for (const auto &node_index : manager->node_users()[bn]) { const AnfNodePtr &output = node_index.first; @@ -113,7 +115,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func // Set input to create node auto iter_data_input0 = (*equiv).find(data_input0_var_); if (iter_data_input0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched." + << " trace: " << trace::DumpSourceLines(node); } std::vector bn_training_reduce_inputs = { NewValueNode(std::make_shared(kBNTrainingReduceOpName)), @@ -124,13 +127,15 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func // Set abstract auto iter_data_input1 = (*equiv).find(data_input1_var_); if (iter_data_input1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto data_input1 = utils::cast(iter_data_input1->second); MS_EXCEPTION_IF_NULL(data_input1); auto iter_data_input2 = (*equiv).find(data_input2_var_); if (iter_data_input2 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto data_input2 = utils::cast(iter_data_input2->second); MS_EXCEPTION_IF_NULL(data_input2); @@ -190,17 +195,19 @@ void FusedBatchNormFusion::GetBNTrainingUpdateAbstractList(const EquivPtr &equiv MS_EXCEPTION_IF_NULL(bn_abstract_tuple); if (bn_abstract_tuple->elements().size() < kBnOutputNum) { MS_LOG(EXCEPTION) << "The abstract size of node bn must not be less than " << kBnOutputNum << ", but it is " - << bn_abstract_tuple->elements().size(); + << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn); } auto iter_variable_input0 = (*equiv).find(variable_input0_var_); if (iter_variable_input0 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched." + << " trace: " << trace::DumpSourceLines(bn); } auto variable_input0 = utils::cast(iter_variable_input0->second); MS_EXCEPTION_IF_NULL(variable_input0); auto iter_variable_input1 = (*equiv).find(variable_input1_var_); if (iter_variable_input1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched." + << " trace: " << trace::DumpSourceLines(bn); } auto variable_input1 = utils::cast(iter_variable_input1->second); MS_EXCEPTION_IF_NULL(variable_input1); @@ -222,7 +229,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingUpdate( // Set abstract auto iter_batch_norm = (*equiv).find(batch_norm_var_); if (iter_batch_norm == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched." + << " trace: " << trace::DumpSourceLines(node); } AnfNodePtr bn = utils::cast(iter_batch_norm->second); MS_EXCEPTION_IF_NULL(bn); @@ -260,12 +268,13 @@ const AnfNodePtr FusedBatchNormFusion::Process(const FuncGraphPtr &func_graph, c &bn_training_update_outputs); if (bn_training_update_outputs.size() < kBNTrainingUpdateOutputNum) { MS_LOG(EXCEPTION) << "The output size of node bn must be " << kBNTrainingUpdateOutputNum << ", but it is " - << bn_training_update_outputs.size(); + << bn_training_update_outputs.size() << " trace: " << trace::DumpSourceLines(node); } // Replace old bn outputs with new outputs auto iter_batch_norm = (*equiv).find(batch_norm_var_); if (iter_batch_norm == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched." + << " trace: " << trace::DumpSourceLines(node); } AnfNodePtr bn = utils::cast(iter_batch_norm->second); std::vector bn_outputs; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc index 4ef3fa269f..78c265da80 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc @@ -17,7 +17,7 @@ #include #include "backend/session/anf_runtime_algorithm.h" #include "frontend/optimizer/opt.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { AnfNodePtr LambNextMVWithDecayRule::GetLambNextMVWithDecayOutput(const FuncGraphPtr &func_graph, @@ -97,7 +97,8 @@ const AnfNodePtr LambNextMVWithDecayRule::Process(const FuncGraphPtr &func_graph auto manager = func_graph->manager(); MS_EXCEPTION_IF_NULL(manager); if (manager->node_users().find(mul4) == manager->node_users().end()) { - MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"; + MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input" + << " trace: " << trace::DumpSourceLines(node); } AnfNodeIndexSet mul4_outputs = manager->node_users()[mul4]; auto iter = std::find_if(mul4_outputs.begin(), mul4_outputs.end(), diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc index 955eb9d022..1ad0db72b9 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc @@ -22,7 +22,7 @@ #include "backend/session/anf_runtime_algorithm.h" #include "frontend/optimizer/opt.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { namespace { @@ -31,27 +31,31 @@ std::tuple GetSharedNodes(const auto add3 = node->cast(); MS_EXCEPTION_IF_NULL(add3); if (add3->inputs().size() < kAddInputNum) { - MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum; + MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum + << " trace: " << trace::DumpSourceLines(node); } auto real_div2_anf = add3->input(1); MS_EXCEPTION_IF_NULL(real_div2_anf); auto real_div2 = real_div2_anf->cast(); MS_EXCEPTION_IF_NULL(real_div2); if (real_div2->inputs().size() < kRealDivInputNum) { - MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum; + MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum + << " trace: " << trace::DumpSourceLines(node); } auto sqrt0_anf = real_div2->input(2); MS_EXCEPTION_IF_NULL(sqrt0_anf); auto sqrt0 = sqrt0_anf->cast(); MS_EXCEPTION_IF_NULL(sqrt0); if (sqrt0->inputs().size() < kRsqrtInputNum) { - MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum; + MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum + << " trace: " << trace::DumpSourceLines(node); } auto add2_anf = sqrt0->input(1); MS_EXCEPTION_IF_NULL(add2_anf); auto add2 = add2_anf->cast(); if (add2->inputs().size() < kAddInputNum) { - MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum; + MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum + << " trace: " << trace::DumpSourceLines(node); } return std::make_tuple(add3->input(2), real_div2->input(1), add2->input(1), add2->input(2)); } @@ -101,10 +105,12 @@ std::tuple GetAdd0Add1Nodes(const AnfNodePtr &real_div0_ MS_EXCEPTION_IF_NULL(real_div0); MS_EXCEPTION_IF_NULL(real_div1); if (real_div0->inputs().size() != kRealDivInputNum) { - MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size"; + MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size" + << " trace: " << trace::DumpSourceLines(real_div0_anf); } if (real_div1->inputs().size() != kRealDivInputNum) { - MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size"; + MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size" + << " trace: " << trace::DumpSourceLines(real_div1_anf); } return std::make_tuple(real_div0->input(1), real_div1->input(1)); } @@ -165,7 +171,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra auto manager = func_graph->manager(); MS_EXCEPTION_IF_NULL(manager); if (manager->node_users().find(mul4) == manager->node_users().end()) { - MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"; + MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input" + << " trace: " << trace::DumpSourceLines(node); } AnfNodeIndexSet mul4_output_node_index_set = manager->node_users()[mul4]; auto iter = std::find_if( @@ -195,7 +202,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra std::vector fusion_node_outputs; CreateMultipleOutputsOfAnfNode(func_graph, fusion_node, kLambNextMVWithDecayV1OutputNum, &fusion_node_outputs); if (fusion_node_outputs.size() != kLambNextMVWithDecayV1OutputNum) { - MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!"; + MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!" + << " trace: " << trace::DumpSourceLines(node); } (void)manager->Replace(add0, fusion_node_outputs[1]); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc index 03bc1e0484..08de5e17c1 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc @@ -16,7 +16,7 @@ #include "backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.h" #include #include "backend/optimizer/common/helper.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { AnfNodePtr LambNextRightRule::CreateLambNextRightNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const { @@ -69,7 +69,8 @@ const AnfNodePtr LambNextRightRule::Process(const FuncGraphPtr &func_graph, cons // Set abstract of new node auto iter_add1 = (*equiv).find(add1_var_); if (iter_add1 == (*equiv).end()) { - MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."; + MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched." + << " trace: " << trace::DumpSourceLines(node); } auto add1 = utils::cast(iter_add1->second); MS_EXCEPTION_IF_NULL(add1); diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc index fa1e92120d..8a23ac2f84 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc @@ -17,7 +17,7 @@ #include #include #include "backend/session/anf_runtime_algorithm.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { using common::SafeCStr; @@ -36,7 +36,7 @@ void GetOutputCastNodes(const FuncGraphPtr &func_graph, const AnfNodePtr &node, MS_EXCEPTION_IF_NULL(output_cnode); if (AnfAlgo::GetCNodeName(output_cnode) != prim::kPrimTupleGetItem->name()) { MS_LOG(EXCEPTION) << "The output of node " << node->DebugString() << " should be " - << prim::kPrimTupleGetItem->name(); + << prim::kPrimTupleGetItem->name() << " trace: " << trace::DumpSourceLines(node); } if (manager->node_users().find(output) == manager->node_users().end() || manager->node_users()[output].size() != 1) { @@ -149,11 +149,13 @@ const AnfNodePtr LayerNormBetaGammaBackpropFusion::Process(const FuncGraphPtr &f MS_EXCEPTION_IF_NULL(cast_nodes[0]); MS_EXCEPTION_IF_NULL(cast_nodes[1]); if (cast_nodes[0]->inputs().size() != kCastInputNum) { - MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum; + MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum + << " trace: " << trace::DumpSourceLines(node); } (void)manager->Replace(cast_nodes[0], cast_nodes[0]->input(1)); if (cast_nodes[1]->inputs().size() != kCastInputNum) { - MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum; + MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum + << " trace: " << trace::DumpSourceLines(node); } (void)manager->Replace(cast_nodes[1], cast_nodes[1]->input(1)); return nullptr; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc index 4fa4b5e7b0..dd729c5bc2 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc @@ -18,7 +18,7 @@ #include "backend/optimizer/common/helper.h" #include "backend/session/anf_runtime_algorithm.h" #include "utils/utils.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { const BaseRef MatmulBiasaddFusion::DefinePattern() const { @@ -43,7 +43,8 @@ const AnfNodePtr MatmulBiasaddFusion::Process(const FuncGraphPtr &graph, const A auto matmul = GetAnfNodeByVar(equiv, matmul_var_); if (matmul == nullptr || !matmul->isa()) { - MS_LOG(EXCEPTION) << "Get CNode MatMul failed!"; + MS_LOG(EXCEPTION) << "Get CNode MatMul failed!" + << " trace: " << trace::DumpSourceLines(node); } AnfAlgo::CopyNodeAttrs(matmul, new_node); return new_node; diff --git a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc index e8f198ccd7..de2cb2e287 100644 --- a/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc +++ b/mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc @@ -25,7 +25,7 @@ #include "base/core_ops.h" #include "backend/optimizer/common/helper.h" #include "runtime/device/kernel_info.h" - +#include "utils/trace_base.h" namespace mindspore { namespace opt { namespace { @@ -34,7 +34,8 @@ CNodePtr GenerateSquareSumV1(const FuncGraphPtr &graph, const CNodePtr &square, MS_EXCEPTION_IF_NULL(square); MS_EXCEPTION_IF_NULL(sum); if (square->inputs().size() != kSquareNodeInputNum) { - MS_LOG(EXCEPTION) << "Square node has wrong input size"; + MS_LOG(EXCEPTION) << "Square node has wrong input size" + << " trace: " << trace::DumpSourceLines(square); } auto prim = std::make_shared(kSquareSumV1OpName); MS_EXCEPTION_IF_NULL(prim); @@ -60,7 +61,8 @@ CNodePtr GenerateSquareSumV2(const FuncGraphPtr &graph, const CNodePtr &square, MS_EXCEPTION_IF_NULL(square); MS_EXCEPTION_IF_NULL(sum); if (square->inputs().size() != kSquareNodeInputNum) { - MS_LOG(EXCEPTION) << "Square node has wrong input size"; + MS_LOG(EXCEPTION) << "Square node has wrong input size" + << " trace: " << trace::DumpSourceLines(square); } auto prim = std::make_shared(kSquareSumV2OpName); MS_EXCEPTION_IF_NULL(prim); @@ -83,7 +85,8 @@ std::tuple GetPrevNodes(const AnfNodePtr &node) auto sum = node->cast(); MS_EXCEPTION_IF_NULL(sum); if (sum->inputs().size() != kSumNodeInputNum) { - MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size"; + MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size" + << " trace: " << trace::DumpSourceLines(sum); } auto square_anf = sum->input(1); MS_EXCEPTION_IF_NULL(square_anf); @@ -111,7 +114,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo auto manager = graph->manager(); MS_EXCEPTION_IF_NULL(manager); if (manager->node_users().find(square_anf) == manager->node_users().end()) { - MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap"; + MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap" + << " trace: " << trace::DumpSourceLines(node); } AnfNodePtr ret_node = nullptr; if (manager->node_users()[square_anf].size() == 1) { @@ -122,7 +126,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo std::vector square_sumv2_outputs; CreateMultipleOutputsOfAnfNode(graph, square_sumv2, kSquareSumv2OutputNum, &square_sumv2_outputs); if (square_sumv2_outputs.size() != kSquareSumv2OutputNum) { - MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail"; + MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail" + << " trace: " << trace::DumpSourceLines(square_sumv2); } (void)manager->Replace(square, square_sumv2_outputs[1]); ret_node = square_sumv2_outputs[0]; diff --git a/mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc b/mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc index ebe8fc39b2..ce26df37c9 100644 --- a/mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc +++ b/mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc @@ -31,6 +31,7 @@ #include "common/trans.h" #include "abstract/param_validator.h" #include "pipeline/jit/static_analysis/static_analysis.h" +#include "utils/trace_base.h" namespace mindspore { namespace session { @@ -164,7 +165,7 @@ KernelWithIndex AnfRuntimeAlgorithm::VisitKernelWithReturnType(const AnfNodePtr } if (opt::IsNopNode(cnode) && visit_nop_node) { if (cnode->size() != kNopNodeInputSize) { - MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString(); + MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString() << " trace: " << trace::DumpSourceLines(cnode); } return VisitKernelWithReturnType(cnode->input(kNopNodeRealInputIndex), 0, visit_nop_node, return_types); } @@ -250,7 +251,7 @@ std::string AnfRuntimeAlgorithm::GetCNodeName(const AnfNodePtr &node) { } return func_graph->ToString(); } - MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString(); + MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString() << " trace: " << trace::DumpSourceLines(node); } std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) { @@ -261,7 +262,8 @@ std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) { void AnfRuntimeAlgorithm::SetNodeAttr(const std::string &key, const ValuePtr &value, const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(node); if (!node->isa()) { - MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString(); + MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString() + << " trace: " << trace::DumpSourceLines(node); } // single op cnode. auto primitive = AnfAlgo::GetCNodePrimitive(node); @@ -285,7 +287,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttr(const std::string &old_key, const std::st MS_EXCEPTION_IF_NULL(to); if (!from->isa() || !to->isa()) { MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << " ,to_node is " - << to->DebugString(); + << to->DebugString() << " trace: " << trace::DumpSourceLines(from); } auto from_primitive = AnfAlgo::GetCNodePrimitive(from); MS_EXCEPTION_IF_NULL(from_primitive); @@ -299,7 +301,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr MS_EXCEPTION_IF_NULL(to); if (!from->isa() || !to->isa()) { MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << ",to_node is " - << from->DebugString(); + << from->DebugString() << " trace: " << trace::DumpSourceLines(from); } auto from_primitive = AnfAlgo::GetCNodePrimitive(from); MS_EXCEPTION_IF_NULL(from_primitive); @@ -311,7 +313,8 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr void AnfRuntimeAlgorithm::EraseNodeAttr(const std::string &key, const AnfNodePtr node) { MS_EXCEPTION_IF_NULL(node); if (!node->isa()) { - MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString(); + MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString() + << " trace: " << trace::DumpSourceLines(node); } // single op cnode. auto primitive = AnfAlgo::GetCNodePrimitive(node); @@ -345,13 +348,15 @@ bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const CNodePtr &no size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(node); if (!node->isa()) { - MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString(); + MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString() + << " trace: " << trace::DumpSourceLines(node); } auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); size_t input_num = cnode->inputs().size(); if (input_num == 0) { - MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero"; + MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero" + << " trace: " << trace::DumpSourceLines(node); } // exclude intputs[0],which is value_node storing attr,inputs left are real input return input_num - 1; @@ -380,7 +385,8 @@ std::vector AnfRuntimeAlgorithm::GetAllOutputFormats(const AnfNodeP MS_EXCEPTION_IF_NULL(node); if (!AnfAlgo::IsRealKernel(node)) { MS_LOG(EXCEPTION) << "Not real kernel:" - << "#node [" << node->DebugString() << "]"; + << "#node [" << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } auto kernel_info = static_cast(node->kernel_info()); MS_EXCEPTION_IF_NULL(kernel_info); @@ -394,7 +400,8 @@ std::vector AnfRuntimeAlgorithm::GetAllInputFormats(const AnfNodePt MS_EXCEPTION_IF_NULL(node); if (!AnfAlgo::IsRealKernel(node)) { MS_LOG(EXCEPTION) << "Not real kernel:" - << "#node [" << node->DebugString() << "]"; + << "#node [" << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } auto kernel_info = static_cast(node->kernel_info()); MS_EXCEPTION_IF_NULL(kernel_info); @@ -408,7 +415,8 @@ std::string AnfRuntimeAlgorithm::GetOriginDataFormat(const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(node); if (!AnfAlgo::IsRealKernel(node)) { MS_LOG(EXCEPTION) << "Not real kernel:" - << "#node [" << node->DebugString() << "]"; + << "#node [" << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } auto kernel_info = static_cast(node->kernel_info()); MS_EXCEPTION_IF_NULL(kernel_info); @@ -423,7 +431,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t if (output_idx > GetOutputTensorNum(node)) { MS_LOG(EXCEPTION) << "Output index:" << output_idx << " is out of the node output range :" << GetOutputTensorNum(node) << " #node [" - << node->DebugString() << "]"; + << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!AnfAlgo::IsRealKernel(node)) { return AnfAlgo::GetPrevNodeOutputFormat(node, output_idx); @@ -435,7 +444,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t auto format = build_info->GetOutputFormat(output_idx); if (format == kernel::KernelBuildInfo::kInvalidFormat) { MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]" - << " has a invalid output format"; + << " has a invalid output format" + << " trace: " << trace::DumpSourceLines(node); } return format; } @@ -445,7 +455,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i if (input_idx > GetInputTensorNum(node)) { MS_LOG(EXCEPTION) << "Input index :" << input_idx << " is out of the number node Input range :" << GetInputTensorNum(node) << "#node [" - << node->DebugString() << "]"; + << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!IsRealKernel(node)) { return GetPrevNodeOutputFormat(node, input_idx); @@ -457,7 +468,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i auto format = build_info->GetInputFormat(input_idx); if (format == kernel::KernelBuildInfo::kInvalidFormat) { MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]" - << " has a invalid input format"; + << " has a invalid input format" + << " trace: " << trace::DumpSourceLines(node); } return format; } @@ -465,7 +477,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i KernelWithIndex AnfRuntimeAlgorithm::GetPrevNodeOutput(const AnfNodePtr &anf_node, size_t input_idx) { MS_EXCEPTION_IF_NULL(anf_node); if (!anf_node->isa()) { - MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode."; + MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode." + << " trace: " << trace::DumpSourceLines(anf_node); } auto input_node = AnfAlgo::GetInputNode(anf_node->cast(), input_idx); MS_EXCEPTION_IF_NULL(input_node); @@ -491,13 +504,15 @@ std::vector AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n return TransShapeToSizet(base_shape->cast()); } MS_LOG(EXCEPTION) << "The node " << node->DebugString() << "is a single output node but got index [" << output_idx - << "."; + << "." + << " trace: " << trace::DumpSourceLines(node); } else if (base_shape->isa()) { auto tuple_shape = base_shape->cast(); MS_EXCEPTION_IF_NULL(tuple_shape); if (output_idx >= tuple_shape->size()) { MS_LOG(EXCEPTION) << "Output index " << output_idx << "is larger than output number " << tuple_shape->size() - << " node:" << node->DebugString() << "."; + << " node:" << node->DebugString() << "." + << " trace: " << trace::DumpSourceLines(node); } auto b_shp = (*tuple_shape)[output_idx]; if (b_shp->isa()) { @@ -507,13 +522,15 @@ std::vector AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n } else { MS_LOG(EXCEPTION) << "The output type of ApplyKernel index:" << output_idx << " should be a NoShape , ArrayShape or a TupleShape, but it is " << base_shape->ToString() - << "node :" << node->DebugString() << "."; + << "node :" << node->DebugString() << "." + << " trace: " << trace::DumpSourceLines(node); } } else if (base_shape->isa()) { return std::vector(); } MS_LOG(EXCEPTION) << "The output type of ApplyKernel should be a NoShape , ArrayShape or a TupleShape, but it is " - << base_shape->ToString() << " node : " << node->DebugString(); + << base_shape->ToString() << " node : " << node->DebugString() + << " trace: " << trace::DumpSourceLines(node); } std::vector AnfRuntimeAlgorithm::GetPrevNodeOutputInferShape(const AnfNodePtr &node, size_t input_idx) { @@ -556,7 +573,8 @@ std::vector AnfRuntimeAlgorithm::GetInputReshapeType(const AnfNodePtr &nod if (input_idx > GetInputTensorNum(node)) { MS_LOG(EXCEPTION) << "The index:" << input_idx << " is out of range of the node's input size : " << GetInputTensorNum(node) << "#node[" - << node->DebugString() << "]"; + << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!IsRealKernel(node)) { return GetPrevNodeOutputReshapeType(node, input_idx); @@ -575,7 +593,8 @@ std::vector AnfRuntimeAlgorithm::GetOutputReshapeType(const AnfNodePtr &no MS_EXCEPTION_IF_NULL(node); if (output_idx > GetOutputTensorNum(node)) { MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ " - << GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]"; + << GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!IsRealKernel(node)) { return GetPrevNodeOutputReshapeType(node, output_idx); @@ -631,7 +650,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size MS_EXCEPTION_IF_NULL(node); if (output_idx > GetOutputTensorNum(node)) { MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ " - << GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]"; + << GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!IsRealKernel(node)) { return GetPrevNodeOutputDeviceDataType(node, output_idx); @@ -643,7 +663,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size auto dtype = build_info->GetOutputDeviceType(output_idx); if (dtype == TypeId::kNumberTypeEnd) { MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]" - << " has a invalid dtype"; + << " has a invalid dtype" + << " trace: " << trace::DumpSourceLines(node); } return dtype; } @@ -652,7 +673,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_ MS_EXCEPTION_IF_NULL(node); if (input_idx > GetInputTensorNum(node)) { MS_LOG(EXCEPTION) << "The index [" << input_idx << "] is out of range of the node's input size [ " - << GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]"; + << GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]" + << " trace: " << trace::DumpSourceLines(node); } if (!IsRealKernel(node)) { return GetPrevNodeOutputDeviceDataType(node, 0); @@ -664,7 +686,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_ auto dtype = build_info->GetInputDeviceType(input_idx); if (dtype == TypeId::kNumberTypeEnd) { MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]" - << " has a invalid dtype"; + << " has a invalid dtype" + << " trace: " << trace::DumpSourceLines(node); } return dtype; } @@ -684,7 +707,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node, if (cnode->size() == kNopNodeInputSize) { return AnfRuntimeAlgorithm::GetPrevNodeOutputAddr(cnode, 0); } else { - MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node"; + MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node" + << " trace: " << trace::DumpSourceLines(node); } } auto kernel_info = static_cast(node->kernel_info()); @@ -692,7 +716,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node, auto addr = kernel_info->GetOutputAddr(output_idx); if (addr == nullptr) { MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString() - << " output addr is not exist"; + << " output addr is not exist" + << " trace: " << trace::DumpSourceLines(node); } return addr; } @@ -706,7 +731,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod if (cnode->inputs().size() == kNopNodeInputSize) { return AnfRuntimeAlgorithm::GetPrevNodeMutableOutputAddr(cnode, 0); } else { - MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node."; + MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node." + << " trace: " << trace::DumpSourceLines(node); } } // Critical path performance optimization: `KernelInfo` is unique subclass of `KernelInfoDevice` @@ -714,8 +740,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod MS_EXCEPTION_IF_NULL(kernel_info); auto addr = kernel_info->GetMutableOutputAddr(output_idx); if (addr == nullptr) { - MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString() - << " output addr is not exist"; + MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString() << " output addr is not exist" + << " trace: " << trace::DumpSourceLines(node); } return addr; } @@ -747,7 +773,8 @@ void AnfRuntimeAlgorithm::SetOutputAddr(const DeviceAddressPtr &addr, size_t out auto kernel_info = static_cast(node->kernel_info()); MS_EXCEPTION_IF_NULL(kernel_info); if (!kernel_info->SetOutputAddr(addr, output_idx)) { - MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail"; + MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail." + << " trace: " << trace::DumpSourceLines(node); } } @@ -757,7 +784,8 @@ void AnfRuntimeAlgorithm::SetWorkspaceAddr(const DeviceAddressPtr &addr, size_t auto kernel_info = static_cast(node->kernel_info()); MS_EXCEPTION_IF_NULL(kernel_info); if (!kernel_info->SetWorkspaceAddr(addr, output_idx)) { - MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail"; + MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail。" + << " trace: " << trace::DumpSourceLines(node); } } @@ -769,7 +797,8 @@ DeviceAddress *AnfRuntimeAlgorithm::GetWorkspaceAddr(const AnfNodePtr &node, siz auto addr = kernel_info->GetWorkspaceAddr(output_idx); if (addr == nullptr) { MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString() - << "] workspace addr is not exist"; + << "] workspace addr is not exist" + << " trace: " << trace::DumpSourceLines(node); } return addr; } @@ -781,7 +810,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableWorkspaceAddr(const AnfNodePtr & MS_EXCEPTION_IF_NULL(kernel_info); auto addr = kernel_info->GetMutableWorkspaceAddr(index); if (addr == nullptr) { - MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist"; + MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist" + << " trace: " << trace::DumpSourceLines(node); } return addr; } @@ -791,7 +821,8 @@ void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector & const std::vector> &shapes, AnfNode *node) { MS_EXCEPTION_IF_NULL(node); if (types.size() != shapes.size()) { - MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size(); + MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size() + << " trace: " << trace::DumpSourceLines(node); } if (shapes.empty()) { node->set_abstract(std::make_shared()); @@ -898,7 +929,8 @@ bool AnfRuntimeAlgorithm::IsRealKernel(const AnfNodePtr &node) { auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); if (cnode->inputs().empty()) { - MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString(); + MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString() + << " trace: " << trace::DumpSourceLines(node); } auto input = cnode->inputs()[0]; bool is_virtual_node = IsPrimitive(input, prim::kPrimImageSummary) || IsPrimitive(input, prim::kPrimScalarSummary) || @@ -1007,7 +1039,7 @@ AnfNodePtr AnfRuntimeAlgorithm::GetInputNode(const CNodePtr &node, size_t index) auto get_input_index = index + 1; if (index + 1 >= node->inputs().size()) { MS_LOG(EXCEPTION) << "Input index size " << get_input_index << "but the node input size just" - << node->inputs().size(); + << node->inputs().size() << " trace: " << trace::DumpSourceLines(node); } // input 0 is primitive node return node->input(get_input_index); @@ -1025,7 +1057,8 @@ bool AnfRuntimeAlgorithm::IsFeatureMapOutput(const AnfNodePtr &node) { bool AnfRuntimeAlgorithm::IsFeatureMapInput(const AnfNodePtr &node, size_t input_index) { if (!node->isa()) { - MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map"; + MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map" + << " trace: " << trace::DumpSourceLines(node); } auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); @@ -1116,7 +1149,8 @@ FuncGraphPtr AnfRuntimeAlgorithm::GetValueNodeFuncGraph(const AnfNodePtr &node) std::vector AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const CNodePtr &cnode) { MS_EXCEPTION_IF_NULL(cnode); if (!(AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall) || AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimSwitch))) { - MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node."; + MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node." + << " trace: " << trace::DumpSourceLines(cnode); } if (AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall)) { auto input1 = cnode->input(kCallKernelGraphIndex); @@ -1153,7 +1187,8 @@ std::vector AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) { MS_EXCEPTION_IF_NULL(call_node); if (!CheckPrimitiveType(call_node, prim::kPrimCall)) { - MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString(); + MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString() + << " trace: " << trace::DumpSourceLines(call_node); } auto input1 = call_node->input(1); if (input1->isa()) { @@ -1161,7 +1196,8 @@ bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) { } else if (input1->isa() && AnfAlgo::CheckPrimitiveType(input1, prim::kPrimSwitch)) { return true; } - MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString(); + MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString() + << " trace: " << trace::DumpSourceLines(call_node); } bool AnfRuntimeAlgorithm::IsScalarInput(const CNodePtr &cnode, size_t index) { @@ -1212,7 +1248,8 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) { } else if (output_type_str == "float32") { except_type = kNumberTypeFloat32; } else { - MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str; + MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str + << " trace: " << trace::DumpSourceLines(node); } } @@ -1221,12 +1258,14 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) { TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, size_t input_idx) { if (!node->isa()) { - MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode."; + MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode." + << " trace: " << trace::DumpSourceLines(node); } auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); if (input_idx + 1 >= cnode->inputs().size()) { - MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode); + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode) + << " trace: " << trace::DumpSourceLines(node); } auto input_node = cnode->input(input_idx + 1); MS_EXCEPTION_IF_NULL(input_node); @@ -1240,7 +1279,8 @@ TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, s bool AnfRuntimeAlgorithm::IsCondControlKernel(const CNodePtr &node) { MS_EXCEPTION_IF_NULL(node); if (node->inputs().empty()) { - MS_LOG(EXCEPTION) << "Illegal null input of cnode."; + MS_LOG(EXCEPTION) << "Illegal null input of cnode." + << " trace: " << trace::DumpSourceLines(node); } auto input = node->input(kAnfPrimitiveIndex); return IsPrimitive(input, prim::kPrimLabelGoto) || IsPrimitive(input, prim::kPrimLabelSwitch); @@ -1346,7 +1386,8 @@ std::vector AnfRuntimeAlgorithm::GetOutputMaxShape(const AnfNodePtr &an } else if (shape->isa()) { return {}; } else { - MS_LOG(EXCEPTION) << "Invalid Shape Type"; + MS_LOG(EXCEPTION) << "Invalid Shape Type" + << " trace: " << trace::DumpSourceLines(anf_node); } } @@ -1363,7 +1404,8 @@ std::vector AnfRuntimeAlgorithm::GetOutputMinShape(const AnfNodePtr &an } else if (shape->isa()) { return {}; } else { - MS_LOG(EXCEPTION) << "Invalid Shape Type"; + MS_LOG(EXCEPTION) << "Invalid Shape Type" + << " trace: " << trace::DumpSourceLines(anf_node); } } diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc index 5fef61b3c0..79cdeb5c4b 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc @@ -44,6 +44,7 @@ #include "debug/data_dump/dump_json_parser.h" #include "toolchain/adx_datadump_server.h" #include "utils/shape_utils.h" +#include "utils/trace_base.h" #ifdef MEM_REUSE_DEBUG #include "backend/optimizer/mem_reuse/mem_reuse_checker.h" #endif @@ -111,7 +112,7 @@ std::string GetRankId() { } } // namespace -std::vector AscendKernelRuntime::exception_infos_; +std::vector AscendKernelRuntime::exception_infoes_; AscendKernelRuntime::~AscendKernelRuntime() { graph_model_map_.clear(); } void AscendKernelRuntime::SetContext() { @@ -543,7 +544,7 @@ void AscendKernelRuntime::LaunchDataDump(GraphId graph_id) { void AscendKernelRuntime::ExceptionCallback(rtExceptionInfo *exception_info) { static std::mutex exception_mutex; std::lock_guard lock(exception_mutex); - exception_infos_.push_back(*exception_info); + exception_infoes_.push_back(*exception_info); } void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *graph) { @@ -551,11 +552,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap std::vector full_scope_name{}; // Find node name(full scope name) auto runtime_info_map = ModelRunner::Instance().GetRuntimeInfoMap(graph->graph_id()); - MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infos_.size() << ". first example: " - << ", task_id: " << exception_infos_.at(0).taskid << ", stream_id: " << exception_infos_.at(0).streamid - << ", tid: " << exception_infos_.at(0).tid << ", device_id: " << exception_infos_.at(0).deviceid; + MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infoes_.size() << ". first example: " + << ", task_id: " << exception_infoes_.at(0).taskid + << ", stream_id: " << exception_infoes_.at(0).streamid << ", tid: " << exception_infoes_.at(0).tid + << ", device_id: " << exception_infoes_.at(0).deviceid; - for (const auto &exception_info : exception_infos_) { + for (const auto &exception_info : exception_infoes_) { for (const auto &iter : runtime_info_map) { auto task_id = std::get(*iter.second); auto stream_id = std::get(*iter.second); @@ -566,11 +568,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap } } // Dump error data in local path - const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infos_.at(0).deviceid); + const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infoes_.at(0).deviceid); for (const auto &node : graph->execution_order()) { for (auto &name : full_scope_name) { if (node->fullname_with_scope() == name) { - MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path."; + MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path." + << " trace: " << trace::DumpSourceLines(node); E2eDumpUtil::DumpInputImpl(node, false, local_path, &name, nullptr); E2eDumpUtil::DumpOutputImpl(node, false, local_path, &name, nullptr); } @@ -686,6 +689,7 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { DumpTaskExceptionInfo(graph); return false; } + exception_infoes_.clear(); return true; } diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h index 5622757488..c7822040be 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h @@ -88,8 +88,8 @@ class AscendKernelRuntime : public KernelRuntime { unordered_map>> task_map_; unordered_map> graph_model_map_; unordered_map> graph_data_dumper_; - static std::vector exception_infos_; std::map, std::string> stream_id_task_id_op_name_map_; + static std::vector exception_infoes_; }; MS_REG_KERNEL_RUNTIME(kAscendDevice, AscendKernelRuntime); diff --git a/mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc b/mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc index d212e26750..51e03fa964 100644 --- a/mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc +++ b/mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc @@ -33,7 +33,7 @@ #include "frontend/operator/ops.h" #include "utils/ms_context.h" #include "utils/ms_utils.h" - +#include "utils/trace_base.h" namespace mindspore { namespace device { namespace ascend { @@ -321,7 +321,8 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) { if (!AnfAlgo::HasNodeAttr(kAttrPynativeNextIndex, kernel_node) || !AnfAlgo::HasNodeAttr(kAttrPynativeNextOpName, kernel_node)) { MS_LOG(EXCEPTION) << "The node [" << kernel_node->DebugString() << "] attr of " << kAttrPynativeNextIndex << " or " - << kAttrPynativeNextOpName << " has been not setted yet!"; + << kAttrPynativeNextOpName << " has been not setted yet!" + << " trace: " << trace::DumpSourceLines(kernel_node); } auto next_index = AnfAlgo::GetNodeAttr(kernel_node, kAttrPynativeNextIndex); auto next_op_name = AnfAlgo::GetNodeAttr(kernel_node, kAttrPynativeNextOpName); @@ -332,7 +333,7 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) { } if (iter->second.size() < next_index) { MS_LOG(EXCEPTION) << "Next input index " << next_index << "is out of range in the next op map max size is " - << iter->second.size(); + << iter->second.size() << " trace: " << trace::DumpSourceLines(kernel_node); } if (AnfAlgo::GetCNodeName(kernel_node) != prim::kPrimCast->name()) { MS_LOG(INFO) << "Only supported to change the node Cast's build info!!!"; @@ -469,7 +470,8 @@ KernelSelectStatus SelectKernelInfo(const CNodePtr &kernel_node, KernelType kern MS_LOG(WARNING) << " <<<"; MS_EXCEPTION(TypeError) << "The node [" << kernel_node->DebugString() << "] cannot find valid kernel info, not supported the type:" << buffer.str() - << ", please refer to the supported dtypes in candidates kernel info list"; + << ", please refer to the supported dtypes in candidates kernel info list." + << " trace: " << trace::DumpSourceLines(kernel_node); } } return select_status; diff --git a/mindspore/core/utils/trace_base.cc b/mindspore/core/utils/trace_base.cc index ff83005420..61657ab612 100644 --- a/mindspore/core/utils/trace_base.cc +++ b/mindspore/core/utils/trace_base.cc @@ -111,7 +111,9 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix, So return oss.str(); } -std::string DumpSourceLines(const AnfNodePtr node) { +std::string DumpSourceLines(const AnfNodePtr node) { return DumpSourceLines(node.get()); } + +std::string DumpSourceLines(AnfNode *node) { if (node == nullptr) { MS_LOG(WARNING) << "Node is null"; return ""; diff --git a/mindspore/core/utils/trace_base.h b/mindspore/core/utils/trace_base.h index fa63759f23..56dcee6e45 100644 --- a/mindspore/core/utils/trace_base.h +++ b/mindspore/core/utils/trace_base.h @@ -35,6 +35,7 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix, SourceLineTip tip = kSourceLineTipNextLine); // Generate the call stack of python source code to a string std::string DumpSourceLines(const AnfNodePtr node); +std::string DumpSourceLines(AnfNode *node); // Generate the call stack of python source code to a vector std::vector GetSourceLineList(const AnfNodePtr node); // Get the locations of the call stack of python source code