add trace when mindspore error

mindspore/ccsrc/backend/optimizer/ascend/ascend_helper.cc

@@ -27,7 +27,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/session/kernel_graph.h"
 #include "utils/ms_context.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder;
@@ -133,7 +133,7 @@ AnfNodePtr InsertTransOpForSingleOutput(const FuncGraphPtr &func_graph, const An
   std::vector<size_t> origin_shape = AnfAlgo::GetOutputInferShape(node, 0);
   if (output_format == kOpFormat_NC1KHKWHWC0) {
     MS_LOG(EXCEPTION) << "got the hw format " << output_format << "when insert the transdata node "
-                      << node->DebugString();
+                      << node->DebugString() << " trace: " << trace::DumpSourceLines(node);
   }
   if (kCommonFormatSet.find(output_format) == kCommonFormatSet.end() && origin_shape.size() > 1) {
     MS_LOG(DEBUG) << "Inserted Transdata " << output_format << " To default , index :0";
@@ -166,7 +166,7 @@ AnfNodePtr InsertTransOpForMultipleOutput(const FuncGraphPtr &func_graph, const
     std::string output_format = AnfAlgo::GetOutputFormat(node, output_idx);
     if (output_format == kOpFormat_NC1KHKWHWC0) {
       MS_LOG(EXCEPTION) << "Got the special format" << output_format << " when insert the transdata node "
-                        << node->DebugString();
+                        << node->DebugString() << " trace: " << trace::DumpSourceLines(node);
     }
     auto tuple_getitem = CreatTupleGetItemNode(func_graph, node, output_idx);
     std::vector<size_t> origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx);

mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_cascade.cc

@@ -19,6 +19,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "frontend/optimizer/opt.h"
 #include "backend/optimizer/ascend/ascend_helper.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -45,7 +46,8 @@ bool IsPartOutputsOfHcclOp(const AnfNodePtr &node, const CNodePtr &cur_hccl, con
   auto &node_users = manager->node_users();
   auto iter = node_users.find(prev_hccl_op);
   if (iter == node_users.end()) {
-    MS_LOG(EXCEPTION) << "node has no output in manager";
+    MS_LOG(EXCEPTION) << "node has no output in manager"
+                      << " trace: " << trace::DumpSourceLines(cur_hccl);
   }
   for (const auto &node_index : iter->second) {
     AnfNodePtr output = node_index.first;
@@ -79,7 +81,8 @@ AnfNodePtr InsertMemcpyAsyncForCascade::InsertMemcpyAsync(const FuncGraphPtr &gr
     if (IsPartOutputsOfHcclOp(input, hccl_node, graph)) {
       auto memcpy_async = CreateMemcpyAsyncOp(graph, input);
       if (memcpy_async == nullptr) {
-        MS_LOG(EXCEPTION) << "Create memcpy_async op failed.";
+        MS_LOG(EXCEPTION) << "Create memcpy_async op failed."
+                          << " trace: " << trace::DumpSourceLines(hccl_node);
       }
       if (AnfAlgo::IsNodeDynamicShape(input)) {
         AnfAlgo::SetNodeAttr(kAttrIsDynamicShape, MakeValue(true), memcpy_async);

mindspore/ccsrc/backend/optimizer/ascend/enhancer/insert_memcpy_async_for_hccl_op.cc

@@ -21,7 +21,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "frontend/optimizer/opt.h"
 #include "backend/optimizer/ascend/ascend_helper.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 namespace {
@@ -61,7 +61,8 @@ void DealControlForGetitem(const CNodePtr &tuple_getitem, const FuncGraphPtr &gr
   auto &node_users = manager->node_users();
   auto iter = node_users.find(tuple_getitem);
   if (iter == node_users.end()) {
-    MS_LOG(EXCEPTION) << "node has no output in manager";
+    MS_LOG(EXCEPTION) << "node has no output in manager"
+                      << " trace: " << trace::DumpSourceLines(hccl_node);
   }
   for (const auto &node_index : iter->second) {
     AnfNodePtr output = node_index.first;
@@ -81,7 +82,8 @@ void TransferControl(const CNodePtr &hccl_node, const std::vector<AnfNodePtr> &m
   auto &node_users = manager->node_users();
   auto iter = node_users.find(hccl_node);
   if (iter == node_users.end()) {
-    MS_LOG(EXCEPTION) << "node has no output in manager";
+    MS_LOG(EXCEPTION) << "node has no output in manager"
+                      << " trace: " << trace::DumpSourceLines(hccl_node);
   }
   // find hccl_node's output which is a control depend
   for (const auto &node_index : iter->second) {
@@ -140,7 +142,8 @@ bool InsertMemcpyAsyncForHcclOp::NeedInsertMemcpy(const FuncGraphPtr &graph, con
     // when input is used by others
     auto iter = node_users.find(input);
     if (iter == node_users.end()) {
-      MS_LOG(EXCEPTION) << "node has no output in manager";
+      MS_LOG(EXCEPTION) << "node has no output in manager"
+                        << " trace: " << trace::DumpSourceLines(input);
     }
     if (IsNodeOutPutUsedByOtherRealKernel(iter->second)) {
       return true;

mindspore/ccsrc/backend/optimizer/ascend/format_type/split_unsupported_transdata.cc

@@ -18,6 +18,7 @@
 #include <vector>
 #include <memory>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -39,7 +40,7 @@ const AnfNodePtr SplitUnsupportedTransData::Process(const FuncGraphPtr &func_gra
   MS_EXCEPTION_IF_NULL(kernel_info);
   if (kernel_info->GetInputNum() != 1 || kernel_info->GetOutputNum() != 1) {
     MS_LOG(EXCEPTION) << "Transdata node's kernel info's input and output format size is not 1"
-                      << ori_trans_data->DebugString();
+                      << ori_trans_data->DebugString() << trace::DumpSourceLines(node);
   }
   return SplitTransData(func_graph, ori_trans_data);
 }

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_bert_fission.cc

@@ -19,6 +19,7 @@
 #include <algorithm>
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/optimizer/common/helper.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -65,7 +66,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP
   MS_EXCEPTION_IF_NULL(bn_cnode);
   if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
     MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
-                      << kBatchNormRealInputNum + 1;
+                      << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
   }
   std::vector<AnfNodePtr> bn_training_reduce_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)), bn_cnode->input(1)};
@@ -91,11 +92,12 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod
   MS_EXCEPTION_IF_NULL(bn_cnode);
   if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
     MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
-                      << kBatchNormRealInputNum + 1;
+                      << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
   }
   if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
     MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum
-                      << ", but it is " << bn_training_reduce_outputs.size();
+                      << ", but it is " << bn_training_reduce_outputs.size()
+                      << " trace: " << trace::DumpSourceLines(bn);
   }
   std::vector<AnfNodePtr> bn_training_update_v2_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateV2OpName)),
@@ -111,7 +113,7 @@ AnfNodePtr CreateBNTrainingUpdateV2(const FuncGraphPtr &func_graph, const AnfNod
   MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
   if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) {
     MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is "
-                      << bn_abstract_tuple->elements().size();
+                      << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
   }
   std::vector<AbstractBasePtr> abstract_list{bn_abstract_tuple->elements()[0], bn_abstract_tuple->elements()[3],
                                              bn_abstract_tuple->elements()[4]};
@@ -155,7 +157,8 @@ const AnfNodePtr BatchNormBertFission::Process(const FuncGraphPtr &func_graph, c
                                  &bn_training_update_v2_outputs);
   if (bn_training_update_v2_outputs.size() != kBNTrainingUpdateV2OutputNum) {
     MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingUpdateV2OutputNum
-                      << ", but it is " << bn_training_update_v2_outputs.size();
+                      << ", but it is " << bn_training_update_v2_outputs.size()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto manager = func_graph->manager();
   MS_EXCEPTION_IF_NULL(manager);

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_infer_fission.cc

@@ -17,6 +17,7 @@
 #include <vector>
 #include "backend/optimizer/common/helper.h"
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -61,15 +62,18 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func
   // Set inputs
   auto iter_input0 = (*equiv).find(input0_var_);
   if (iter_input0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   auto iter_input2 = (*equiv).find(input2_var_);
   if (iter_input2 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input2 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   auto iter_input4 = (*equiv).find(input4_var_);
   if (iter_input4 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   std::vector<AnfNodePtr> bn_infer_grad_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNInferGradOpName)), utils::cast<AnfNodePtr>(iter_input0->second),
@@ -80,7 +84,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNInferGrad(const FuncGraphPtr &func
   auto bn_grad_abstract_tuple = dyn_cast<abstract::AbstractTuple>(bn_grad->abstract());
   MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple);
   if (bn_grad_abstract_tuple->elements().empty()) {
-    MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty";
+    MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be empty"
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   bn_infer_grad->set_abstract(bn_grad_abstract_tuple->elements()[0]);
   AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_grad, bn_infer_grad);
@@ -97,19 +102,23 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph
   // Set inputs
   auto iter_input0 = (*equiv).find(input0_var_);
   if (iter_input0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   auto iter_input1 = (*equiv).find(input1_var_);
   if (iter_input1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   auto iter_input3 = (*equiv).find(input3_var_);
   if (iter_input3 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input3 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   auto iter_input4 = (*equiv).find(input4_var_);
   if (iter_input4 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the input4 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn_grad);
   }
   std::vector<AnfNodePtr> bn_training_update_grad_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)),
@@ -121,7 +130,8 @@ AnfNodePtr BatchNormGradInferFission::CreateBNTrainingUpdateGrad(const FuncGraph
   auto bn_grad_abstract_tuple = dyn_cast<abstract::AbstractTuple>(bn_grad->abstract());
   MS_EXCEPTION_IF_NULL(bn_grad_abstract_tuple);
   if (bn_grad_abstract_tuple->elements().size() < kBatchNormGradInferOutputNum) {
-    MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3";
+    MS_LOG(EXCEPTION) << "The abstract tuple of node " << bn_grad->DebugString() << "should not be less than 3"
+                      << trace::DumpSourceLines(bn_grad);
   }
   std::vector<AbstractBasePtr> abstract_list{bn_grad_abstract_tuple->elements()[1],
                                              bn_grad_abstract_tuple->elements()[2]};
@@ -160,7 +170,8 @@ const AnfNodePtr BatchNormGradInferFission::Process(const FuncGraphPtr &func_gra
                                  &bn_training_update_grad_outputs);
   if (bn_training_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
     MS_LOG(EXCEPTION) << "The output size of " << bn_training_update_grad << " should be "
-                      << kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size();
+                      << kBNTrainingUpdateGradOutputNum << ", but it is " << bn_training_update_grad_outputs.size()
+                      << trace::DumpSourceLines(node);
   }
   std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_infer_grad,
                                                bn_training_update_grad_outputs[0], bn_training_update_grad_outputs[1]};

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/batch_norm_grad_split.cc

@@ -23,6 +23,7 @@
 #include "backend/optimizer/common/helper.h"
 #include "runtime/device/kernel_info.h"
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
   MS_EXCEPTION_IF_NULL(bn_grad_node);
   const auto &bn_grad_inputs = bn_grad_node->inputs();
   if (bn_grad_inputs.size() < kBNGradInputNum) {
-    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
+    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size."
+                      << " trace: " << trace::DumpSourceLines(bn_grad_node);
   }
   std::vector<AnfNodePtr> bn_update_grad_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2],
@@ -59,10 +61,12 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
   MS_EXCEPTION_IF_NULL(bn_reduce_grad_outputs);
   const auto &bn_grad_inputs = bn_grad_node->inputs();
   if (bn_grad_inputs.size() < kBNGradInputNum) {
-    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
+    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
+                      << " trace: " << trace::DumpSourceLines(bn_grad_node);
   }
   if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
-    MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "BNTrainingReduceGrad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(bn_grad_node);
   }
   std::vector<AnfNodePtr> bn_reduce_grad_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceGradOpName)),
@@ -101,7 +105,8 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con
   auto primitive = AnfAlgo::GetCNodePrimitive(cnode);
   MS_EXCEPTION_IF_NULL(primitive);
   if (!primitive->HasAttr(kAttrIsTraining)) {
-    MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training";
+    MS_LOG(INFO) << "Op BatchNormGrad must have attrs of is_training"
+                 << " trace: " << trace::DumpSourceLines(node);
     return nullptr;
   }
   if (!AnfAlgo::GetNodeAttr<bool>(cnode, kAttrIsTraining)) {
@@ -112,13 +117,15 @@ const AnfNodePtr BatchNormGradSplit::Process(const FuncGraphPtr &func_graph, con
   std::vector<AnfNodePtr> bn_update_grad_outputs;
   CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs);
   if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
-    MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   std::vector<AnfNodePtr> bn_reduce_grad_outputs;
   CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs);
   if (bn_reduce_grad_outputs.size() != kSingleOutputNum) {
-    MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0],

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_grad_split.cc

@@ -23,6 +23,7 @@
 #include "backend/optimizer/common/helper.h"
 #include "runtime/device/kernel_info.h"
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -33,7 +34,8 @@ void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
   MS_EXCEPTION_IF_NULL(bn_grad_node);
   auto bn_grad_inputs = bn_grad_node->inputs();
   if (bn_grad_inputs.size() != kBNGradInputNum) {
-    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
+    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
+                      << " trace: " << trace::DumpSourceLines(bn_grad_node);
   }
   std::vector<AnfNodePtr> bn_update_grad_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2],
@@ -58,7 +60,8 @@ void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_gra
   MS_EXCEPTION_IF_NULL(bn_grad_node);
   auto bn_grad_inputs = bn_grad_node->inputs();
   if (bn_grad_inputs.size() != kBNGradInputNum) {
-    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size";
+    MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"
+                      << " trace: " << trace::DumpSourceLines(bn_grad_node);
   }
   if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
     MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
@@ -90,13 +93,15 @@ CNodePtr BNGradSplitForTBE(const FuncGraphPtr &func_graph, const CNodePtr &cnode
   std::vector<AnfNodePtr> bn_update_grad_outputs;
   CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs);
   if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) {
-    MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(cnode);
   }
 
   std::vector<AnfNodePtr> bn_reduce_grad_outputs;
   CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs);
   if (bn_reduce_grad_outputs.size() != 1) {
-    MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(cnode);
   }
 
   std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0],

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/bn_split.cc

@@ -23,6 +23,7 @@
 #include "backend/optimizer/common/helper.h"
 #include "runtime/device/kernel_info.h"
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -64,10 +65,12 @@ AnfNodePtr CreateOutputsOfBNTrainingUpdate(const FuncGraphPtr &graph, const CNod
   MS_EXCEPTION_IF_NULL(graph);
   MS_EXCEPTION_IF_NULL(bn_cnode);
   if (bn_cnode->inputs().size() != kBnInputNum) {
-    MS_LOG(EXCEPTION) << "BN node has wrong input size";
+    MS_LOG(EXCEPTION) << "BN node has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(bn_cnode);
   }
   if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
-    MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size";
+    MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(bn_cnode);
   }
   // the inputs of BNTrainingUpdate are from the outputs of BNTrainingReduce and the inputs of BN
   std::vector<AnfNodePtr> bn_training_update_inputs = {
@@ -110,7 +113,8 @@ AnfNodePtr SplitFusedBatchNormForTBE(const FuncGraphPtr &func_graph, const AnfNo
     return nullptr;
   }
   if (bn_training_reduce_outputs.size() != kBN1OutputNum) {
-    MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail";
+    MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   // Create BNTrainingUpdate node

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/concat_fission.cc

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -47,7 +48,8 @@ AnfNodePtr CreateNewConcat(const FuncGraphPtr &func_graph, const CNodePtr &origi
   }
   auto output_shape = AnfAlgo::GetOutputInferShape(origin_concat_cnode, 0);
   if (axis < 0 || axis >= SizeToLong(output_shape.size()) || axis >= SizeToLong(input_shape.size())) {
-    MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range";
+    MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"
+                      << " trace: " << trace::DumpSourceLines(origin_concat_cnode);
   }
   output_shape[axis] = input_shape[axis] * offset;
   AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(origin_concat_cnode, 0)}, {output_shape},

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/dynamic_rnn_grad_fission_v2.cc

@@ -17,6 +17,7 @@
 #include <vector>
 #include <memory>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -290,7 +291,8 @@ AnfNodePtr CreateHConcat(const FuncGraphPtr &func_graph, const CNodePtr &dynamic
   std::vector<AnfNodePtr> splitv_outputs;
   CreateMultipleOutputsOfAnfNode(func_graph, splitv, kSplitVOutputNum, &splitv_outputs);
   if (splitv_outputs.size() != kSplitVOutputNum) {
-    MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed";
+    MS_LOG(EXCEPTION) << "Create outputs of node " << splitv->DebugString() << " failed"
+                      << " trace: " << trace::DumpSourceLines(dynamic_rnn_grad_cnode);
   }
   auto origin_input4 = dynamic_rnn_grad_cnode->input(5);
   auto origin_input4_shape = AnfAlgo::GetOutputInferShape(origin_input4, 0);

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/lars_v2_fission.cc

@@ -19,6 +19,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/optimizer/common/helper.h"
 #include "utils/utils.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -28,7 +29,8 @@ void CreateOutputsOfSquareSumAll(const FuncGraphPtr &graph, const CNodePtr &lars
   MS_EXCEPTION_IF_NULL(graph);
   MS_EXCEPTION_IF_NULL(lars_v2);
   if (lars_v2->size() != kLarsV2InputNum) {
-    MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum;
+    MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum
+                      << " trace: " << trace::DumpSourceLines(lars_v2);
   }
 
   std::vector<AnfNodePtr> inputs = {NewValueNode(std::make_shared<Primitive>(kSquareSumAllOpName)), lars_v2->input(1),
@@ -50,10 +52,12 @@ CNodePtr CreateLarsV2Update(const FuncGraphPtr &graph, const CNodePtr &lars_v2,
   MS_EXCEPTION_IF_NULL(graph);
   MS_EXCEPTION_IF_NULL(lars_v2);
   if (square_sum_all_outputs.size() != 2) {
-    MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2";
+    MS_LOG(EXCEPTION) << "square_sum_all_outputs' size not equal 2"
+                      << " trace: " << trace::DumpSourceLines(lars_v2);
   }
   if (lars_v2->size() != kLarsV2InputNum) {
-    MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum;
+    MS_LOG(EXCEPTION) << "Op lars_v2's input not equal " << kLarsV2InputNum
+                      << " trace: " << trace::DumpSourceLines(lars_v2);
   }
   std::vector<AnfNodePtr> inputs = {NewValueNode(std::make_shared<Primitive>(kLarsV2UpdateOpName)),
                                     lars_v2->input(1),

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/layer_norm_grad_split.cc

@@ -22,6 +22,7 @@
 #include "runtime/device/kernel_info.h"
 #include "ir/primitive.h"
 #include "utils/utils.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -98,14 +99,16 @@ const AnfNodePtr LayerNormGradSplit::Process(const FuncGraphPtr &graph, const An
   std::vector<AnfNodePtr> layer_norm_x_backprop_outputs;
   CreateOutputsOfLayerNormXBackprop(graph, cnode, &layer_norm_x_backprop_outputs);
   if (layer_norm_x_backprop_outputs.size() != kSingleOutputNum) {
-    MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   // create layer_norm_beta_gamma_backprop
   std::vector<AnfNodePtr> layer_norm_beta_gamma_backprop_outputs;
   CreateOutputsOfLayerNormBetaGammaBackprop(graph, cnode, &layer_norm_beta_gamma_backprop_outputs);
   if (layer_norm_beta_gamma_backprop_outputs.size() != kLayerNormBetaGammaBackpropOutputNum) {
-    MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size";
+    MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   std::vector<AnfNodePtr> make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), layer_norm_x_backprop_outputs[0],

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/pack_fission.cc

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -45,7 +46,8 @@ AnfNodePtr CreateNewPack(const FuncGraphPtr &func_graph, const CNodePtr &origin_
     axis += output_shape.size();
   }
   if (axis < 0) {
-    MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range";
+    MS_LOG(EXCEPTION) << "The concat_dim value " << axis << "is out of range"
+                      << " trace: " << trace::DumpSourceLines(origin_pack_cnode);
   }
   std::vector<size_t> new_shape;
   for (size_t i = 0; i < output_shape.size() + 1; ++i) {

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/single_batch_norm_fission.cc

@@ -18,6 +18,7 @@
 #include <memory>
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/optimizer/common/helper.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -31,7 +32,7 @@ AnfNodePtr CreateBNTrainingReduce(const FuncGraphPtr &func_graph, const AnfNodeP
   MS_EXCEPTION_IF_NULL(bn_cnode);
   if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
     MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
-                      << kBatchNormRealInputNum + 1;
+                      << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
   }
   std::vector<AnfNodePtr> bn_training_reduce_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)), bn_cnode->input(1)};
@@ -56,11 +57,12 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod
   MS_EXCEPTION_IF_NULL(bn_cnode);
   if (bn_cnode->inputs().size() < kBatchNormRealInputNum + 1) {
     MS_LOG(EXCEPTION) << "The input size of node " + bn_cnode->DebugString() + " is less than "
-                      << kBatchNormRealInputNum + 1;
+                      << kBatchNormRealInputNum + 1 << " trace: " << trace::DumpSourceLines(bn);
   }
   if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) {
     MS_LOG(EXCEPTION) << "The output size of node bn_training_reduce must be " << kBNTrainingReduceOutputNum
-                      << ", but it is " << bn_training_reduce_outputs.size();
+                      << ", but it is " << bn_training_reduce_outputs.size()
+                      << " trace: " << trace::DumpSourceLines(bn);
   }
   std::vector<AnfNodePtr> bn_training_update_v3_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingUpdateV3OpName)),
@@ -76,7 +78,7 @@ AnfNodePtr CreateBNTrainingUpdateV3(const FuncGraphPtr &func_graph, const AnfNod
   MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
   if (bn_abstract_tuple->elements().size() != kBatchNormOutputNum) {
     MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBatchNormOutputNum << ", but it is "
-                      << bn_abstract_tuple->elements().size();
+                      << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
   }
   bn_training_update_v3->set_abstract(bn->abstract());
   bn_training_update_v3->set_scope(bn->scope());

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/split_fission.cc

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -35,7 +36,7 @@ CNodePtr CreateBaseSplitVNode(const FuncGraphPtr &func_graph, const CNodePtr &or
   MS_EXCEPTION_IF_NULL(origin_cnode);
   if (origin_cnode->inputs().size() < kSplitInputNum) {
     MS_LOG(EXCEPTION) << "The input number of split: " << origin_cnode->DebugString() << " should be "
-                      << kSplitInputNum - 1;
+                      << kSplitInputNum - 1 << " trace: " << trace::DumpSourceLines(origin_cnode);
   }
   return CreateSplitVNode(func_graph, origin_cnode->input(1));
 }

mindspore/ccsrc/backend/optimizer/ascend/ir_fission/transdata_split.cc

@@ -18,6 +18,7 @@
 #include "backend/optimizer/ascend/ascend_helper.h"
 #include "backend/session/anf_runtime_algorithm.h"
 #include "debug/anf_ir_dump.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -99,7 +100,8 @@ bool TransDataSplit::DoSplit(const FuncGraphPtr &func_graph, const AnfNodePtr &n
   manager->AddFuncGraph(func_graph);
 
   if (!manager->Replace(node, new_replace_node)) {
-    MS_LOG(EXCEPTION) << "Manager replace node failed";
+    MS_LOG(EXCEPTION) << "Manager replace node failed"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   MS_LOG(INFO) << "Transdata node:" << cnode->DebugString() << "split success.";
   return true;

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.cc

@@ -16,6 +16,7 @@
 #include "backend/optimizer/ascend/ir_fusion/adam_apply_one_fusion.h"
 #include "backend/optimizer/common/helper.h"
 #include "backend/session/anf_runtime_algorithm.h"
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 const BaseRef AdamApplyOneFusion::DefinePattern() const {
@@ -218,7 +219,8 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
   if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) {
     auto iter_sub0 = (*equiv).find(sub0_var_);
     if (iter_sub0 == (*equiv).end()) {
-      MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched.";
+      MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."
+                        << " trace: " << trace::DumpSourceLines(node);
     }
     sub0 = utils::cast<AnfNodePtr>(iter_sub0->second);
   }
@@ -233,11 +235,13 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
   AbstractBasePtrList new_node_abstract_list;
   auto iter_add0 = (*equiv).find(add0_var_);
   if (iter_add0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto iter_add1 = (*equiv).find(add1_var_);
   if (iter_add1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto add0 = utils::cast<AnfNodePtr>(iter_add0->second);
   MS_EXCEPTION_IF_NULL(add0);
@@ -253,7 +257,7 @@ const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, con
   CreateMultipleOutputsOfAnfNode(func_graph, new_node, kAdamApplyOneOutputNum, &new_node_outputs);
   if (new_node_outputs.size() != kAdamApplyOneOutputNum) {
     MS_LOG(EXCEPTION) << "The output size of node " << new_node->DebugString() << " should be "
-                      << kAdamApplyOneOutputNum;
+                      << kAdamApplyOneOutputNum << " trace: " << trace::DumpSourceLines(node);
   }
   auto manager = func_graph->manager();
   MS_EXCEPTION_IF_NULL(manager);

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc

@@ -21,6 +21,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "ir/primitive.h"
 #include "backend/optimizer/common/helper.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -281,7 +282,8 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c
   if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend)) {
     auto iter_sub0 = (*equiv).find(sub0_var_);
     if (iter_sub0 == (*equiv).end()) {
-      MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched.";
+      MS_LOG(EXCEPTION) << "The equiv map is expected to contains the sub0 var after matched."
+                        << " trace: " << trace::DumpSourceLines(node);
     }
     sub0 = utils::cast<AnfNodePtr>(iter_sub0->second);
   }
@@ -296,11 +298,13 @@ const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, c
 
   auto iter_add0 = (*equiv).find(add0_var_);
   if (iter_add0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto iter_add1 = (*equiv).find(add1_var_);
   if (iter_add1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto add0 = utils::cast<AnfNodePtr>(iter_add0->second);
   MS_EXCEPTION_IF_NULL(add0);

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/add_input_to_output.cc

@@ -19,6 +19,7 @@
 #include "backend/optimizer/ascend/ir_fusion/input_to_output_registry.h"
 #include "backend/session/anf_runtime_algorithm.h"
 #include "backend/kernel_compiler/oplib/oplib.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -107,7 +108,8 @@ const AnfNodePtr AddInputToOutput::Process(const FuncGraphPtr &func_graph, const
   MS_EXCEPTION_IF_NULL(new_abstract_tuple);
   CreateMultipleOutputsOfAnfNode(func_graph, cnode, new_abstract_tuple->size(), &new_outputs);
   if (new_outputs.size() != new_abstract_tuple->size()) {
-    MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString();
+    MS_LOG(EXCEPTION) << "Failed to create outputs of " << cnode->DebugString()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return new_outputs[0];
 }

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/confusion_mul_grad_fusion.cc

@@ -24,7 +24,7 @@
 #include "utils/utils.h"
 #include "abstract/abstract_value.h"
 #include "backend/optimizer/common/helper.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 namespace {
@@ -58,7 +58,8 @@ AnfNodePtr GetMul0(const FuncGraphPtr &graph, const AnfNodePtr &input2, const An
   auto manager = graph->manager();
   MS_EXCEPTION_IF_NULL(manager);
   if (manager->node_users().find(input2) == manager->node_users().end()) {
-    MS_LOG(EXCEPTION) << "node has no output in manager";
+    MS_LOG(EXCEPTION) << "node has no output in manager"
+                      << " trace: " << trace::DumpSourceLines(input2);
   }
 
   AnfNodePtr mul0 = nullptr;

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/fused_batch_norm_fusion.cc

@@ -19,6 +19,7 @@
 #include "backend/optimizer/common/helper.h"
 #include "backend/session/anf_runtime_algorithm.h"
 #include "utils/utils.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace opt {
@@ -41,7 +42,8 @@ void GetBNOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &bn, std::vect
   auto manager = func_graph->manager();
   MS_EXCEPTION_IF_NULL(manager);
   if (manager->node_users().find(bn) == manager->node_users().end()) {
-    MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs";
+    MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs"
+                      << " trace: " << trace::DumpSourceLines(bn);
   }
   for (const auto &node_index : manager->node_users()[bn]) {
     const AnfNodePtr &output = node_index.first;
@@ -113,7 +115,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func
   // Set input to create node
   auto iter_data_input0 = (*equiv).find(data_input0_var_);
   if (iter_data_input0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   std::vector<AnfNodePtr> bn_training_reduce_inputs = {
     NewValueNode(std::make_shared<Primitive>(kBNTrainingReduceOpName)),
@@ -124,13 +127,15 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingReduce(const FuncGraphPtr &func
   // Set abstract
   auto iter_data_input1 = (*equiv).find(data_input1_var_);
   if (iter_data_input1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto data_input1 = utils::cast<AnfNodePtr>(iter_data_input1->second);
   MS_EXCEPTION_IF_NULL(data_input1);
   auto iter_data_input2 = (*equiv).find(data_input2_var_);
   if (iter_data_input2 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the data_input2 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto data_input2 = utils::cast<AnfNodePtr>(iter_data_input2->second);
   MS_EXCEPTION_IF_NULL(data_input2);
@@ -190,17 +195,19 @@ void FusedBatchNormFusion::GetBNTrainingUpdateAbstractList(const EquivPtr &equiv
   MS_EXCEPTION_IF_NULL(bn_abstract_tuple);
   if (bn_abstract_tuple->elements().size() < kBnOutputNum) {
     MS_LOG(EXCEPTION) << "The abstract size of node bn must not be less than " << kBnOutputNum << ", but it is "
-                      << bn_abstract_tuple->elements().size();
+                      << bn_abstract_tuple->elements().size() << " trace: " << trace::DumpSourceLines(bn);
   }
   auto iter_variable_input0 = (*equiv).find(variable_input0_var_);
   if (iter_variable_input0 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input0 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn);
   }
   auto variable_input0 = utils::cast<AnfNodePtr>(iter_variable_input0->second);
   MS_EXCEPTION_IF_NULL(variable_input0);
   auto iter_variable_input1 = (*equiv).find(variable_input1_var_);
   if (iter_variable_input1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the variable_input1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(bn);
   }
   auto variable_input1 = utils::cast<AnfNodePtr>(iter_variable_input1->second);
   MS_EXCEPTION_IF_NULL(variable_input1);
@@ -222,7 +229,8 @@ AnfNodePtr FusedBatchNormFusion::CreateBNTrainingUpdate(
   // Set abstract
   auto iter_batch_norm = (*equiv).find(batch_norm_var_);
   if (iter_batch_norm == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfNodePtr bn = utils::cast<AnfNodePtr>(iter_batch_norm->second);
   MS_EXCEPTION_IF_NULL(bn);
@@ -260,12 +268,13 @@ const AnfNodePtr FusedBatchNormFusion::Process(const FuncGraphPtr &func_graph, c
                                  &bn_training_update_outputs);
   if (bn_training_update_outputs.size() < kBNTrainingUpdateOutputNum) {
     MS_LOG(EXCEPTION) << "The output size of node bn must be " << kBNTrainingUpdateOutputNum << ", but it is "
-                      << bn_training_update_outputs.size();
+                      << bn_training_update_outputs.size() << " trace: " << trace::DumpSourceLines(node);
   }
   // Replace old bn outputs with new outputs
   auto iter_batch_norm = (*equiv).find(batch_norm_var_);
   if (iter_batch_norm == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the batch_norm var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfNodePtr bn = utils::cast<AnfNodePtr>(iter_batch_norm->second);
   std::vector<AnfNodePtr> bn_outputs;

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc

@@ -17,7 +17,7 @@
 #include <utility>
 #include "backend/session/anf_runtime_algorithm.h"
 #include "frontend/optimizer/opt.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 AnfNodePtr LambNextMVWithDecayRule::GetLambNextMVWithDecayOutput(const FuncGraphPtr &func_graph,
@@ -97,7 +97,8 @@ const AnfNodePtr LambNextMVWithDecayRule::Process(const FuncGraphPtr &func_graph
   auto manager = func_graph->manager();
   MS_EXCEPTION_IF_NULL(manager);
   if (manager->node_users().find(mul4) == manager->node_users().end()) {
-    MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input";
+    MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfNodeIndexSet mul4_outputs = manager->node_users()[mul4];
   auto iter = std::find_if(mul4_outputs.begin(), mul4_outputs.end(),

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc

@@ -22,7 +22,7 @@
 
 #include "backend/session/anf_runtime_algorithm.h"
 #include "frontend/optimizer/opt.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 namespace {
@@ -31,27 +31,31 @@ std::tuple<AnfNodePtr, AnfNodePtr, AnfNodePtr, AnfNodePtr> GetSharedNodes(const
   auto add3 = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(add3);
   if (add3->inputs().size() < kAddInputNum) {
-    MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum;
+    MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto real_div2_anf = add3->input(1);
   MS_EXCEPTION_IF_NULL(real_div2_anf);
   auto real_div2 = real_div2_anf->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(real_div2);
   if (real_div2->inputs().size() < kRealDivInputNum) {
-    MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum;
+    MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto sqrt0_anf = real_div2->input(2);
   MS_EXCEPTION_IF_NULL(sqrt0_anf);
   auto sqrt0 = sqrt0_anf->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(sqrt0);
   if (sqrt0->inputs().size() < kRsqrtInputNum) {
-    MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum;
+    MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto add2_anf = sqrt0->input(1);
   MS_EXCEPTION_IF_NULL(add2_anf);
   auto add2 = add2_anf->cast<CNodePtr>();
   if (add2->inputs().size() < kAddInputNum) {
-    MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum;
+    MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return std::make_tuple(add3->input(2), real_div2->input(1), add2->input(1), add2->input(2));
 }
@@ -101,10 +105,12 @@ std::tuple<AnfNodePtr, AnfNodePtr> GetAdd0Add1Nodes(const AnfNodePtr &real_div0_
   MS_EXCEPTION_IF_NULL(real_div0);
   MS_EXCEPTION_IF_NULL(real_div1);
   if (real_div0->inputs().size() != kRealDivInputNum) {
-    MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size";
+    MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(real_div0_anf);
   }
   if (real_div1->inputs().size() != kRealDivInputNum) {
-    MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size";
+    MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(real_div1_anf);
   }
   return std::make_tuple(real_div0->input(1), real_div1->input(1));
 }
@@ -165,7 +171,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra
   auto manager = func_graph->manager();
   MS_EXCEPTION_IF_NULL(manager);
   if (manager->node_users().find(mul4) == manager->node_users().end()) {
-    MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input";
+    MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfNodeIndexSet mul4_output_node_index_set = manager->node_users()[mul4];
   auto iter = std::find_if(
@@ -195,7 +202,8 @@ const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_gra
   std::vector<AnfNodePtr> fusion_node_outputs;
   CreateMultipleOutputsOfAnfNode(func_graph, fusion_node, kLambNextMVWithDecayV1OutputNum, &fusion_node_outputs);
   if (fusion_node_outputs.size() != kLambNextMVWithDecayV1OutputNum) {
-    MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!";
+    MS_LOG(EXCEPTION) << "create multiple outputs for fusion node fail!"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 
   (void)manager->Replace(add0, fusion_node_outputs[1]);

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.cc

@@ -16,7 +16,7 @@
 #include "backend/optimizer/ascend/ir_fusion/lamb_next_right_rule.h"
 #include <vector>
 #include "backend/optimizer/common/helper.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 AnfNodePtr LambNextRightRule::CreateLambNextRightNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const {
@@ -69,7 +69,8 @@ const AnfNodePtr LambNextRightRule::Process(const FuncGraphPtr &func_graph, cons
   // Set abstract of new node
   auto iter_add1 = (*equiv).find(add1_var_);
   if (iter_add1 == (*equiv).end()) {
-    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched.";
+    MS_LOG(EXCEPTION) << "The equiv map is expected to contains the add1 var after matched."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto add1 = utils::cast<AnfNodePtr>(iter_add1->second);
   MS_EXCEPTION_IF_NULL(add1);

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc

@@ -17,7 +17,7 @@
 #include <memory>
 #include <vector>
 #include "backend/session/anf_runtime_algorithm.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 using common::SafeCStr;
@@ -36,7 +36,7 @@ void GetOutputCastNodes(const FuncGraphPtr &func_graph, const AnfNodePtr &node,
     MS_EXCEPTION_IF_NULL(output_cnode);
     if (AnfAlgo::GetCNodeName(output_cnode) != prim::kPrimTupleGetItem->name()) {
       MS_LOG(EXCEPTION) << "The output of node " << node->DebugString() << " should be "
-                        << prim::kPrimTupleGetItem->name();
+                        << prim::kPrimTupleGetItem->name() << " trace: " << trace::DumpSourceLines(node);
     }
     if (manager->node_users().find(output) == manager->node_users().end() ||
         manager->node_users()[output].size() != 1) {
@@ -149,11 +149,13 @@ const AnfNodePtr LayerNormBetaGammaBackpropFusion::Process(const FuncGraphPtr &f
   MS_EXCEPTION_IF_NULL(cast_nodes[0]);
   MS_EXCEPTION_IF_NULL(cast_nodes[1]);
   if (cast_nodes[0]->inputs().size() != kCastInputNum) {
-    MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum;
+    MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   (void)manager->Replace(cast_nodes[0], cast_nodes[0]->input(1));
   if (cast_nodes[1]->inputs().size() != kCastInputNum) {
-    MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum;
+    MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   (void)manager->Replace(cast_nodes[1], cast_nodes[1]->input(1));
   return nullptr;

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/matmul_biasadd_fusion.cc

@@ -18,7 +18,7 @@
 #include "backend/optimizer/common/helper.h"
 #include "backend/session/anf_runtime_algorithm.h"
 #include "utils/utils.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 const BaseRef MatmulBiasaddFusion::DefinePattern() const {
@@ -43,7 +43,8 @@ const AnfNodePtr MatmulBiasaddFusion::Process(const FuncGraphPtr &graph, const A
 
   auto matmul = GetAnfNodeByVar(equiv, matmul_var_);
   if (matmul == nullptr || !matmul->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << "Get CNode MatMul failed!";
+    MS_LOG(EXCEPTION) << "Get CNode MatMul failed!"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfAlgo::CopyNodeAttrs(matmul, new_node);
   return new_node;

mindspore/ccsrc/backend/optimizer/ascend/ir_fusion/square_sum_fusion.cc

@@ -25,7 +25,7 @@
 #include "base/core_ops.h"
 #include "backend/optimizer/common/helper.h"
 #include "runtime/device/kernel_info.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace opt {
 namespace {
@@ -34,7 +34,8 @@ CNodePtr GenerateSquareSumV1(const FuncGraphPtr &graph, const CNodePtr &square,
   MS_EXCEPTION_IF_NULL(square);
   MS_EXCEPTION_IF_NULL(sum);
   if (square->inputs().size() != kSquareNodeInputNum) {
-    MS_LOG(EXCEPTION) << "Square node has wrong input size";
+    MS_LOG(EXCEPTION) << "Square node has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(square);
   }
   auto prim = std::make_shared<Primitive>(kSquareSumV1OpName);
   MS_EXCEPTION_IF_NULL(prim);
@@ -60,7 +61,8 @@ CNodePtr GenerateSquareSumV2(const FuncGraphPtr &graph, const CNodePtr &square,
   MS_EXCEPTION_IF_NULL(square);
   MS_EXCEPTION_IF_NULL(sum);
   if (square->inputs().size() != kSquareNodeInputNum) {
-    MS_LOG(EXCEPTION) << "Square node has wrong input size";
+    MS_LOG(EXCEPTION) << "Square node has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(square);
   }
   auto prim = std::make_shared<Primitive>(kSquareSumV2OpName);
   MS_EXCEPTION_IF_NULL(prim);
@@ -83,7 +85,8 @@ std::tuple<CNodePtr, AnfNodePtr, CNodePtr> GetPrevNodes(const AnfNodePtr &node)
   auto sum = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(sum);
   if (sum->inputs().size() != kSumNodeInputNum) {
-    MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size";
+    MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size"
+                      << " trace: " << trace::DumpSourceLines(sum);
   }
   auto square_anf = sum->input(1);
   MS_EXCEPTION_IF_NULL(square_anf);
@@ -111,7 +114,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo
   auto manager = graph->manager();
   MS_EXCEPTION_IF_NULL(manager);
   if (manager->node_users().find(square_anf) == manager->node_users().end()) {
-    MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap";
+    MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   AnfNodePtr ret_node = nullptr;
   if (manager->node_users()[square_anf].size() == 1) {
@@ -122,7 +126,8 @@ const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNo
     std::vector<AnfNodePtr> square_sumv2_outputs;
     CreateMultipleOutputsOfAnfNode(graph, square_sumv2, kSquareSumv2OutputNum, &square_sumv2_outputs);
     if (square_sumv2_outputs.size() != kSquareSumv2OutputNum) {
-      MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail";
+      MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail"
+                        << " trace: " << trace::DumpSourceLines(square_sumv2);
     }
     (void)manager->Replace(square, square_sumv2_outputs[1]);
     ret_node = square_sumv2_outputs[0];

mindspore/ccsrc/backend/session/anf_runtime_algorithm.cc

@@ -31,6 +31,7 @@
 #include "common/trans.h"
 #include "abstract/param_validator.h"
 #include "pipeline/jit/static_analysis/static_analysis.h"
+#include "utils/trace_base.h"
 
 namespace mindspore {
 namespace session {
@@ -164,7 +165,7 @@ KernelWithIndex AnfRuntimeAlgorithm::VisitKernelWithReturnType(const AnfNodePtr
   }
   if (opt::IsNopNode(cnode) && visit_nop_node) {
     if (cnode->size() != kNopNodeInputSize) {
-      MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString();
+      MS_LOG(EXCEPTION) << "Invalid nop node " << cnode->DebugString() << " trace: " << trace::DumpSourceLines(cnode);
     }
     return VisitKernelWithReturnType(cnode->input(kNopNodeRealInputIndex), 0, visit_nop_node, return_types);
   }
@@ -250,7 +251,7 @@ std::string AnfRuntimeAlgorithm::GetCNodeName(const AnfNodePtr &node) {
     }
     return func_graph->ToString();
   }
-  MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString();
+  MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString() << " trace: " << trace::DumpSourceLines(node);
 }
 
 std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) {
@@ -261,7 +262,8 @@ std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) {
 void AnfRuntimeAlgorithm::SetNodeAttr(const std::string &key, const ValuePtr &value, const AnfNodePtr &node) {
   MS_EXCEPTION_IF_NULL(node);
   if (!node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString();
+    MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   // single op cnode.
   auto primitive = AnfAlgo::GetCNodePrimitive(node);
@@ -285,7 +287,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttr(const std::string &old_key, const std::st
   MS_EXCEPTION_IF_NULL(to);
   if (!from->isa<CNode>() || !to->isa<CNode>()) {
     MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << " ,to_node is "
-                      << to->DebugString();
+                      << to->DebugString() << " trace: " << trace::DumpSourceLines(from);
   }
   auto from_primitive = AnfAlgo::GetCNodePrimitive(from);
   MS_EXCEPTION_IF_NULL(from_primitive);
@@ -299,7 +301,7 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr
   MS_EXCEPTION_IF_NULL(to);
   if (!from->isa<CNode>() || !to->isa<CNode>()) {
     MS_LOG(EXCEPTION) << "Only cnode has attr, but this from_anf is " << from->DebugString() << ",to_node is "
-                      << from->DebugString();
+                      << from->DebugString() << " trace: " << trace::DumpSourceLines(from);
   }
   auto from_primitive = AnfAlgo::GetCNodePrimitive(from);
   MS_EXCEPTION_IF_NULL(from_primitive);
@@ -311,7 +313,8 @@ void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr
 void AnfRuntimeAlgorithm::EraseNodeAttr(const std::string &key, const AnfNodePtr node) {
   MS_EXCEPTION_IF_NULL(node);
   if (!node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString();
+    MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node->DebugString()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   // single op cnode.
   auto primitive = AnfAlgo::GetCNodePrimitive(node);
@@ -345,13 +348,15 @@ bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const CNodePtr &no
 size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) {
   MS_EXCEPTION_IF_NULL(node);
   if (!node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString();
+    MS_LOG(EXCEPTION) << "Only cnode has real input, but this anf is " << node->DebugString()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto cnode = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(cnode);
   size_t input_num = cnode->inputs().size();
   if (input_num == 0) {
-    MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero";
+    MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   // exclude intputs[0],which is value_node storing attr,inputs left are real input
   return input_num - 1;
@@ -380,7 +385,8 @@ std::vector<std::string> AnfRuntimeAlgorithm::GetAllOutputFormats(const AnfNodeP
   MS_EXCEPTION_IF_NULL(node);
   if (!AnfAlgo::IsRealKernel(node)) {
     MS_LOG(EXCEPTION) << "Not real kernel:"
-                      << "#node [" << node->DebugString() << "]";
+                      << "#node [" << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
   MS_EXCEPTION_IF_NULL(kernel_info);
@@ -394,7 +400,8 @@ std::vector<std::string> AnfRuntimeAlgorithm::GetAllInputFormats(const AnfNodePt
   MS_EXCEPTION_IF_NULL(node);
   if (!AnfAlgo::IsRealKernel(node)) {
     MS_LOG(EXCEPTION) << "Not real kernel:"
-                      << "#node [" << node->DebugString() << "]";
+                      << "#node [" << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
   MS_EXCEPTION_IF_NULL(kernel_info);
@@ -408,7 +415,8 @@ std::string AnfRuntimeAlgorithm::GetOriginDataFormat(const AnfNodePtr &node) {
   MS_EXCEPTION_IF_NULL(node);
   if (!AnfAlgo::IsRealKernel(node)) {
     MS_LOG(EXCEPTION) << "Not real kernel:"
-                      << "#node [" << node->DebugString() << "]";
+                      << "#node [" << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
   MS_EXCEPTION_IF_NULL(kernel_info);
@@ -423,7 +431,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t
   if (output_idx > GetOutputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "Output index:" << output_idx
                       << " is out of the node output range :" << GetOutputTensorNum(node) << " #node ["
-                      << node->DebugString() << "]";
+                      << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!AnfAlgo::IsRealKernel(node)) {
     return AnfAlgo::GetPrevNodeOutputFormat(node, output_idx);
@@ -435,7 +444,8 @@ std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t
   auto format = build_info->GetOutputFormat(output_idx);
   if (format == kernel::KernelBuildInfo::kInvalidFormat) {
     MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
-                      << " has a invalid output format";
+                      << " has a invalid output format"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return format;
 }
@@ -445,7 +455,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
   if (input_idx > GetInputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "Input index :" << input_idx
                       << " is out of the number node Input range :" << GetInputTensorNum(node) << "#node ["
-                      << node->DebugString() << "]";
+                      << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!IsRealKernel(node)) {
     return GetPrevNodeOutputFormat(node, input_idx);
@@ -457,7 +468,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
   auto format = build_info->GetInputFormat(input_idx);
   if (format == kernel::KernelBuildInfo::kInvalidFormat) {
     MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
-                      << " has a invalid input format";
+                      << " has a invalid input format"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return format;
 }
@@ -465,7 +477,8 @@ std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t i
 KernelWithIndex AnfRuntimeAlgorithm::GetPrevNodeOutput(const AnfNodePtr &anf_node, size_t input_idx) {
   MS_EXCEPTION_IF_NULL(anf_node);
   if (!anf_node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode.";
+    MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode."
+                      << " trace: " << trace::DumpSourceLines(anf_node);
   }
   auto input_node = AnfAlgo::GetInputNode(anf_node->cast<CNodePtr>(), input_idx);
   MS_EXCEPTION_IF_NULL(input_node);
@@ -491,13 +504,15 @@ std::vector<size_t> AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n
       return TransShapeToSizet(base_shape->cast<abstract::ShapePtr>());
     }
     MS_LOG(EXCEPTION) << "The node " << node->DebugString() << "is a single output node but got index [" << output_idx
-                      << ".";
+                      << "."
+                      << " trace: " << trace::DumpSourceLines(node);
   } else if (base_shape->isa<abstract::TupleShape>()) {
     auto tuple_shape = base_shape->cast<abstract::TupleShapePtr>();
     MS_EXCEPTION_IF_NULL(tuple_shape);
     if (output_idx >= tuple_shape->size()) {
       MS_LOG(EXCEPTION) << "Output index " << output_idx << "is larger than output number " << tuple_shape->size()
-                        << " node:" << node->DebugString() << ".";
+                        << " node:" << node->DebugString() << "."
+                        << " trace: " << trace::DumpSourceLines(node);
     }
     auto b_shp = (*tuple_shape)[output_idx];
     if (b_shp->isa<abstract::Shape>()) {
@@ -507,13 +522,15 @@ std::vector<size_t> AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &n
     } else {
       MS_LOG(EXCEPTION) << "The output type of ApplyKernel index:" << output_idx
                         << " should be a NoShape , ArrayShape or a TupleShape, but it is " << base_shape->ToString()
-                        << "node :" << node->DebugString() << ".";
+                        << "node :" << node->DebugString() << "."
+                        << " trace: " << trace::DumpSourceLines(node);
     }
   } else if (base_shape->isa<abstract::NoShape>()) {
     return std::vector<size_t>();
   }
   MS_LOG(EXCEPTION) << "The output type of ApplyKernel should be a NoShape , ArrayShape or a TupleShape, but it is "
-                    << base_shape->ToString() << " node : " << node->DebugString();
+                    << base_shape->ToString() << " node : " << node->DebugString()
+                    << " trace: " << trace::DumpSourceLines(node);
 }
 
 std::vector<size_t> AnfRuntimeAlgorithm::GetPrevNodeOutputInferShape(const AnfNodePtr &node, size_t input_idx) {
@@ -556,7 +573,8 @@ std::vector<Axis> AnfRuntimeAlgorithm::GetInputReshapeType(const AnfNodePtr &nod
   if (input_idx > GetInputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "The index:" << input_idx
                       << " is out of range of the node's input size : " << GetInputTensorNum(node) << "#node["
-                      << node->DebugString() << "]";
+                      << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!IsRealKernel(node)) {
     return GetPrevNodeOutputReshapeType(node, input_idx);
@@ -575,7 +593,8 @@ std::vector<Axis> AnfRuntimeAlgorithm::GetOutputReshapeType(const AnfNodePtr &no
   MS_EXCEPTION_IF_NULL(node);
   if (output_idx > GetOutputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ "
-                      << GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]";
+                      << GetOutputTensorNum(node) << "#node[ " << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!IsRealKernel(node)) {
     return GetPrevNodeOutputReshapeType(node, output_idx);
@@ -631,7 +650,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size
   MS_EXCEPTION_IF_NULL(node);
   if (output_idx > GetOutputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "The index [" << output_idx << "] is out of range of the node's output size [ "
-                      << GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]";
+                      << GetOutputTensorNum(node) << "#node [ " << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!IsRealKernel(node)) {
     return GetPrevNodeOutputDeviceDataType(node, output_idx);
@@ -643,7 +663,8 @@ TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size
   auto dtype = build_info->GetOutputDeviceType(output_idx);
   if (dtype == TypeId::kNumberTypeEnd) {
     MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
-                      << " has a invalid dtype";
+                      << " has a invalid dtype"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return dtype;
 }
@@ -652,7 +673,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_
   MS_EXCEPTION_IF_NULL(node);
   if (input_idx > GetInputTensorNum(node)) {
     MS_LOG(EXCEPTION) << "The index [" << input_idx << "] is out of range of the node's input size [ "
-                      << GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]";
+                      << GetInputTensorNum(node) << "#node [ " << node->DebugString() << "]"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (!IsRealKernel(node)) {
     return GetPrevNodeOutputDeviceDataType(node, 0);
@@ -664,7 +686,8 @@ TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_
   auto dtype = build_info->GetInputDeviceType(input_idx);
   if (dtype == TypeId::kNumberTypeEnd) {
     MS_LOG(EXCEPTION) << "Node [" << node->DebugString() << "]"
-                      << " has a invalid dtype";
+                      << " has a invalid dtype"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return dtype;
 }
@@ -684,7 +707,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node,
     if (cnode->size() == kNopNodeInputSize) {
       return AnfRuntimeAlgorithm::GetPrevNodeOutputAddr(cnode, 0);
     } else {
-      MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node";
+      MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node"
+                        << " trace: " << trace::DumpSourceLines(node);
     }
   }
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
@@ -692,7 +716,8 @@ const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node,
   auto addr = kernel_info->GetOutputAddr(output_idx);
   if (addr == nullptr) {
     MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString()
-                      << " output addr is not exist";
+                      << " output addr is not exist"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return addr;
 }
@@ -706,7 +731,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod
     if (cnode->inputs().size() == kNopNodeInputSize) {
       return AnfRuntimeAlgorithm::GetPrevNodeMutableOutputAddr(cnode, 0);
     } else {
-      MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node.";
+      MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node."
+                        << " trace: " << trace::DumpSourceLines(node);
     }
   }
   // Critical path performance optimization: `KernelInfo` is unique subclass of `KernelInfoDevice`
@@ -714,8 +740,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &nod
   MS_EXCEPTION_IF_NULL(kernel_info);
   auto addr = kernel_info->GetMutableOutputAddr(output_idx);
   if (addr == nullptr) {
-    MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString()
-                      << " output addr is not exist";
+    MS_LOG(EXCEPTION) << "Output_idx" << output_idx << " of node " << node->DebugString() << " output addr is not exist"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return addr;
 }
@@ -747,7 +773,8 @@ void AnfRuntimeAlgorithm::SetOutputAddr(const DeviceAddressPtr &addr, size_t out
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
   MS_EXCEPTION_IF_NULL(kernel_info);
   if (!kernel_info->SetOutputAddr(addr, output_idx)) {
-    MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail";
+    MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 }
 
@@ -757,7 +784,8 @@ void AnfRuntimeAlgorithm::SetWorkspaceAddr(const DeviceAddressPtr &addr, size_t
   auto kernel_info = static_cast<device::KernelInfo *>(node->kernel_info());
   MS_EXCEPTION_IF_NULL(kernel_info);
   if (!kernel_info->SetWorkspaceAddr(addr, output_idx)) {
-    MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail";
+    MS_LOG(EXCEPTION) << "Node " << node->DebugString() << "set adr" << output_idx << " fail。"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
 }
 
@@ -769,7 +797,8 @@ DeviceAddress *AnfRuntimeAlgorithm::GetWorkspaceAddr(const AnfNodePtr &node, siz
   auto addr = kernel_info->GetWorkspaceAddr(output_idx);
   if (addr == nullptr) {
     MS_LOG(EXCEPTION) << "Output_idx " << output_idx << " of node " << node->DebugString()
-                      << "] workspace addr is not exist";
+                      << "] workspace addr is not exist"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return addr;
 }
@@ -781,7 +810,8 @@ DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableWorkspaceAddr(const AnfNodePtr &
   MS_EXCEPTION_IF_NULL(kernel_info);
   auto addr = kernel_info->GetMutableWorkspaceAddr(index);
   if (addr == nullptr) {
-    MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist";
+    MS_LOG(EXCEPTION) << "Index " << index << " of node " << node->DebugString() << "] workspace addr is not exist"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   return addr;
 }
@@ -791,7 +821,8 @@ void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector<TypeId> &
                                                      const std::vector<std::vector<size_t>> &shapes, AnfNode *node) {
   MS_EXCEPTION_IF_NULL(node);
   if (types.size() != shapes.size()) {
-    MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size();
+    MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   if (shapes.empty()) {
     node->set_abstract(std::make_shared<abstract::AbstractNone>());
@@ -898,7 +929,8 @@ bool AnfRuntimeAlgorithm::IsRealKernel(const AnfNodePtr &node) {
   auto cnode = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(cnode);
   if (cnode->inputs().empty()) {
-    MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString();
+    MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString()
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto input = cnode->inputs()[0];
   bool is_virtual_node = IsPrimitive(input, prim::kPrimImageSummary) || IsPrimitive(input, prim::kPrimScalarSummary) ||
@@ -1007,7 +1039,7 @@ AnfNodePtr AnfRuntimeAlgorithm::GetInputNode(const CNodePtr &node, size_t index)
   auto get_input_index = index + 1;
   if (index + 1 >= node->inputs().size()) {
     MS_LOG(EXCEPTION) << "Input index size " << get_input_index << "but the node input size just"
-                      << node->inputs().size();
+                      << node->inputs().size() << " trace: " << trace::DumpSourceLines(node);
   }
   // input 0 is primitive node
   return node->input(get_input_index);
@@ -1025,7 +1057,8 @@ bool AnfRuntimeAlgorithm::IsFeatureMapOutput(const AnfNodePtr &node) {
 
 bool AnfRuntimeAlgorithm::IsFeatureMapInput(const AnfNodePtr &node, size_t input_index) {
   if (!node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map";
+    MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature map"
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto cnode = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(cnode);
@@ -1116,7 +1149,8 @@ FuncGraphPtr AnfRuntimeAlgorithm::GetValueNodeFuncGraph(const AnfNodePtr &node)
 std::vector<KernelGraphPtr> AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const CNodePtr &cnode) {
   MS_EXCEPTION_IF_NULL(cnode);
   if (!(AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall) || AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimSwitch))) {
-    MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node.";
+    MS_LOG(EXCEPTION) << "Node: " << cnode->DebugString() << "is not a call or switch node."
+                      << " trace: " << trace::DumpSourceLines(cnode);
   }
   if (AnfAlgo::CheckPrimitiveType(cnode, prim::kPrimCall)) {
     auto input1 = cnode->input(kCallKernelGraphIndex);
@@ -1153,7 +1187,8 @@ std::vector<KernelGraphPtr> AnfRuntimeAlgorithm::GetCallSwitchKernelGraph(const
 bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) {
   MS_EXCEPTION_IF_NULL(call_node);
   if (!CheckPrimitiveType(call_node, prim::kPrimCall)) {
-    MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString();
+    MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString()
+                      << " trace: " << trace::DumpSourceLines(call_node);
   }
   auto input1 = call_node->input(1);
   if (input1->isa<ValueNode>()) {
@@ -1161,7 +1196,8 @@ bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) {
   } else if (input1->isa<CNode>() && AnfAlgo::CheckPrimitiveType(input1, prim::kPrimSwitch)) {
     return true;
   }
-  MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString();
+  MS_LOG(EXCEPTION) << "Unexpected input1 of call node,input1:" << input1->DebugString()
+                    << " trace: " << trace::DumpSourceLines(call_node);
 }
 
 bool AnfRuntimeAlgorithm::IsScalarInput(const CNodePtr &cnode, size_t index) {
@@ -1212,7 +1248,8 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) {
     } else if (output_type_str == "float32") {
       except_type = kNumberTypeFloat32;
     } else {
-      MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str;
+      MS_LOG(EXCEPTION) << "The fix precision must be float16 or float32, but got " << output_type_str
+                        << " trace: " << trace::DumpSourceLines(node);
     }
   }
 
@@ -1221,12 +1258,14 @@ TypeId AnfRuntimeAlgorithm::GetCNodeOutputPrecision(const AnfNodePtr &node) {
 
 TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, size_t input_idx) {
   if (!node->isa<CNode>()) {
-    MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode.";
+    MS_LOG(EXCEPTION) << node->DebugString() << ", input node is not CNode."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto cnode = node->cast<CNodePtr>();
   MS_EXCEPTION_IF_NULL(cnode);
   if (input_idx + 1 >= cnode->inputs().size()) {
-    MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode);
+    MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode)
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto input_node = cnode->input(input_idx + 1);
   MS_EXCEPTION_IF_NULL(input_node);
@@ -1240,7 +1279,8 @@ TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputPrecision(const AnfNodePtr &node, s
 bool AnfRuntimeAlgorithm::IsCondControlKernel(const CNodePtr &node) {
   MS_EXCEPTION_IF_NULL(node);
   if (node->inputs().empty()) {
-    MS_LOG(EXCEPTION) << "Illegal null input of cnode.";
+    MS_LOG(EXCEPTION) << "Illegal null input of cnode."
+                      << " trace: " << trace::DumpSourceLines(node);
   }
   auto input = node->input(kAnfPrimitiveIndex);
   return IsPrimitive(input, prim::kPrimLabelGoto) || IsPrimitive(input, prim::kPrimLabelSwitch);
@@ -1346,7 +1386,8 @@ std::vector<int64_t> AnfRuntimeAlgorithm::GetOutputMaxShape(const AnfNodePtr &an
   } else if (shape->isa<abstract::NoShape>()) {
     return {};
   } else {
-    MS_LOG(EXCEPTION) << "Invalid Shape Type";
+    MS_LOG(EXCEPTION) << "Invalid Shape Type"
+                      << " trace: " << trace::DumpSourceLines(anf_node);
   }
 }
 
@@ -1363,7 +1404,8 @@ std::vector<int64_t> AnfRuntimeAlgorithm::GetOutputMinShape(const AnfNodePtr &an
   } else if (shape->isa<abstract::NoShape>()) {
     return {};
   } else {
-    MS_LOG(EXCEPTION) << "Invalid Shape Type";
+    MS_LOG(EXCEPTION) << "Invalid Shape Type"
+                      << " trace: " << trace::DumpSourceLines(anf_node);
   }
 }
 

mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc

@@ -44,6 +44,7 @@
 #include "debug/data_dump/dump_json_parser.h"
 #include "toolchain/adx_datadump_server.h"
 #include "utils/shape_utils.h"
+#include "utils/trace_base.h"
 #ifdef MEM_REUSE_DEBUG
 #include "backend/optimizer/mem_reuse/mem_reuse_checker.h"
 #endif
@@ -111,7 +112,7 @@ std::string GetRankId() {
 }
 }  // namespace
 
-std::vector<rtExceptionInfo> AscendKernelRuntime::exception_infos_;
+std::vector<rtExceptionInfo> AscendKernelRuntime::exception_infoes_;
 AscendKernelRuntime::~AscendKernelRuntime() { graph_model_map_.clear(); }
 
 void AscendKernelRuntime::SetContext() {
@@ -543,7 +544,7 @@ void AscendKernelRuntime::LaunchDataDump(GraphId graph_id) {
 void AscendKernelRuntime::ExceptionCallback(rtExceptionInfo *exception_info) {
   static std::mutex exception_mutex;
   std::lock_guard<std::mutex> lock(exception_mutex);
-  exception_infos_.push_back(*exception_info);
+  exception_infoes_.push_back(*exception_info);
 }
 
 void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *graph) {
@@ -551,11 +552,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap
   std::vector<std::string> full_scope_name{};
   // Find node name(full scope name)
   auto runtime_info_map = ModelRunner::Instance().GetRuntimeInfoMap(graph->graph_id());
-  MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infos_.size() << ". first example: "
-                << ", task_id: " << exception_infos_.at(0).taskid << ", stream_id: " << exception_infos_.at(0).streamid
-                << ", tid: " << exception_infos_.at(0).tid << ", device_id: " << exception_infos_.at(0).deviceid;
+  MS_LOG(ERROR) << "Exception_infos_ size: " << exception_infoes_.size() << ". first example: "
+                << ", task_id: " << exception_infoes_.at(0).taskid
+                << ", stream_id: " << exception_infoes_.at(0).streamid << ", tid: " << exception_infoes_.at(0).tid
+                << ", device_id: " << exception_infoes_.at(0).deviceid;
 
-  for (const auto &exception_info : exception_infos_) {
+  for (const auto &exception_info : exception_infoes_) {
     for (const auto &iter : runtime_info_map) {
       auto task_id = std::get<kTupleTaskId>(*iter.second);
       auto stream_id = std::get<kTupleStreamId>(*iter.second);
@@ -566,11 +568,12 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap
     }
   }
   // Dump error data in local path
-  const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infos_.at(0).deviceid);
+  const std::string local_path = std::string("./task_error_dump/") + std::to_string(exception_infoes_.at(0).deviceid);
   for (const auto &node : graph->execution_order()) {
     for (auto &name : full_scope_name) {
       if (node->fullname_with_scope() == name) {
-        MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path.";
+        MS_LOG(ERROR) << "Begin to dump node (" << name << ") task error input/output data in local path."
+                      << " trace: " << trace::DumpSourceLines(node);
         E2eDumpUtil::DumpInputImpl(node, false, local_path, &name, nullptr);
         E2eDumpUtil::DumpOutputImpl(node, false, local_path, &name, nullptr);
       }
@@ -686,6 +689,7 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) {
     DumpTaskExceptionInfo(graph);
     return false;
   }
+  exception_infoes_.clear();
   return true;
 }
 

mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.h

@@ -88,8 +88,8 @@ class AscendKernelRuntime : public KernelRuntime {
   unordered_map<GraphId, vector<std::shared_ptr<TaskInfo>>> task_map_;
   unordered_map<GraphId, std::shared_ptr<ge::model_runner::DavinciModel>> graph_model_map_;
   unordered_map<GraphId, std::shared_ptr<DataDumper>> graph_data_dumper_;
-  static std::vector<rtExceptionInfo> exception_infos_;
   std::map<std::pair<uint32_t, uint32_t>, std::string> stream_id_task_id_op_name_map_;
+  static std::vector<rtExceptionInfo> exception_infoes_;
 };
 
 MS_REG_KERNEL_RUNTIME(kAscendDevice, AscendKernelRuntime);

mindspore/ccsrc/runtime/device/ascend/kernel_select_ascend.cc

@@ -33,7 +33,7 @@
 #include "frontend/operator/ops.h"
 #include "utils/ms_context.h"
 #include "utils/ms_utils.h"
-
+#include "utils/trace_base.h"
 namespace mindspore {
 namespace device {
 namespace ascend {
@@ -321,7 +321,8 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) {
   if (!AnfAlgo::HasNodeAttr(kAttrPynativeNextIndex, kernel_node) ||
       !AnfAlgo::HasNodeAttr(kAttrPynativeNextOpName, kernel_node)) {
     MS_LOG(EXCEPTION) << "The node [" << kernel_node->DebugString() << "] attr of " << kAttrPynativeNextIndex << " or "
-                      << kAttrPynativeNextOpName << " has been not setted yet!";
+                      << kAttrPynativeNextOpName << " has been not setted yet!"
+                      << " trace: " << trace::DumpSourceLines(kernel_node);
   }
   auto next_index = AnfAlgo::GetNodeAttr<size_t>(kernel_node, kAttrPynativeNextIndex);
   auto next_op_name = AnfAlgo::GetNodeAttr<std::string>(kernel_node, kAttrPynativeNextOpName);
@@ -332,7 +333,7 @@ void SetCastAndWeightFormat(const CNodePtr &kernel_node) {
   }
   if (iter->second.size() < next_index) {
     MS_LOG(EXCEPTION) << "Next input index " << next_index << "is out of range in the next op map max size is "
-                      << iter->second.size();
+                      << iter->second.size() << " trace: " << trace::DumpSourceLines(kernel_node);
   }
   if (AnfAlgo::GetCNodeName(kernel_node) != prim::kPrimCast->name()) {
     MS_LOG(INFO) << "Only supported to change the node Cast's build info!!!";
@@ -469,7 +470,8 @@ KernelSelectStatus SelectKernelInfo(const CNodePtr &kernel_node, KernelType kern
       MS_LOG(WARNING) << " <<<";
       MS_EXCEPTION(TypeError) << "The node [" << kernel_node->DebugString()
                               << "] cannot find valid kernel info, not supported the type:" << buffer.str()
-                              << ", please refer to the supported dtypes in candidates kernel info list";
+                              << ", please refer to the supported dtypes in candidates kernel info list."
+                              << " trace: " << trace::DumpSourceLines(kernel_node);
     }
   }
   return select_status;

mindspore/core/utils/trace_base.cc

@@ -111,7 +111,9 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix, So
   return oss.str();
 }
 
-std::string DumpSourceLines(const AnfNodePtr node) {
+std::string DumpSourceLines(const AnfNodePtr node) { return DumpSourceLines(node.get()); }
+
+std::string DumpSourceLines(AnfNode *node) {
   if (node == nullptr) {
     MS_LOG(WARNING) << "Node is null";
     return "";

mindspore/core/utils/trace_base.h

@@ -35,6 +35,7 @@ std::string GetDebugInfo(const DebugInfoPtr &info, const std::string &prefix,
                          SourceLineTip tip = kSourceLineTipNextLine);
 // Generate the call stack of python source code to a string
 std::string DumpSourceLines(const AnfNodePtr node);
+std::string DumpSourceLines(AnfNode *node);
 // Generate the call stack of python source code to a vector
 std::vector<std::string> GetSourceLineList(const AnfNodePtr node);
 // Get the locations of the call stack of python source code