diff --git a/mindspore/ccsrc/debug/trace.cc b/mindspore/ccsrc/debug/trace.cc index 8f5bf7bbef..7c25ad67d6 100644 --- a/mindspore/ccsrc/debug/trace.cc +++ b/mindspore/ccsrc/debug/trace.cc @@ -73,7 +73,7 @@ std::string GetGraphParamString(const FuncGraphPtr &graph, abstract::AbstractBas } void DumpInferStack(std::ostringstream &oss) { - auto &infer_stack = GetCurrenGraphInferStack(); + auto &infer_stack = GetCurrenGraphEvalStack(); if (infer_stack.empty()) { return; } @@ -104,7 +104,7 @@ void DumpInferStack(std::ostringstream &oss) { } void TraceGraphEval() { - auto &infer_stack = GetCurrenGraphInferStack(); + auto &infer_stack = GetCurrenGraphEvalStack(); std::ostringstream oss; if (infer_stack.empty()) { return; @@ -551,9 +551,10 @@ void TraceEvalCNodeLeave() { cnode_debug_stack.pop_back(); } std::vector &GetCNodeDebugStack() { return cnode_debug_stack; } -std::stack> &GetCurrenGraphInferStack() { +std::stack> &GetCurrenGraphEvalStack() { return graph_infer_stack; } + void ClearTraceStack() { while (!graph_infer_stack.empty()) { graph_infer_stack.pop(); diff --git a/mindspore/ccsrc/debug/trace.h b/mindspore/ccsrc/debug/trace.h index 19add25876..496d03e162 100644 --- a/mindspore/ccsrc/debug/trace.h +++ b/mindspore/ccsrc/debug/trace.h @@ -40,7 +40,7 @@ void TraceGraphEvalLeave(const abstract::EvaluatorPtr &eval); void TraceEvalCNodeEnter(const abstract::AnfNodeConfigPtr &node_cfg); void TraceEvalCNodeLeave(); std::vector &GetCNodeDebugStack(); -std::stack> &GetCurrenGraphInferStack(); +std::stack> &GetCurrenGraphEvalStack(); std::string GetAbstractStr(const abstract::AbstractBasePtr &abs); void ClearTraceStack(); } // namespace trace diff --git a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc index eb0031a783..119b6e19a8 100644 --- a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc +++ b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc @@ -30,6 +30,7 @@ #include "pipeline/jit/parse/data_converter.h" #include "pipeline/jit/static_analysis/evaluator.h" #include "debug/trace.h" +#include "debug/anf_ir_dump.h" namespace mindspore { namespace abstract { @@ -181,6 +182,7 @@ EvalResultPtr AnalysisEngine::Eval(const AnfNodeConfigPtr &conf) { auto value_node = node->cast(); eval_result = std::make_shared(EvalValueNode(value_node, conf), nullptr); } else if (node->isa()) { + CheckNoStackInSameFuncGraph(conf); auto cnode = node->cast(); trace::TraceEvalCNodeEnter(conf); eval_result = EvalCNode(cnode, conf); @@ -202,6 +204,44 @@ EvalResultPtr AnalysisEngine::Eval(const AnfNodeConfigPtr &conf) { return eval_result; } +void AnalysisEngine::CheckNoStackInSameFuncGraph(const AnfNodeConfigPtr &conf) { + auto &list = trace::GetCNodeDebugStack(); + if (list.empty()) { + return; + } + auto &previous_stack = list.back(); + auto previous_cnode_fg = previous_stack->node()->func_graph(); + auto current_cnode_fg = conf->node()->func_graph(); + if (previous_cnode_fg != current_cnode_fg) { // Normal. + return; + } + if (forward_count_ != 0) { // Ignore Forward Config. + return; + } + auto &infer_stack = trace::GetCurrenGraphEvalStack(); + if (infer_stack.empty()) { + return; + } + auto top_evaluator = infer_stack.top().first; + if (!top_evaluator->isa()) { + MS_LOG(EXCEPTION) << "Top evaluator is " << top_evaluator->ToString(); + } + auto top_fg_evaluator = dyn_cast(top_evaluator); + auto top_context_fg = top_fg_evaluator->graph_context()->func_graph(); + if (current_cnode_fg != top_context_fg) { // Ignore FV call. + return; + } + MS_LOG(ERROR) << "Should not use call stack in the same function: " << top_context_fg->ToString() << ", for " + << conf->node()->DebugString(2); + for (size_t i = 0; i < list.size(); ++i) { + auto old_conf = list[i]; + MS_LOG(ERROR) << " #" << i << ": " << old_conf->node()->DebugString(2) << ", in " + << old_conf->context()->func_graph()->ToString(); + } + DumpIR("use_stack_error.ir", conf->node()->func_graph()); + MS_LOG(EXCEPTION) << "To check above CNode stack and dumped use_stack_error.ir"; +} + AbstractBasePtr AnalysisEngine::EvalValueNode(const ValueNodePtr &value_node, const AnfNodeConfigPtr &conf) { MS_EXCEPTION_IF_NULL(conf); MS_EXCEPTION_IF_NULL(value_node); diff --git a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h index bba403bcfd..31d65c8e66 100644 --- a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h +++ b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h @@ -187,6 +187,7 @@ class AnalysisEngine : public std::enable_shared_from_this { AnalysisEngine(const PrimEvaluatorMap &prim_evaluator_map, const FuncGraphManagerPtr &func_graph_manager) : cache_(AnalysisCache()), prim_constructors_(prim_evaluator_map), func_graph_manager_(func_graph_manager) { function_call_depth_ = 0; + forward_count_ = 0; } ~AnalysisEngine() = default; @@ -226,7 +227,10 @@ class AnalysisEngine : public std::enable_shared_from_this { (void)anfnode_config_map_.emplace(orig_conf, new_conf); MS_LOG(DEBUG) << "Forward orig_conf: " << orig_conf->node()->DebugString() << ", to new_conf: " << new_conf->node()->DebugString(); - return GetEvaluatedValue(new_conf); + forward_count_++; + auto res = GetEvaluatedValue(new_conf); + forward_count_--; + return res; } const PrimEvaluatorMap &PrimConstructors() const { return prim_constructors_; } @@ -246,6 +250,8 @@ class AnalysisEngine : public std::enable_shared_from_this { uint64_t function_call_depth() { return function_call_depth_; } + void CheckNoStackInSameFuncGraph(const AnfNodeConfigPtr &conf); + private: void SetUndeterminedFlag(const EvaluatorPtr &evaluator); EvaluatorPtr HandleNestedRecursion(const std::vector &evaluators, const EvaluatorPtr &eval, @@ -275,6 +281,8 @@ class AnalysisEngine : public std::enable_shared_from_this { // record current depth of function call statck uint64_t function_call_depth_; + uint64_t forward_count_; + #ifdef DEBUG std::vector compute_conf_stack_; #endif