!26892 ME compiler environment variables normalization

Merge pull request !26892 from chenfei_mindspore/environment_variable_fix

mindspore/_extends/parse/parser.py

@@ -134,7 +134,7 @@ def get_bprop_method_of_class(obj, parse_method=None):
 
 # The fallback feature is enabled in default.
 # Not support change the flag during the process is alive.
-support_fallback_ = os.getenv('ENV_SUPPORT_FALLBACK')
+support_fallback_ = os.getenv('DEV_ENV_ENABLE_FALLBACK')
 
 
 def resolve_symbol(namespace, symbol):
@@ -519,7 +519,7 @@ def eval_script(exp_str, params):
         obj = eval(exp_str, global_params, local_params)
     except Exception as e:
         error_info = f"When eval '{exp_str}' by using Fallback feature, an error occurred: " + str(e) + \
-            ". You can try to turn off the Fallback feature by 'export ENV_SUPPORT_FALLBACK=0'."
+            ". You can try to turn off the Fallback feature by 'export DEV_ENV_ENABLE_FALLBACK=0'."
         logger.error(error_info)
         raise e
 

mindspore/ccsrc/backend/optimizer/common/pass_manager.cc

@@ -130,7 +130,7 @@ void PassManager::DumpPassIR(const FuncGraphPtr &func_graph, const std::string &
   auto context_ptr = MsContext::GetInstance();
   MS_EXCEPTION_IF_NULL(context_ptr);
   bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
-  static const auto enable_dump = (common::GetEnv("ENV_NO_DUMP_BE_PASS_IR") != "1");
+  static const auto enable_dump = !GetDumpConfig().disable_backend_dump;
   if (save_graphs && enable_dump) {
     std::ostringstream oss;
     oss << "verbose_ir_files"

mindspore/ccsrc/debug/anf_ir_dump.cc

@@ -585,16 +585,96 @@ void DumpSubgraph(const OrderedMap<FuncGraphPtr, std::shared_ptr<SubGraphIRInfo>
   }
 }
 
-void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) {
+void SetDumpConfigByString(const std::string &str, DumpConfig *dump_config) {
   static mindspore::HashMap<std::string, enum LocDumpMode> dump_level_map = {
-    {std::to_string(kOff), kOff}, {std::to_string(kTopStack), kTopStack}, {std::to_string(kWholeStack), kWholeStack}};
-  static const auto dump_level_in_env = common::GetEnv("ENV_DUMP_IR_LINE_LEVEL");
-  auto it = dump_level_map.find(dump_level_in_env);
-  if (it == dump_level_map.end()) {
+    {kDumpConfigLineLevel0, kOff}, {kDumpConfigLineLevel1, kTopStack}, {kDumpConfigLineLevel2, kWholeStack}};
+  auto it = dump_level_map.find(str);
+  if (it != dump_level_map.end()) {
+    dump_config->enable_dump_pass_ir = it->second;
+    return;
+  }
+  if (str == kDumpConfigDisableBackend) {
+    dump_config->disable_backend_dump = true;
+    return;
+  }
+  if (str == kDumpConfigEnablePassIR) {
+    dump_config->enable_dump_pass_ir = true;
     return;
   }
-  // Use the env setting instead parameter setting.
-  *dump_location = it->second;
+}
+
+DumpConfig GetDumpConfig() {
+  static std::vector<HashSet<std::string>> config_white_list = {
+    {kDumpConfigLineLevel0, kDumpConfigLineLevel1, kDumpConfigLineLevel2},
+    {kDumpConfigDisableBackend},
+    {kDumpConfigEnablePassIR}};
+  static DumpConfig dump_config;
+  static bool parsed = false;
+  if (parsed) {
+    return dump_config;
+  }
+  parsed = true;
+  // Start parse config.
+  std::string str(common::GetEnv("DEV_ENV_DUMP_IR_CONFIG"));
+  std::vector<std::shared_ptr<HashSet<std::string>>> configs = {std::make_shared<HashSet<std::string>>(),
+                                                                std::make_shared<HashSet<std::string>>(),
+                                                                std::make_shared<HashSet<std::string>>()};
+  auto constexpr max_string_len = 100;
+  if (str.size() > max_string_len) {
+    MS_LOG(WARNING) << "Dump ir config length exceed max length: " << max_string_len;
+    return dump_config;
+  }
+  if (str.empty()) {
+    return dump_config;
+  }
+  size_t start_pos = 0;
+  // if '#' is the last char of str, the str is illegal, so we use '<=' but not '<'.
+  while (start_pos <= str.size()) {
+    auto pos = str.find('#', start_pos);
+    if (pos == std::string::npos) {
+      pos = str.size();
+    }
+    auto substr = str.substr(start_pos, pos - start_pos);
+    start_pos = pos + 1;
+    bool is_illegal_config = true;
+    for (size_t i = 0; i < config_white_list.size(); i++) {
+      if (config_white_list[i].find(substr) != config_white_list[i].end()) {
+        is_illegal_config = false;
+        (void)configs[i]->insert(substr);
+        if (configs[i]->size() > 1) {
+          std::ostringstream buffer;
+          std::for_each(configs[i]->begin(), configs[i]->end(), [&buffer](const std::string &config) {
+            buffer << "\n" << config;
+          });
+          MS_LOG(WARNING) << "Dump configs are conflict. Conflict configs: " << buffer.str() << "\n"
+                          << "Please keep only one of them.";
+          return dump_config;
+        }
+      }
+    }
+    if (is_illegal_config) {
+      std::ostringstream buffer;
+      buffer << "Support configs:\n"
+             << "[0]: " << kDumpConfigLineLevel0 << "\n"
+             << "[1]: " << kDumpConfigLineLevel1 << "\n"
+             << "[2]: " << kDumpConfigLineLevel2 << "\n"
+             << "[3]: " << kDumpConfigDisableBackend << "\n"
+             << "[4]: " << kDumpConfigEnablePassIR;
+      MS_LOG(WARNING) << "Illegal dump config:\n" << substr << "\n" << buffer.str();
+      return {};
+    }
+  }
+  for (auto &config : configs) {
+    SetDumpConfigByString(*config->begin(), &dump_config);
+  }
+  return dump_config;
+}
+
+void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) {
+  const auto &config = GetDumpConfig();
+  if (config.dump_line_level != kInValid) {
+    *dump_location = config.dump_line_level;
+  }
 }
 
 #ifdef ENABLE_DUMP_IR

mindspore/ccsrc/debug/anf_ir_dump.h

@@ -21,9 +21,21 @@
 #include "ir/dtype/type.h"
 #include "ir/anf.h"
 #include "debug/common.h"
+#include "utils/hash_set.h"
 
 namespace mindspore {
-enum LocDumpMode { kOff = 0, kTopStack = 1, kWholeStack = 2 };
+enum LocDumpMode { kOff = 0, kTopStack = 1, kWholeStack = 2, kInValid = 3 };
+auto constexpr kDumpConfigLineLevel0 = "LINE_LEVEL0";
+auto constexpr kDumpConfigLineLevel1 = "LINE_LEVEL1";
+auto constexpr kDumpConfigLineLevel2 = "LINE_LEVEL2";
+auto constexpr kDumpConfigDisableBackend = "DISABLE_BACKEND";
+auto constexpr kDumpConfigEnablePassIR = "ENABLE_PASS_IR";
+typedef struct {
+  LocDumpMode dump_line_level = kInValid;
+  bool disable_backend_dump = false;
+  bool enable_dump_pass_ir = false;
+} DumpConfig;
+
 constexpr char PARALLEL_STRATEGY[] = "strategy";
 void DumpIR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
             LocDumpMode dump_location = kOff, const std::string &target_file = "");
@@ -32,6 +44,7 @@ void GatherInputAndOutputInferType(std::ostringstream &buffer, const AnfNodePtr
 void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
                   LocDumpMode dump_location = kOff);
 const std::string ToShortString(const TypeId &typeId);
+DumpConfig GetDumpConfig();
 }  // namespace mindspore
 
 #endif  // MINDSPORE_CCSRC_DEBUG_ANF_IR_DUMP_H_

mindspore/ccsrc/frontend/optimizer/ad/grad.cc

@@ -80,8 +80,7 @@ FuncGraphPtr Grad(const FuncGraphPtr &func_graph, const pipeline::ResourceBasePt
   manager_ptr->AddFuncGraph(func_graph);
 
   FuncGraphPtr grad_fg = func_graph;
-  lift_fv_before_grad = (common::GetEnv("ENV_DONT_LIFT_FV_BEFORE_GRAD") != "1");
-  if (lift_fv_before_grad && func_graph->func_graphs_used().size() != 0) {
+  if (func_graph->func_graphs_used().size() != 0) {
     grad_fg = LiftFv(resources, func_graph);
   }
   auto multi_graph_sink = [&func_graph](const FuncGraphPtr &f) {

mindspore/ccsrc/frontend/optimizer/opt.cc

@@ -301,7 +301,7 @@ bool SubstitutionList::ApplySubstitutionsToIR(const OptimizerPtr &optimizer, con
       changes = changes || change;
       loop = loop || change;
 #ifdef ENABLE_DUMP_IR
-      static const auto enable_dump_pass_ir = (common::GetEnv("ENV_DUMP_PASS_IR") == "1");
+      static const auto enable_dump_pass_ir = GetDumpConfig().enable_dump_pass_ir;
       if (enable_dump_pass_ir && MsContext::GetInstance()->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG)) {
         auto fg_name = optimizer->name() + "_r" + std::to_string(optimizer->CurPass_.counter) + "_" +
                        optimizer->CurPass_.name + "_" + substitution->name_;
@@ -338,8 +338,8 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize
   manager->AddFuncGraph(func_graph);
   bool changes = false;
   static const auto traverse_mode =
-    (common::GetEnv("ENV_TRAVERSE_SUBSTITUTIONS_MODE") != "1" ? kOptTraverseFromIRToSubstitutions
-                                                              : kOptTraverseFromSubstitutionsToIR);
+    (common::GetEnv("DEV_ENV_TRAVERSE_SUBSTITUTIONS_MODE") != "1" ? kOptTraverseFromIRToSubstitutions
+                                                                  : kOptTraverseFromSubstitutionsToIR);
   if (traverse_mode == kOptTraverseFromIRToSubstitutions &&
       MsContext::GetInstance()->get_param<int>(MS_CTX_EXECUTION_MODE) != kPynativeMode &&
       optimizer->traverse_nodes_first() && !is_once_ && !global_sensitive_) {

mindspore/ccsrc/frontend/optimizer/optimizer.h

@@ -195,7 +195,7 @@ class Optimizer : public std::enable_shared_from_this<Optimizer> {
           };
           use_profile ? (WITH(MsProfile::GetProfile()->Step(pass_names_[i])) opt_func) : opt_func();
 #ifdef ENABLE_DUMP_IR
-          static const auto enable_dump_pass_ir = (common::GetEnv("ENV_DUMP_PASS_IR") == "1");
+          static const auto enable_dump_pass_ir = GetDumpConfig().enable_dump_pass_ir;
           if (enable_dump_pass_ir && MsContext::GetInstance()->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG)) {
             auto fg_name =
               "opt_substep_" + name_ + "_r" + std::to_string(counter) + "_" + std::to_string(i) + "_" + pass_names_[i];

mindspore/ccsrc/pipeline/jit/parse/data_converter.cc

@@ -382,7 +382,7 @@ ValuePtr ConvertOtherObj(const py::object &obj) {
   // Start RESOLVE_TYPE_INVALID...
   // The fallback feature is enabled in default.
   // Not support change the flag during the process is alive.
-  static const auto support_fallback = common::GetEnv("ENV_SUPPORT_FALLBACK");
+  static const auto support_fallback = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
   static const auto use_fallback = (support_fallback != "0");
   if (use_fallback) {
     auto res = std::make_shared<InterpretedObject>(obj, py::str(obj));

mindspore/ccsrc/pipeline/jit/parse/parse.cc

@@ -92,8 +92,8 @@ AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNo
 FuncGraphWeakPtr Parser::top_func_graph_ = FuncGraphWeakPtr();
 
 Parser::Parser(const std::shared_ptr<ParseFunctionAst> &ast) : ast_(ast) {
-  max_for_loop_count_str_ = common::GetEnv("ENV_FOR_TO_WHILE_LOOP");
-  support_fallback_ = common::GetEnv("ENV_SUPPORT_FALLBACK");
+  max_for_loop_count_str_ = common::GetEnv("DEV_ENV_FOR_TO_WHILE_LOOP");
+  support_fallback_ = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
   errcode_ = PARSE_SUCCESS;
   BuildMethodMap();
 }

mindspore/ccsrc/pipeline/jit/parse/resolve.cc

@@ -59,7 +59,7 @@ abstract::AbstractBasePtr ClassType::ToAbstract() {
 
   // The fallback feature is enabled in default.
   // Not support change the flag during the process is alive.
-  static const auto support_fallback = common::GetEnv("ENV_SUPPORT_FALLBACK");
+  static const auto support_fallback = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
   static const auto use_fallback = (support_fallback != "0");
   if (use_fallback && !IsSupportedCreateInstanceType(obj())) {
     return abs_scalar;

mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc

@@ -573,7 +573,7 @@ EvalResultPtr AnalysisEngine::ExecuteEvaluators(const std::vector<EvaluatorPtr>
     MS_EXCEPTION_IF_NULL(eval);
     return eval->Run(shared_from_this(), args_conf_list, out_conf);
   }
-  static bool enable_singleThread = (common::GetEnv("ENV_SINGLE_EVAL") == "1");
+  static bool enable_singleThread = (common::GetEnv("DEV_ENV_SINGLE_EVAL") == "1");
   if (enable_singleThread) {
     return ExecuteMultipleEvaluators(evaluators, out_conf, args_conf_list);
   } else {

mindspore/core/utils/ms_utils.h

@@ -45,7 +45,7 @@ inline const char *SafeCStr(const std::string &str) { return str.c_str(); }
 const char *SafeCStr(const std::string &&str);
 
 static inline std::string GetEnv(const std::string &envvar) {
-  const char *value = ::getenv(envvar.c_str());
+  const char *value = std::getenv(envvar.c_str());
 
   if (value == nullptr) {
     return std::string();

tests/st/control/test_for_to_while.py

@@ -53,7 +53,7 @@ def test_single_for_01():
     y = Tensor([5], mstype.int32)
     z = Tensor([4], mstype.int32)
 
-    os.environ['ENV_FOR_TO_WHILE_LOOP'] = '1'
+    os.environ['DEV_ENV_FOR_TO_WHILE_LOOP'] = '1'
     # graph mode
     context.set_context(mode=context.GRAPH_MODE)
     for_net = SingleForNet()
@@ -67,7 +67,7 @@ def test_single_for_01():
     net = GradNet(for_net)
     pynative_forward_res = for_net(x, y, z)
     pynative_backward_res = net(x, y, z)
-    os.environ['ENV_FOR_TO_WHILE_LOOP'] = ''
+    os.environ['DEV_ENV_FOR_TO_WHILE_LOOP'] = ''
 
     assert graph_forward_res == pynative_forward_res
     assert graph_backward_res == pynative_backward_res