when env variable RANK_ID changed,RDR files should be following

mindspore/ccsrc/debug/CMakeLists.txt

@@ -6,6 +6,7 @@ set(_DEBUG_SRC_LIST
     "${CMAKE_CURRENT_SOURCE_DIR}/anf_ir_utils.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/draw.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/dump_utils.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc"
@@ -52,6 +53,7 @@ if(NOT ENABLE_SECURITY)
     if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
         list(APPEND _DEBUG_SRC_LIST
             "${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
+            "${CMAKE_CURRENT_SOURCE_DIR}/dump_utils.cc"
             "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc"
             )
     endif()

mindspore/ccsrc/debug/dump_utils.cc

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug/dump_utils.h"
+
+#include <string>
+#include "utils/log_adapter.h"
+#include "utils/ms_context.h"
+#include "utils/comm_manager.h"
+#include "frontend/parallel/context.h"
+
+namespace mindspore {
+uint32_t DumpUtils::GetRankId() {
+  uint32_t rank_id = 0;
+  auto parallel_context = parallel::ParallelContext::GetInstance();
+  MS_EXCEPTION_IF_NULL(parallel_context);
+  auto parallel_mode = parallel_context->parallel_mode();
+  if (parallel_mode == parallel::STAND_ALONE) {
+    MS_LOG(INFO) << "parallel_mode is stand_alone, use 0 as default rank id.";
+    return rank_id;
+  }
+
+  auto ms_context = MsContext::GetInstance();
+  MS_EXCEPTION_IF_NULL(ms_context);
+  std::string world_group;
+  std::string backend = ms_context->get_param<std::string>(MS_CTX_DEVICE_TARGET);
+  if (backend == kAscendDevice) {
+    world_group = kHcclWorldGroup;
+  } else if (backend == kGPUDevice) {
+    world_group = kNcclWorldGroup;
+  } else {
+    return rank_id;
+  }
+
+  if (!CommManager::GetInstance().GetRankID(world_group, &rank_id)) {
+    MS_LOG(WARNING) << "Failed to get rank id.";
+  }
+
+  return rank_id;
+}
+}  // namespace mindspore

mindspore/ccsrc/debug/dump_utils.h

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_CCSRC_DEBUG_DUMP_UTILS_H_
+#define MINDSPORE_CCSRC_DEBUG_DUMP_UTILS_H_
+#include <stdint.h>
+
+namespace mindspore {
+class DumpUtils {
+ public:
+  DumpUtils() = default;
+  ~DumpUtils() = default;
+  static uint32_t GetRankId();
+};
+}  // namespace mindspore
+#endif  // MINDSPORE_CCSRC_DEBUG_DUMP_UTILS_H_

mindspore/ccsrc/debug/env_config_parser.cc

@@ -19,6 +19,7 @@
 #include "nlohmann/json.hpp"
 #include "utils/log_adapter.h"
 #include "debug/common.h"
+#include "debug/dump_utils.h"
 #include "utils/ms_context.h"
 #include "utils/convert_utils_base.h"
 
@@ -102,17 +103,19 @@ void EnvConfigParser::ParseFromEnv() {
     has_rdr_setting_ = true;
     rdr_enabled_ = rdr_enable_env.value();
   }
+  std::string path = "";
   auto path_env = GetRdrPathFromEnv();
   if (path_env.has_value()) {
     has_rdr_setting_ = true;
-    std::string path = path_env.value();
+    path = path_env.value();
     if (!path.empty()) {
       if (path.back() != '/') {
         path += '/';
       }
-      rdr_path_ = path;
     }
   }
+  uint32_t rank_id = DumpUtils::GetRankId();
+  rdr_path_ = path + "rank_" + std::to_string(rank_id) + "/rdr/";
 #endif
 }