!7855 Move backend opt pass ir files to the fold: verbose_ir_files

Merge pull request !7855 from huanghui/move_ir_files

mindspore/ccsrc/backend/optimizer/common/pass_manager.cc

@@ -67,8 +67,11 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
       MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us";
 #endif
       if (save_graphs) {
-        auto file_name = "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";
-        DumpIR(file_name, func_graph, true);
+        std::ostringstream oss;
+        oss << "verbose_ir_files"
+            << "/";
+        oss << "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";
+        DumpIR(oss.str(), func_graph, true);
       }
       num++;
     }

mindspore/ccsrc/debug/CMakeLists.txt

@@ -4,6 +4,7 @@ set(_DEBUG_SRC_LIST
     "${CMAKE_CURRENT_SOURCE_DIR}/draw.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc"
     "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
 )
 
 if (ENABLE_DEBUGGER)
@@ -12,7 +13,6 @@ if (ENABLE_DEBUGGER)
         "${CMAKE_CURRENT_SOURCE_DIR}/debugger/grpc_client.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc"
         "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc"
-        "${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
         )
 endif (ENABLE_DEBUGGER)
 

mindspore/ccsrc/debug/anf_ir_dump.cc

@@ -28,6 +28,7 @@
 #include "backend/session/anf_runtime_algorithm.h"
 #include "frontend/parallel/ops_info/operator_info.h"
 #include "pipeline/jit/base.h"
+#include "debug/common.h"
 
 namespace mindspore {
 const std::string ToShortString(const TypeId &typeId) {
@@ -483,35 +484,23 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
   if (graph == nullptr) {
     return;
   }
-  auto real_filename = pipeline::GetSaveGraphsPathName(AddGlobalId(filename));
-  if (real_filename.size() > PATH_MAX) {
-    MS_LOG(ERROR) << "File path " << real_filename << " is too long.";
+  auto path = pipeline::GetSaveGraphsPathName(AddGlobalId(filename));
+  auto realpath = Common::GetRealPath(path);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path failed. path=" << path;
     return;
   }
-  char real_path[PATH_MAX] = {0};
-#if defined(_WIN32) || defined(_WIN64)
-  if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
-    MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
-  }
-#else
-  if (nullptr == realpath(real_filename.c_str(), real_path)) {
-    MS_LOG(DEBUG) << "Dir " << real_filename << " does not exit.";
-  }
-#endif
 
-  OrderedMap<AnfNodePtr, int32_t> para_map;
-  std::string path_string = real_path;
-  ChangeFileMode(path_string, S_IRWXU);
-  std::ofstream fout(real_path);
+  ChangeFileMode(realpath.value(), S_IRWXU);
+  std::ofstream fout(realpath.value());
   std::ostringstream buffer;
-
   if (!fout.is_open()) {
-    MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!";
+    MS_LOG(ERROR) << "Open dump file '" << realpath.value() << "' failed!";
     return;
   }
 
   auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude);
-
+  OrderedMap<AnfNodePtr, int32_t> para_map;
   // dump global info
   DumpGlobalInfoEntry(graph, buffer);
   DumpParams(graph, buffer, &para_map);
@@ -528,7 +517,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
 
   fout.close();
   // set file mode to read only by user
-  ChangeFileMode(path_string, S_IRUSR);
+  ChangeFileMode(realpath.value(), S_IRUSR);
 }
 #else
 void DumpIR(const std::string &, const FuncGraphPtr &, bool) {

mindspore/ccsrc/debug/common.cc

@@ -43,11 +43,19 @@ std::optional<std::string> Common::GetRealPath(const std::string &input_path) {
       MS_LOG(ERROR) << "CreateNotExistDirs Failed!";
       return std::nullopt;
     }
-
+#if defined(SYSTEM_ENV_POSIX)
     if (nullptr == realpath(prefix_path.c_str(), real_path)) {
-      MS_LOG(ERROR) << "dir " << prefix_path << " does not exit.";
+      MS_LOG(ERROR) << "dir " << prefix_path << " does not exist.";
+      return std::nullopt;
+    }
+#elif defined(SYSTEM_ENV_WINDOWS)
+    if (nullptr == _fullpath(real_path, prefix_path.c_str(), PATH_MAX)) {
+      MS_LOG(ERROR) << "dir " << prefix_path << " does not exist.";
       return std::nullopt;
     }
+#else
+    MS_LOG(EXCEPTION) << "Unsupported platform.";
+#endif
     out_path = std::string(real_path) + last_path;
   }
 
@@ -56,9 +64,17 @@ std::optional<std::string> Common::GetRealPath(const std::string &input_path) {
       MS_LOG(ERROR) << "Prefix path is too longer!";
       return std::nullopt;
     }
+#if defined(SYSTEM_ENV_POSIX)
     if (nullptr == realpath(input_path.c_str(), real_path)) {
-      MS_LOG(ERROR) << "File " << input_path << " does not exit, it will be created.";
+      MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created.";
+    }
+#elif defined(SYSTEM_ENV_WINDOWS)
+    if (nullptr == _fullpath(real_path, input_path.c_str(), PATH_MAX)) {
+      MS_LOG(ERROR) << "File " << input_path << " does not exist, it will be created.";
     }
+#else
+    MS_LOG(EXCEPTION) << "Unsupported platform.";
+#endif
     out_path = std::string(real_path);
   }
   return out_path;

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -23,6 +23,7 @@
 #include <unordered_map>
 #include <cstdlib>
 #include <algorithm>
+#include <iomanip>
 
 #include "ir/param_info.h"
 #include "pipeline/jit/pass.h"
@@ -85,7 +86,7 @@ std::unordered_map<abstract::AbstractBasePtrList, int, abstract::AbstractBasePtr
 namespace {
 std::string GetBaseNameForIR(int stage_idx, const std::string &action_name) {
   std::ostringstream oss;
-  oss << stage_idx << "_" << action_name;
+  oss << std::setfill('0') << std::setw(2) << stage_idx << "_" << action_name;
   return oss.str();
 }
 

mindspore/ccsrc/utils/system/base.h

@@ -49,7 +49,7 @@ using uint64 = uint64_t;
 
 #define SYSTEM_ENV_POSIX
 
-#elif defined(_WIN32)
+#elif defined(_WIN32) || defined(_WIN64)
 
 #define SYSTEM_ENV_WINDOWS
 

mindspore/ccsrc/utils/system/env.h

@@ -34,6 +34,9 @@ class Env {
 #if defined(SYSTEM_ENV_POSIX)
     auto fs = std::make_shared<PosixFileSystem>();
     return fs;
+#elif defined(SYSTEM_ENV_WINDOWS)
+    auto fs = std::make_shared<WinFileSystem>();
+    return fs;
 #else
     MS_LOG(EXCEPTION) << "Now not support the platform.";
 #endif

mindspore/ccsrc/utils/system/file_system.cc

@@ -16,14 +16,18 @@
 
 #include "utils/system/file_system.h"
 
+#if defined(SYSTEM_ENV_POSIX)
 #include <sys/stat.h>
 #include <unistd.h>
+#elif defined(SYSTEM_ENV_WINDOWS)
+#include <direct.h>
+#endif
 
 namespace mindspore {
 namespace system {
 
 #if defined(SYSTEM_ENV_POSIX)
-// Implement the Posix file systen
+// Implement the Posix file system
 WriteFilePtr PosixFileSystem::CreateWriteFile(const string &file_name) {
   if (file_name.empty()) {
     MS_LOG(ERROR) << "Create write file failed because the file name is null.";
@@ -49,7 +53,7 @@ bool PosixFileSystem::FileExist(const string &file_name) {
   }
   auto result = access(file_name.c_str(), F_OK);
   if (result != 0) {
-    MS_LOG(INFO) << "The file(" << file_name << ") not exist.";
+    MS_LOG(DEBUG) << "The file(" << file_name << ") not exist.";
     return false;
   }
   return true;
@@ -62,7 +66,7 @@ bool PosixFileSystem::DeleteFile(const string &file_name) {
   }
   auto result = unlink(file_name.c_str());
   if (result != 0) {
-    MS_LOG(ERROR) << "Delete the file(" << file_name << ") is falire, error(" << errno << ").";
+    MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ").";
     return false;
   }
   return true;
@@ -77,7 +81,7 @@ bool PosixFileSystem::CreateDir(const string &dir_name) {
   auto result = mkdir(dir_name.c_str(), DEFAULT_MKDIR_MODE);
   if (result != 0) {
     if (errno != EEXIST) {
-      MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is falire, error(" << errno << ").";
+      MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << errno << ").";
       return false;
     }
   }
@@ -98,5 +102,147 @@ bool PosixFileSystem::DeleteDir(const string &dir_name) {
 }
 #endif
 
+#if defined(SYSTEM_ENV_WINDOWS)
+// Implement the Windows file system
+WriteFilePtr WinFileSystem::CreateWriteFile(const string &file_name) {
+  if (file_name.empty()) {
+    MS_LOG(ERROR) << "Create write file failed because the file name is null.";
+    return nullptr;
+  }
+  auto fp = std::make_shared<WinWriteFile>(file_name);
+  if (fp == nullptr) {
+    MS_LOG(ERROR) << "Create write file(" << file_name << ") failed.";
+    return nullptr;
+  }
+  bool result = fp->Open();
+  if (!result) {
+    MS_LOG(ERROR) << "Open the write file(" << file_name << ") failed.";
+    return nullptr;
+  }
+  return fp;
+}
+
+bool WinFileSystem::FileExist(const string &file_name) {
+  if (file_name.empty()) {
+    MS_LOG(WARNING) << "The file name is null.";
+    return false;
+  }
+  auto result = access(file_name.c_str(), F_OK);
+  if (result != 0) {
+    MS_LOG(DEBUG) << "The file(" << file_name << ") not exist.";
+    return false;
+  }
+  return true;
+}
+
+bool WinFileSystem::CreateDir(const string &dir_name) {
+  if (dir_name.empty()) {
+    MS_LOG(WARNING) << "The directory name is null.";
+    return false;
+  }
+  auto result = mkdir(dir_name.c_str());
+  if (result != 0) {
+    MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is failed, error(" << result << ").";
+    return false;
+  }
+  return true;
+}
+
+bool WinFileSystem::DeleteDir(const string &dir_name) {
+  if (dir_name.empty()) {
+    MS_LOG(WARNING) << "The directory name is null.";
+    return false;
+  }
+  auto result = rmdir(dir_name.c_str());
+  if (result != 0) {
+    MS_LOG(ERROR) << "Delete the dir(" << dir_name << ") is failed, error(" << result << ").";
+    return false;
+  }
+  return true;
+}
+
+bool WinFileSystem::DeleteFile(const string &file_name) {
+  if (file_name.empty()) {
+    MS_LOG(WARNING) << "The file name is null.";
+    return false;
+  }
+  auto result = unlink(file_name.c_str());
+  if (result != 0) {
+    MS_LOG(ERROR) << "Delete the file(" << file_name << ") is failed, error(" << errno << ").";
+    return false;
+  }
+  return true;
+}
+
+bool WinWriteFile::Open() {
+  if (file_ != nullptr) {
+    MS_LOG(WARNING) << "The File(" << file_name_ << ") already open.";
+    return true;
+  }
+  // check the path
+  if (nullptr == file_name_.c_str()) {
+    MS_LOG(EXCEPTION) << "The file path is null.";
+  }
+  char path[PATH_MAX + 1] = {0x00};
+  if (file_name_.size() > PATH_MAX || nullptr == _fullpath(path, file_name_.c_str(), PATH_MAX)) {
+    MS_LOG(EXCEPTION) << "Convert to real path fail, file name is " << file_name_ << ".";
+  }
+
+  // open the file
+  file_ = fopen(path, "w+");
+  if (file_ == nullptr) {
+    MS_LOG(ERROR) << "File(" << path << ") IO ERROR: " << errno << ".";
+    return false;
+  }
+  return true;
+}
+
+bool WinWriteFile::Write(const std::string &data) {
+  MS_LOG(DEBUG) << "Write data(" << data.size() << ") to file(" << this->file_name_ << ").";
+  size_t r = fwrite(data.data(), 1, data.size(), file_);
+  if (r != data.size()) {
+    MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << ".";
+    return false;
+  }
+  return true;
+}
+
+bool WinWriteFile::Close() {
+  if (file_ == nullptr) {
+    MS_LOG(WARNING) << "File(" << file_name_ << ") already close.";
+    return true;
+  }
+  bool result = true;
+  if (fclose(file_) != 0) {
+    MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << ".";
+    result = false;
+  }
+  file_ = nullptr;
+  return result;
+}
+
+bool WinWriteFile::Flush() {
+  if (fflush(file_) != 0) {
+    MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << EBADF << ".";
+    return false;
+  }
+  return true;
+}
+
+bool WinWriteFile::Sync() { return Flush(); }
+
+WinWriteFile::~WinWriteFile() {
+  try {
+    if (file_ != nullptr) {
+      (void)fclose(file_);
+      file_ = nullptr;
+    }
+  } catch (const std::exception &e) {
+    MS_LOG(ERROR) << "Exception when closing file.";
+  } catch (...) {
+    MS_LOG(ERROR) << "Non standard exception when closing file.";
+  }
+}
+#endif
 }  // namespace system
 }  // namespace mindspore

mindspore/ccsrc/utils/system/file_system.h

@@ -198,6 +198,51 @@ class PosixWriteFile : public WriteFile {
 };
 #endif
 
+#if defined(SYSTEM_ENV_WINDOWS)
+// File system of create or delete directory for windows system
+class WinFileSystem : public FileSystem {
+ public:
+  WinFileSystem() = default;
+
+  ~WinFileSystem() override = default;
+
+  // create a new write file
+  WriteFilePtr CreateWriteFile(const string &file_name) override;
+
+  // check the file is exist?
+  bool FileExist(const string &file_name) override;
+
+  // delete the file
+  bool DeleteFile(const string &file_name) override;
+
+  // Create a Directory
+  bool CreateDir(const string &dir_name) override;
+
+  // Delete the specified directory.
+  bool DeleteDir(const string &dir_name) override;
+};
+
+// A file that can be read and write for windows
+class WinWriteFile : public WriteFile {
+ public:
+  explicit WinWriteFile(const string &file_name) : WriteFile(file_name), file_(nullptr) {}
+
+  ~WinWriteFile() override;
+
+  bool Open() override;
+
+  bool Write(const std::string &data) override;
+
+  bool Close() override;
+
+  bool Flush() override;
+
+  bool Sync() override;
+
+ private:
+  FILE *file_;
+};
+#endif
 }  // namespace system
 }  // namespace mindspore
 

tests/ut/python/pynative_mode/test_backend.py

@@ -14,6 +14,7 @@
 # ============================================================================
 """ test_backend """
 import os
+import shutil
 import pytest
 
 import mindspore.nn as nn
@@ -86,6 +87,6 @@ def teardown_module():
         if not os.path.exists(item_name):
             continue
         if os.path.isdir(item_name):
-            os.rmdir(item_name)
+            shutil.rmtree(item_name)
         elif os.path.isfile(item_name):
             os.remove(item_name)

tests/ut/python/pynative_mode/test_context.py

@@ -14,6 +14,7 @@
 # ============================================================================
 """ test_context """
 import os
+import shutil
 import pytest
 
 from mindspore import context
@@ -150,6 +151,6 @@ def teardown_module():
         if not os.path.exists(item_name):
             continue
         if os.path.isdir(item_name):
-            os.rmdir(item_name)
+            shutil.rmtree(item_name)
         elif os.path.isfile(item_name):
             os.remove(item_name)