Browse Source

!13304 refactor RDR to support single name

From: @luopengting
Reviewed-by: @ouwenchang,@lixiaohui33
Signed-off-by: @lixiaohui33
tags/v1.2.0-rc1
mindspore-ci-bot Gitee 4 years ago
parent
commit
defcc51641
26 changed files with 144 additions and 129 deletions
  1. +2
    -2
      mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc
  2. +6
    -8
      mindspore/ccsrc/backend/optimizer/somas/somas.cc
  3. +2
    -2
      mindspore/ccsrc/backend/session/ascend_session.cc
  4. +2
    -2
      mindspore/ccsrc/backend/session/gpu_session.cc
  5. +4
    -4
      mindspore/ccsrc/debug/rdr/base_recorder.cc
  6. +11
    -10
      mindspore/ccsrc/debug/rdr/base_recorder.h
  7. +1
    -1
      mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc
  8. +3
    -4
      mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h
  9. +1
    -7
      mindspore/ccsrc/debug/rdr/graph_recorder.cc
  10. +2
    -2
      mindspore/ccsrc/debug/rdr/graph_recorder.h
  11. +9
    -4
      mindspore/ccsrc/debug/rdr/mem_address_recorder.cc
  12. +6
    -7
      mindspore/ccsrc/debug/rdr/mem_address_recorder.h
  13. +16
    -6
      mindspore/ccsrc/debug/rdr/recorder_manager.cc
  14. +29
    -4
      mindspore/ccsrc/debug/rdr/recorder_manager.h
  15. +19
    -25
      mindspore/ccsrc/debug/rdr/running_data_recorder.cc
  16. +8
    -10
      mindspore/ccsrc/debug/rdr/running_data_recorder.h
  17. +1
    -2
      mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.cc
  18. +2
    -4
      mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h
  19. +2
    -5
      mindspore/ccsrc/debug/rdr/string_recorder.h
  20. +1
    -1
      mindspore/ccsrc/debug/rdr/task_debug_info_recorder.cc
  21. +3
    -4
      mindspore/ccsrc/debug/rdr/task_debug_info_recorder.h
  22. +2
    -2
      mindspore/ccsrc/pipeline/jit/pipeline.cc
  23. +2
    -2
      mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc
  24. +2
    -2
      mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc
  25. +4
    -4
      mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc
  26. +4
    -5
      mindspore/ccsrc/runtime/device/memory_manager.cc

+ 2
- 2
mindspore/ccsrc/backend/optimizer/ascend/ascend_backend_optimization.cc View File

@@ -399,8 +399,8 @@ void AscendBackendOptimization(const std::shared_ptr<session::KernelGraph> &kern
kernel_graph->SetExecOrderByDefault();
#ifdef ENABLE_DUMP_IR
const std::vector<CNodePtr> &exec_order = kernel_graph->execution_order();
std::string exec_order_tag = "graph_exec_order";
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, exec_order_tag, exec_order, kernel_graph->graph_id());
std::string exec_order_name = "graph_exec_order." + std::to_string(kernel_graph->graph_id());
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_OPTIMIZER, exec_order_name, exec_order);
#endif
if (save_graphs) {
std::string file_name = "hwopt_d_end_graph_" + std::to_string(kernel_graph->graph_id()) + ".ir";


+ 6
- 8
mindspore/ccsrc/backend/optimizer/somas/somas.cc View File

@@ -379,11 +379,10 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) {

#ifdef ENABLE_DUMP_IR
SubModuleId module = SubModuleId::SM_OPTIMIZER;
std::string tag = "somas";
std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
filename = "somas_offline_log_" + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, tag, Offline(), filename);
std::string name = "somas_pre_processed_info." + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, name, SomasInfo());
name = "somas_offline_log." + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, name, Offline());
#endif

if (save_graphs_) {
@@ -671,9 +670,8 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) {

#ifdef ENABLE_DUMP_IR
SubModuleId module = SubModuleId::SM_OPTIMIZER;
std::string tag = "somas";
std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, tag, SomasInfo(), filename);
std::string name = "somas_initial_info." + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, name, SomasInfo());
#endif

save_graphs_ = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);


+ 2
- 2
mindspore/ccsrc/backend/session/ascend_session.cc View File

@@ -1010,8 +1010,8 @@ void AscendSession::DumpAllGraphs(const std::vector<KernelGraphPtr> &all_graphs)
uint32_t device_id = kernel_runtime->device_id();
for (auto &graph : all_graphs) {
MS_EXCEPTION_IF_NULL(graph);
std::string tag = "graph_build";
mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph, true, ".ir;.pb");
std::string name = "graph_build." + std::to_string(graph->graph_id());
mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph, true, ".ir;.pb");
if (save_graphs) {
std::string file_name = "graph_build_" + std::to_string(graph->graph_id()) + ".ir";
DumpIR(file_name, graph, true, kWholeStack);


+ 2
- 2
mindspore/ccsrc/backend/session/gpu_session.cc View File

@@ -372,8 +372,8 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) {
// Build kernel if node is cnode
BuildKernel(graph);
#ifdef ENABLE_DUMP_IR
std::string tag = "graph_build";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, tag, graph, false, ".ir,.pb");
std::string name = "graph_build";
mindspore::RDR::RecordAnfGraph(SubModuleId::SM_SESSION, name, graph, false, ".ir,.pb");
#endif
// Get summary nodes.
SetSummaryNodes(graph.get());


+ 4
- 4
mindspore/ccsrc/debug/rdr/base_recorder.cc View File

@@ -21,7 +21,7 @@

namespace mindspore {
void BaseRecorder::SetDirectory(const std::string &directory) {
std::string error_message = module_ + ":" + tag_ + " set directory failed.";
std::string error_message = module_ + ":" + name_ + " set directory failed.";
if (Common::IsPathValid(directory, maxDirectoryLength, error_message)) {
directory_ = directory;
if (directory_.back() != '/') {
@@ -31,7 +31,7 @@ void BaseRecorder::SetDirectory(const std::string &directory) {
}

void BaseRecorder::SetFilename(const std::string &filename) {
std::string error_message = module_ + ":" + tag_ + " set filename failed.";
std::string error_message = module_ + ":" + name_ + " set filename failed.";
if (Common::IsFilenameValid(filename, maxDirectoryLength, error_message)) {
filename_ = filename;
}
@@ -40,7 +40,7 @@ void BaseRecorder::SetFilename(const std::string &filename) {
std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suffix) {
std::string filename;
if (filename_.empty()) {
filename = module_ + delimiter_ + tag_;
filename = module_ + delimiter_ + name_;
if (!suffix.empty()) {
filename += delimiter_ + suffix;
}
@@ -55,7 +55,7 @@ std::optional<std::string> BaseRecorder::GetFileRealPath(const std::string &suff
auto realpath = Common::GetRealPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed. "
<< "Info: module=" << module_ << ", tag=" << tag_ << ", "
<< "Info: module=" << module_ << ", name=" << name_ << ", "
<< "path=" << file_path << ".";
}



+ 11
- 10
mindspore/ccsrc/debug/rdr/base_recorder.h View File

@@ -24,21 +24,21 @@
#include "debug/env_config_parser.h"
#include "mindspore/core/utils/log_adapter.h"

const int maxTagLength = 32;
const int maxNameLength = 32;
namespace mindspore {
class BaseRecorder {
public:
BaseRecorder() : module_(""), tag_(""), directory_(""), filename_(""), timestamp_("") {}
BaseRecorder(const std::string &module, const std::string &tag) : module_(module), tag_(tag), filename_("") {
BaseRecorder() : module_(""), name_(""), directory_(""), filename_(""), timestamp_("") {}
BaseRecorder(const std::string &module, const std::string &name) : module_(module), name_(name), filename_("") {
directory_ = mindspore::EnvConfigParser::GetInstance().rdr_path();

if (tag.length() > maxTagLength) {
tag_ = tag.substr(0, maxTagLength);
MS_LOG(WARNING) << "The tag length is " << tag.length() << ", exceeding the limit " << maxTagLength
<< ". It will be intercepted as '" << tag_ << "'.";
if (name.length() > maxNameLength) {
name_ = name.substr(0, maxNameLength);
MS_LOG(WARNING) << "The name length is " << name.length() << ", exceeding the limit " << maxNameLength
<< ". It will be intercepted as '" << name_ << "'.";
}

std::string err_msg = module_ + ":" + tag_ + " set filename failed.";
std::string err_msg = module_ + ":" + name_ + " set filename failed.";
if (!filename_.empty() && !Common::IsFilenameValid(filename_, maxFilenameLength, err_msg)) {
filename_ = "";
}
@@ -57,7 +57,7 @@ class BaseRecorder {
virtual ~BaseRecorder() {}

std::string GetModule() const { return module_; }
std::string GetTag() const { return tag_; }
std::string GetName() const { return name_; }
std::string GetTimeStamp() const { return timestamp_; }
std::optional<std::string> GetFileRealPath(const std::string &suffix = "");

@@ -65,10 +65,11 @@ class BaseRecorder {
void SetFilename(const std::string &filename);
void SetModule(const std::string &module) { module_ = module; }
virtual void Export() {}
virtual void UpdateInfo(const BaseRecorder &recorder) {}

protected:
std::string module_;
std::string tag_;
std::string name_;
std::string directory_;
std::string filename_;
std::string timestamp_; // year,month,day,hour,minute,second


+ 1
- 1
mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.cc View File

@@ -49,7 +49,7 @@ bool DumpGraphExeOrder(const std::string &filename, const std::vector<CNodePtr>
} // namespace

void GraphExecOrderRecorder::Export() {
auto realpath = GetFileRealPath(std::to_string(graph_id_));
auto realpath = GetFileRealPath();
if (!realpath.has_value()) {
return;
}


+ 3
- 4
mindspore/ccsrc/debug/rdr/graph_exec_order_recorder.h View File

@@ -27,16 +27,15 @@ using CNodePtr = std::shared_ptr<CNode>;
class GraphExecOrderRecorder : public BaseRecorder {
public:
GraphExecOrderRecorder() : BaseRecorder() {}
GraphExecOrderRecorder(const std::string &module, const std::string &tag,
const std::vector<CNodePtr> &final_exec_order, int graph_id)
: BaseRecorder(module, tag), exec_order_(final_exec_order), graph_id_(graph_id) {}
GraphExecOrderRecorder(const std::string &module, const std::string &name,
const std::vector<CNodePtr> &final_exec_order)
: BaseRecorder(module, name), exec_order_(final_exec_order) {}
~GraphExecOrderRecorder() {}
void SetExecOrder(const std::vector<CNodePtr> &final_exec_order) { exec_order_ = final_exec_order; }
virtual void Export();

private:
std::vector<CNodePtr> exec_order_;
int graph_id_;
};
using GraphExecOrderRecorderPtr = std::shared_ptr<GraphExecOrderRecorder>;
} // namespace mindspore


+ 1
- 7
mindspore/ccsrc/debug/rdr/graph_recorder.cc View File

@@ -58,13 +58,7 @@ void DumpIRProto(const std::string &, const FuncGraphPtr &) {

void GraphRecorder::Export() {
bool save_flag = false;
int graph_id = -1;
if (func_graph_->isa<session::KernelGraph>()) {
auto kernel_graph = func_graph_->cast<KernelGraphPtr>();
graph_id = kernel_graph->graph_id();
}
std::string suffix = graph_id >= 0 ? std::to_string(graph_id) : "";
auto tmp_realpath = GetFileRealPath(suffix);
auto tmp_realpath = GetFileRealPath();
if (!tmp_realpath.has_value()) {
return;
}


+ 2
- 2
mindspore/ccsrc/debug/rdr/graph_recorder.h View File

@@ -27,9 +27,9 @@ using FuncGraphPtr = std::shared_ptr<FuncGraph>;
class GraphRecorder : public BaseRecorder {
public:
GraphRecorder() : BaseRecorder(), func_graph_(nullptr), graph_type_("") {}
GraphRecorder(const std::string &module, const std::string &tag, const FuncGraphPtr &graph,
GraphRecorder(const std::string &module, const std::string &name, const FuncGraphPtr &graph,
const std::string &file_type)
: BaseRecorder(module, tag), func_graph_(graph), graph_type_(file_type) {}
: BaseRecorder(module, name), func_graph_(graph), graph_type_(file_type) {}
~GraphRecorder() {}
void SetGraphType(const std::string &file_type) { graph_type_ = file_type; }
void SetFuncGraph(const FuncGraphPtr &func_graph) { func_graph_ = func_graph; }


+ 9
- 4
mindspore/ccsrc/debug/rdr/mem_address_recorder.cc View File

@@ -32,10 +32,6 @@ std::string MemInfo2String(const std::string &label, const AddressPtrList &info)
return ss.str();
}
} // namespace
MemAddressRecorder &MemAddressRecorder::Instance() {
static MemAddressRecorder recorder;
return recorder;
}

void MemAddressRecorder::SaveMemInfo(const std::string &op_name, const GPUMemInfo &mem_info) {
std::lock_guard<std::mutex> lock(mtx_);
@@ -48,6 +44,7 @@ void MemAddressRecorder::SaveMemInfo(const std::string &op_name, const GPUMemInf
auto outputs = mem_info.outputs_;
mem_info_stream << MemInfo2String("kernel_outputs", *outputs);
mem_info_stream << std::endl;

std::string mem_info_str = mem_info_stream.str();
mem_info_container_[op_name] = mem_info_str;
}
@@ -71,4 +68,12 @@ void MemAddressRecorder::Export() {
fout.close();
ChangeFileMode(file_path, S_IRUSR);
}

void MemAddressRecorder::UpdateInfo(const BaseRecorder &recorder) {
const MemAddressRecorder *mem_recorder = reinterpret_cast<const MemAddressRecorder *>(&recorder);
std::map<std::string, std::string> mem_info = mem_recorder->MemInfo();
for (const auto &info : mem_info) {
mem_info_container_[info.first] = info.second;
}
}
} // namespace mindspore

+ 6
- 7
mindspore/ccsrc/debug/rdr/mem_address_recorder.h View File

@@ -36,17 +36,16 @@ struct GPUMemInfo {
};
class MemAddressRecorder : public BaseRecorder {
public:
static MemAddressRecorder &Instance();
MemAddressRecorder() {}
MemAddressRecorder(const std::string &module, const std::string &name) : BaseRecorder(module, name) {}
~MemAddressRecorder() {}

virtual void Export();
void SaveMemInfo(const std::string &op_name, const GPUMemInfo &mem_info);
void SetTag(const std::string &tag) { tag_ = tag; }
void UpdateInfo(const BaseRecorder &recorder);
std::map<std::string, std::string> MemInfo() const { return mem_info_container_; }

private:
MemAddressRecorder() {}
MemAddressRecorder(const MemAddressRecorder &recorder);
~MemAddressRecorder() {}
MemAddressRecorder &operator=(const MemAddressRecorder &recorder);

mutable std::mutex mtx_;

std::map<std::string, std::string> mem_info_container_;


+ 16
- 6
mindspore/ccsrc/debug/rdr/recorder_manager.cc View File

@@ -42,7 +42,7 @@ void RecorderManager::UpdateRdrEnable() {
updated = true;
}

bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) {
bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder, const bool &replace) {
if (!rdr_enable_) {
return false;
}
@@ -52,8 +52,20 @@ bool RecorderManager::RecordObject(const BaseRecorderPtr &recorder) {
return false;
}
std::string module = recorder->GetModule();
std::string name = recorder->GetName();
std::pair<std::string, std::string> recorder_key(module, name);
std::lock_guard<std::mutex> lock(mtx_);
recorder_container_[module].push_back(std::move(recorder));
if (replace) {
recorder_container_[recorder_key] = recorder;
return true;
}
std::unordered_map<std::pair<std::string, std::string>, BaseRecorderPtr, pair_hash>::iterator item =
recorder_container_.find(recorder_key);
if (item == recorder_container_.end()) {
recorder_container_[recorder_key] = recorder;
} else {
recorder_container_[recorder_key]->UpdateInfo(*recorder);
}
return true;
}

@@ -65,10 +77,8 @@ void RecorderManager::TriggerAll() {
bool trigger = false;
std::lock_guard<std::mutex> lock(mtx_);
for (auto iter = recorder_container_.begin(); iter != recorder_container_.end(); ++iter) {
for (auto &recorder : iter->second) {
recorder->Export();
trigger = true;
}
iter->second->Export();
trigger = true;
}
if (!trigger) {
MS_LOG(WARNING) << "There is no recorder to export.";


+ 29
- 4
mindspore/ccsrc/debug/rdr/recorder_manager.h View File

@@ -22,11 +22,36 @@
#include <unordered_map>
#include <memory>
#include <mutex>
#include <utility>

namespace mindspore {
template <typename T>
inline void hash_combine(std::size_t *seed, const T &val) {
// The number is the reciprocal of the golden ratio.
unsigned int magic_constant = 0x9e3779b9;

int shift_left = 6;
int shift_right = 2;
*seed ^= std::hash<T>()(val) + magic_constant + (*seed << shift_left) + (*seed >> shift_right);
}

template <typename T1, typename T2>
inline std::size_t hash_seed(const T1 &val1, const T2 &val2) {
std::size_t seed = 0;
hash_combine(&seed, val1);
hash_combine(&seed, val2);
return seed;
}

struct pair_hash {
template <class T1, class T2>
std::size_t operator()(const std::pair<T1, T2> &p) const {
return hash_seed(p.first, p.second);
}
};

class BaseRecorder;
using BaseRecorderPtr = std::shared_ptr<BaseRecorder>;
using BaseRecorderPtrList = std::vector<BaseRecorderPtr>;
class RecorderManager {
public:
static RecorderManager &Instance() {
@@ -37,7 +62,7 @@ class RecorderManager {

void UpdateRdrEnable();
bool RdrEnable() const { return rdr_enable_; }
bool RecordObject(const BaseRecorderPtr &recorder);
bool RecordObject(const BaseRecorderPtr &recorder, const bool &replace = true);
void TriggerAll();
void ClearAll();

@@ -48,8 +73,8 @@ class RecorderManager {
bool rdr_enable_{false};

mutable std::mutex mtx_;
// module, BaserRecorderPtrList
std::unordered_map<std::string, BaseRecorderPtrList> recorder_container_;
// <module, name>, BaserRecorderPtr
std::unordered_map<std::pair<std::string, std::string>, BaseRecorderPtr, pair_hash> recorder_container_;
};
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEBUG_RDR_RECORDER_MANAGER_H_

+ 19
- 25
mindspore/ccsrc/debug/rdr/running_data_recorder.cc View File

@@ -65,84 +65,78 @@ static const char *GetSubModuleName(SubModuleId module_id) {
} // namespace
namespace RDR {
#ifdef ENABLE_D
bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag,
const std::vector<TaskDebugInfoPtr> &task_debug_info_list, int graph_id) {
bool RecordTaskDebugInfo(SubModuleId module, const std::string &name,
const std::vector<TaskDebugInfoPtr> &task_debug_info_list) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
TaskDebugInfoRecorderPtr task_debug_info_recorder =
std::make_shared<TaskDebugInfoRecorder>(submodule_name, tag, task_debug_info_list, graph_id);
std::make_shared<TaskDebugInfoRecorder>(submodule_name, name, task_debug_info_list);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(task_debug_info_recorder));
return ans;
}
#endif // ENABLE_D

bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
bool RecordAnfGraph(const SubModuleId module, const std::string &name, const FuncGraphPtr &graph, bool full_name,
const std::string &file_type) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, tag, graph, file_type);
GraphRecorderPtr graph_recorder = std::make_shared<GraphRecorder>(submodule_name, name, graph, file_type);
graph_recorder->SetDumpFlag(full_name);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_recorder));
return ans;
}

bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
const std::vector<CNodePtr> &final_exec_order, int graph_id) {
bool RecordGraphExecOrder(const SubModuleId module, const std::string &name,
const std::vector<CNodePtr> &final_exec_order) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
GraphExecOrderRecorderPtr graph_exec_order_recorder =
std::make_shared<GraphExecOrderRecorder>(submodule_name, tag, final_exec_order, graph_id);
std::make_shared<GraphExecOrderRecorder>(submodule_name, name, final_exec_order);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(graph_exec_order_recorder));
return ans;
}

bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) {
bool RecordString(SubModuleId module, const std::string &name, const std::string &data) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, tag, data, filename);
string_recorder->SetFilename(filename);
StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, name, data);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(string_recorder));
return ans;
}

bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id,
const std::vector<CNodePtr> &exec_order) {
bool RecordStreamExecOrder(const SubModuleId module, const std::string &name, const std::vector<CNodePtr> &exec_order) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
StreamExecOrderRecorderPtr stream_exec_order_recorder =
std::make_shared<StreamExecOrderRecorder>(submodule_name, tag, graph_id, exec_order);
std::make_shared<StreamExecOrderRecorder>(submodule_name, name, exec_order);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(stream_exec_order_recorder));
return ans;
}

bool RecordMemAddressInfo(const SubModuleId module, const std::string &tag, const std::string &op_name,
bool RecordMemAddressInfo(const SubModuleId module, const std::string &name, const std::string &op_name,
const GPUMemInfo &mem_info) {
if (!mindspore::RecorderManager::Instance().RdrEnable()) {
return false;
}
std::string submodule_name = std::string(GetSubModuleName(module));
std::string directory = mindspore::EnvConfigParser::GetInstance().rdr_path();
MemAddressRecorder::Instance().SetModule(submodule_name);
MemAddressRecorder::Instance().SetFilename(tag); // set filename using tag
MemAddressRecorder::Instance().SetDirectory(directory);
MemAddressRecorder::Instance().SaveMemInfo(op_name, mem_info);
return true;
}
void TriggerAll() {
mindspore::RecorderManager::Instance().TriggerAll();
MemAddressRecorder::Instance().Export();
MemAddressRecorderPtr mem_info_recorder = std::make_shared<MemAddressRecorder>(submodule_name, name);
mem_info_recorder->SaveMemInfo(op_name, mem_info);
bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(mem_info_recorder), false);
return ans;
}

void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); }

void ClearAll() { mindspore::RecorderManager::Instance().ClearAll(); }
} // namespace RDR
} // namespace mindspore

+ 8
- 10
mindspore/ccsrc/debug/rdr/running_data_recorder.h View File

@@ -44,19 +44,17 @@ using TaskDebugInfoPtr = std::shared_ptr<device::ascend::tasksink::TaskDebugInfo
#endif // ENABLE_D

namespace RDR {
bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, bool full_name,
bool RecordAnfGraph(const SubModuleId module, const std::string &name, const FuncGraphPtr &graph, bool full_name,
const std::string &file_type = ".ir;.pb;.dat");
bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag,
const std::vector<CNodePtr> &final_exec_order, int graph_id = 0);
bool RecordString(SubModuleId module, const std::string &tag, const std::string &data,
const std::string &filename = "");
bool RecordStreamExecOrder(const SubModuleId module, const std::string &tag, const int &graph_id,
const std::vector<CNodePtr> &exec_order);
bool RecordMemAddressInfo(const SubModuleId module, const std::string &tag, const std::string &op_name,
bool RecordGraphExecOrder(const SubModuleId module, const std::string &name,
const std::vector<CNodePtr> &final_exec_order);
bool RecordString(SubModuleId module, const std::string &name, const std::string &data);
bool RecordStreamExecOrder(const SubModuleId module, const std::string &name, const std::vector<CNodePtr> &exec_order);
bool RecordMemAddressInfo(const SubModuleId module, const std::string &name, const std::string &op_name,
const GPUMemInfo &mem_info);
#ifdef ENABLE_D
bool RecordTaskDebugInfo(SubModuleId module, const std::string &tag,
const std::vector<TaskDebugInfoPtr> &task_debug_info_list, int graph_id = 0);
bool RecordTaskDebugInfo(SubModuleId module, const std::string &name,
const std::vector<TaskDebugInfoPtr> &task_debug_info_list);
#endif // ENABLE_D
void TriggerAll();
void ClearAll();


+ 1
- 2
mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.cc View File

@@ -49,8 +49,7 @@ json ExecNode::ExecNode2Json() {
}

void StreamExecOrderRecorder::Export() {
std::string file_suffix = std::to_string(graph_id_);
auto realpath = GetFileRealPath(file_suffix);
auto realpath = GetFileRealPath();
if (!realpath.has_value()) {
return;
}


+ 2
- 4
mindspore/ccsrc/debug/rdr/stream_exec_order_recorder.h View File

@@ -61,9 +61,8 @@ using CNodePtr = std::shared_ptr<CNode>;
class StreamExecOrderRecorder : public BaseRecorder {
public:
StreamExecOrderRecorder() : BaseRecorder() {}
StreamExecOrderRecorder(const std::string &module, const std::string &tag, const int &graph_id,
const std::vector<CNodePtr> &exec_order)
: BaseRecorder(module, tag), graph_id_(graph_id) {
StreamExecOrderRecorder(const std::string &module, const std::string &name, const std::vector<CNodePtr> &exec_order)
: BaseRecorder(module, name) {
// Extract information from execute order.
for (size_t i = 0; i < exec_order.size(); i++) {
CNodePtr cur_cnode_ptr = exec_order[i];
@@ -104,7 +103,6 @@ class StreamExecOrderRecorder : public BaseRecorder {

private:
std::vector<ExecNodePtr> exec_order_;
int graph_id_{0};
};
using StreamExecOrderRecorderPtr = std::shared_ptr<StreamExecOrderRecorder>;
} // namespace mindspore


+ 2
- 5
mindspore/ccsrc/debug/rdr/string_recorder.h View File

@@ -24,11 +24,8 @@ namespace mindspore {
class StringRecorder : public BaseRecorder {
public:
StringRecorder() : BaseRecorder() {}
StringRecorder(const std::string &module, const std::string &tag, const std::string &data,
const std::string &filename)
: BaseRecorder(module, tag), data_(data) {
SetFilename(filename);
}
StringRecorder(const std::string &module, const std::string &name, const std::string &data)
: BaseRecorder(module, name), data_(data) {}
~StringRecorder() {}
virtual void Export();



+ 1
- 1
mindspore/ccsrc/debug/rdr/task_debug_info_recorder.cc View File

@@ -18,7 +18,7 @@

namespace mindspore {
void TaskDebugInfoRecorder::Export() {
auto realpath = GetFileRealPath(std::to_string(graph_id_));
auto realpath = GetFileRealPath();
if (!realpath.has_value()) {
return;
}


+ 3
- 4
mindspore/ccsrc/debug/rdr/task_debug_info_recorder.h View File

@@ -34,14 +34,13 @@ using TaskDebugInfoPtr = std::shared_ptr<device::ascend::tasksink::TaskDebugInfo
class TaskDebugInfoRecorder : public BaseRecorder {
public:
TaskDebugInfoRecorder() {}
TaskDebugInfoRecorder(const std::string &module, const std::string &tag,
const std::vector<TaskDebugInfoPtr> &task_debug_info, int graph_id)
: BaseRecorder(module, tag), graph_id_(graph_id), task_debug_info_(task_debug_info) {}
TaskDebugInfoRecorder(const std::string &module, const std::string &name,
const std::vector<TaskDebugInfoPtr> &task_debug_info)
: BaseRecorder(module, name), task_debug_info_(task_debug_info) {}
~TaskDebugInfoRecorder() {}
virtual void Export();

private:
int graph_id_;
std::vector<TaskDebugInfoPtr> task_debug_info_;
};
using TaskDebugInfoRecorderPtr = std::shared_ptr<TaskDebugInfoRecorder>;


+ 2
- 2
mindspore/ccsrc/pipeline/jit/pipeline.cc View File

@@ -737,11 +737,11 @@ void Pipeline::Run() {
#ifdef ENABLE_DUMP_IR
if (mindspore::RecorderManager::Instance().RdrEnable()) {
MS_LOG(INFO) << "Recording FuncGraph in pipeline using RDR.";
std::string tag = GetBaseNameForIR(i, action.first);
std::string name = GetBaseNameForIR(i, action.first);
if (graph != nullptr) {
auto graph_clone = BasicClone(graph);
if (graph_clone != nullptr) {
mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, tag, graph_clone, false, ".ir");
mindspore::RDR::RecordAnfGraph(SUBMODULE_ID, name, graph_clone, false, ".ir");
} else {
MS_LOG(WARNING) << "Clone FuncGraph failed in pipeline, no FuncGraph recording in RDR.";
}


+ 2
- 2
mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc View File

@@ -229,9 +229,9 @@ void AscendStreamAssign::AssignStream(const NotNull<KernelGraphPtr> &graph_ptr)
MS_LOG(INFO) << "After finish stream assign";
#ifdef ENABLE_DUMP_IR
SubModuleId module = SubModuleId::SM_SESSION;
std::string tag = "assign_stream";
std::string name = "assign_stream." + std::to_string(graph_ptr->graph_id());
const std::vector<CNodePtr> &exec_order = graph_ptr->execution_order();
mindspore::RDR::RecordStreamExecOrder(module, tag, graph_ptr->graph_id(), exec_order);
mindspore::RDR::RecordStreamExecOrder(module, name, exec_order);
#endif
graph_ptr->PrintGraphExecuteOrder();



+ 2
- 2
mindspore/ccsrc/runtime/device/ascend/tasksink/task_generator.cc View File

@@ -40,8 +40,8 @@ bool TaskGenerator::GenTasks(const std::vector<CNodePtr> &anf_node_list, std::ve
}
MS_LOG(INFO) << "GenTasks end...";
#ifdef ENABLE_DUMP_IR
string task_info_tag = "task_info_graph";
mindspore::RDR::RecordTaskDebugInfo(SUBMODULE_ID, task_info_tag, task_debug_info_list_, graph_id);
string task_info_name = "task_info_graph." + std::to_string(graph_id);
mindspore::RDR::RecordTaskDebugInfo(SUBMODULE_ID, task_info_name, task_debug_info_list_);
#endif
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);


+ 4
- 4
mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc View File

@@ -650,8 +650,8 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo
auto &kernels = graph->execution_order();
int exec_order = 1;
#ifdef ENABLE_DUMP_IR
std::string exec_order_tag = "graph_exec_order";
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_KERNEL, exec_order_tag, kernels, graph->graph_id());
std::string exec_order_name = "graph_exec_order." + std::to_string(graph->graph_id());
mindspore::RDR::RecordGraphExecOrder(SubModuleId::SM_KERNEL, exec_order_name, kernels);
#endif
auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
@@ -695,9 +695,9 @@ bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph, bo
}
#ifdef ENABLE_DUMP_IR
GPUMemInfo mem_info = {&kernel_inputs, &kernel_workspaces, &kernel_outputs};
std::string tag = "mem_address_list";
std::string name = "mem_address_list";
std::string op_name = kernel->fullname_with_scope();
mindspore::RDR::RecordMemAddressInfo(SubModuleId::SM_KERNEL, tag, op_name, mem_info);
mindspore::RDR::RecordMemAddressInfo(SubModuleId::SM_KERNEL, name, op_name, mem_info);
#endif
if (!mock) {
if (!profiling) {


+ 4
- 5
mindspore/ccsrc/runtime/device/memory_manager.cc View File

@@ -74,13 +74,12 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) {
MS_EXCEPTION_IF_NULL(context_ptr);
#ifdef ENABLE_DUMP_IR
SubModuleId module = SubModuleId::SM_OPTIMIZER;
std::string tag = "somas";

std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename);
std::string name = "somas_allocate_info." + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, name, somas_reuse_util_ptr_->SomasInfo());

filename = "somas_mem_info_" + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename);
name = "somas_mem_info." + std::to_string(graph->graph_id());
mindspore::RDR::RecordString(module, name, somas_reuse_util_ptr_->SomasMemory());
#endif
bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH);


Loading…
Cancel
Save