From: @luopengting Reviewed-by: Signed-off-by:tags/v1.2.0-rc1
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -34,6 +34,7 @@ | |||||
| #include "backend/optimizer/common/helper.h" | #include "backend/optimizer/common/helper.h" | ||||
| #include "utils/ms_context.h" | #include "utils/ms_context.h" | ||||
| #include "debug/common.h" | #include "debug/common.h" | ||||
| #include "debug/rdr/running_data_recorder.h" | |||||
| #include "common/thread_pool.h" | #include "common/thread_pool.h" | ||||
| #include "profiler/device/common/memory_profiling.h" | #include "profiler/device/common/memory_profiling.h" | ||||
| @@ -98,6 +99,15 @@ bool Somas::InitSomasTensors(const session::KernelGraph *graph) { | |||||
| << nodes_list_.size() << " nodes, " << tensors_list_.size() << " tensors, and " | << nodes_list_.size() << " nodes, " << tensors_list_.size() << " tensors, and " | ||||
| << contiguous_tensors_list_.size() << " contiguous lists"; | << contiguous_tensors_list_.size() << " contiguous lists"; | ||||
| #ifdef ENABLE_DUMP_IR | |||||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||||
| std::string tag = "somas"; | |||||
| std::string filename = "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; | |||||
| mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); | |||||
| filename = "somas_offline_log_" + std::to_string(graph->graph_id()) + ".ir"; | |||||
| mindspore::RDR::RecordString(module, tag, Offline(), filename); | |||||
| #endif | |||||
| if (save_graphs_) { | if (save_graphs_) { | ||||
| std::string file_path = | std::string file_path = | ||||
| save_graphs_path_ + "/" + "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; | save_graphs_path_ + "/" + "somas_pre_processed_info_" + std::to_string(graph->graph_id()) + ".ir"; | ||||
| @@ -380,6 +390,14 @@ void Somas::InitBasicInfo(const session::KernelGraph *graph) { | |||||
| auto context_ptr = MsContext::GetInstance(); | auto context_ptr = MsContext::GetInstance(); | ||||
| MS_EXCEPTION_IF_NULL(context_ptr); | MS_EXCEPTION_IF_NULL(context_ptr); | ||||
| #ifdef ENABLE_DUMP_IR | |||||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||||
| std::string tag = "somas"; | |||||
| std::string filename = "somas_initial_info_" + std::to_string(graph->graph_id()) + ".ir"; | |||||
| mindspore::RDR::RecordString(module, tag, SomasInfo(), filename); | |||||
| #endif | |||||
| save_graphs_ = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | save_graphs_ = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | ||||
| save_graphs_path_ = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH); | save_graphs_path_ = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH); | ||||
| if (save_graphs_path_.empty()) { | if (save_graphs_path_.empty()) { | ||||
| @@ -1089,91 +1107,76 @@ std::string Somas::GetSplitName(const std::string &scope_name) const { | |||||
| } | } | ||||
| } | } | ||||
| void Somas::DumpSomasInfoIR(const string filename) { | |||||
| if (filename.size() > PATH_MAX) { | |||||
| MS_LOG(ERROR) << "File path " << filename << " is too long."; | |||||
| return; | |||||
| } | |||||
| auto real_path = Common::GetRealPath(filename); | |||||
| if (!real_path.has_value()) { | |||||
| MS_LOG(ERROR) << "Get real path failed. path=" << filename; | |||||
| return; | |||||
| } | |||||
| std::string Somas::SomasInfo() { | |||||
| std::ostringstream oss; | |||||
| ChangeFileMode(real_path.value(), S_IRWXU); | |||||
| std::ofstream ofs(real_path.value()); | |||||
| if (!ofs.is_open()) { | |||||
| MS_LOG(ERROR) << "Open dump file '" << real_path.value() << "' failed!"; | |||||
| return; | |||||
| } | |||||
| DumpParameters(oss); | |||||
| DumpTensors(oss); | |||||
| DumpNodes(oss); | |||||
| DumpParameters(ofs); | |||||
| DumpTensors(ofs); | |||||
| DumpNodes(ofs); | |||||
| ofs << "\n\nAll Stream Groups:\n\n"; | |||||
| oss << "\n\nAll Stream Groups:\n\n"; | |||||
| for (const auto &stream_group : streams_groups_) { | for (const auto &stream_group : streams_groups_) { | ||||
| for (const auto &stream : stream_group) { | for (const auto &stream : stream_group) { | ||||
| ofs << "stm" << stream << " "; | |||||
| oss << "stm" << stream << " "; | |||||
| } | } | ||||
| ofs << "\n"; | |||||
| oss << "\n"; | |||||
| } | } | ||||
| if (!ref_node_constraints_.empty()) { | if (!ref_node_constraints_.empty()) { | ||||
| ofs << "\n\nAll Ref Node Info:\n\n"; | |||||
| oss << "\n\nAll Ref Node Info:\n\n"; | |||||
| for (const auto &ref_in_out : ref_node_constraints_) { | for (const auto &ref_in_out : ref_node_constraints_) { | ||||
| ofs << "refnode input-output:"; | |||||
| oss << "refnode input-output:"; | |||||
| for (const auto &item : ref_in_out) { | for (const auto &item : ref_in_out) { | ||||
| ofs << "%" << item << "T "; | |||||
| oss << "%" << item << "T "; | |||||
| } | } | ||||
| ofs << "\n"; | |||||
| oss << "\n"; | |||||
| } | } | ||||
| } | } | ||||
| return oss.str(); | |||||
| } | } | ||||
| void Somas::DumpNodes(std::ofstream &ofs) const { | |||||
| ofs << "\n\nAll Nodes:\n\n"; | |||||
| void Somas::DumpNodes(std::ostringstream &oss) const { | |||||
| oss << "\n\nAll Nodes:\n\n"; | |||||
| for (const auto &node : nodes_list_) { | for (const auto &node : nodes_list_) { | ||||
| auto scope_name = node->scope_full_name_; | auto scope_name = node->scope_full_name_; | ||||
| std::string split_name = GetSplitName(scope_name); | std::string split_name = GetSplitName(scope_name); | ||||
| ofs << "$" << node->GetId() << "\t" << split_name << "\t" << static_cast<int>(node->GetType()) << "\t"; | |||||
| oss << "$" << node->GetId() << "\t" << split_name << "\t" << static_cast<int>(node->GetType()) << "\t"; | |||||
| auto input_num = node->input_tensors_.size() + node->input_parameters_map_.size(); | auto input_num = node->input_tensors_.size() + node->input_parameters_map_.size(); | ||||
| ofs << "inputs["; | |||||
| oss << "inputs["; | |||||
| size_t tensor_index = 0; | size_t tensor_index = 0; | ||||
| for (size_t input_index = 0; input_index < input_num; input_index++) { | for (size_t input_index = 0; input_index < input_num; input_index++) { | ||||
| auto iter = node->input_parameters_map_.find(input_index); | auto iter = node->input_parameters_map_.find(input_index); | ||||
| if (iter != node->input_parameters_map_.end()) { | if (iter != node->input_parameters_map_.end()) { | ||||
| ofs << "%" << iter->second->id_ << "P" | |||||
| oss << "%" << iter->second->id_ << "P" | |||||
| << ", "; | << ", "; | ||||
| } else { | } else { | ||||
| ofs << "%" << node->input_tensors_[tensor_index]->GetId() << "T" | |||||
| oss << "%" << node->input_tensors_[tensor_index]->GetId() << "T" | |||||
| << ", "; | << ", "; | ||||
| tensor_index++; | tensor_index++; | ||||
| } | } | ||||
| } | } | ||||
| ofs << "]"; | |||||
| ofs << "\toutputs["; | |||||
| oss << "]"; | |||||
| oss << "\toutputs["; | |||||
| for (const auto &out : node->output_tensors_) { | for (const auto &out : node->output_tensors_) { | ||||
| ofs << "%" << out->GetId() << "T" | |||||
| oss << "%" << out->GetId() << "T" | |||||
| << ", "; | << ", "; | ||||
| } | } | ||||
| ofs << "]"; | |||||
| ofs << "\tworkspace["; | |||||
| oss << "]"; | |||||
| oss << "\tworkspace["; | |||||
| for (const auto &wk : node->workspace_tensors_) { | for (const auto &wk : node->workspace_tensors_) { | ||||
| ofs << "%" << wk->GetId() << "T" | |||||
| oss << "%" << wk->GetId() << "T" | |||||
| << ", "; | << ", "; | ||||
| } | } | ||||
| ofs << "]"; | |||||
| ofs << "\tstreamID[" | |||||
| oss << "]"; | |||||
| oss << "\tstreamID[" | |||||
| << "@" << node->GetStream()->GetId() << "]\n"; | << "@" << node->GetStream()->GetId() << "]\n"; | ||||
| } | } | ||||
| } | } | ||||
| void Somas::DumpTensors(std::ofstream &ofs) const { | |||||
| ofs << "\n\nAll Tensors:\n\n"; | |||||
| ofs << "index:" | |||||
| void Somas::DumpTensors(std::ostringstream &oss) const { | |||||
| oss << "\n\nAll Tensors:\n\n"; | |||||
| oss << "index:" | |||||
| << "\tsize:" | << "\tsize:" | ||||
| << "\treal_size:" | << "\treal_size:" | ||||
| << "\toffset:" | << "\toffset:" | ||||
| @@ -1187,7 +1190,7 @@ void Somas::DumpTensors(std::ofstream &ofs) const { | |||||
| for (const auto &tensor : tensors_list_) { | for (const auto &tensor : tensors_list_) { | ||||
| auto scope_name = tensor->GetSourceNode()->scope_full_name_; | auto scope_name = tensor->GetSourceNode()->scope_full_name_; | ||||
| std::string split_name = GetSplitName(scope_name); | std::string split_name = GetSplitName(scope_name); | ||||
| ofs << "%" << tensor->GetId() << "T" | |||||
| oss << "%" << tensor->GetId() << "T" | |||||
| << "\t" | << "\t" | ||||
| << "#" << tensor->GetAlignedSize() << "S" | << "#" << tensor->GetAlignedSize() << "S" | ||||
| << "\t" | << "\t" | ||||
| @@ -1201,16 +1204,16 @@ void Somas::DumpTensors(std::ofstream &ofs) const { | |||||
| } | } | ||||
| } | } | ||||
| void Somas::DumpParameters(std::ofstream &ofs) const { | |||||
| ofs << "All Parameters:\n\n"; | |||||
| ofs << "index:" | |||||
| void Somas::DumpParameters(std::ostringstream &oss) const { | |||||
| oss << "All Parameters:\n\n"; | |||||
| oss << "index:" | |||||
| << "\tsize:" | << "\tsize:" | ||||
| << "\tstart_addr:" | << "\tstart_addr:" | ||||
| << "\tsource node name:" | << "\tsource node name:" | ||||
| << "\tnode out index:\n"; | << "\tnode out index:\n"; | ||||
| for (const auto ¶m : parameters_list_) { | for (const auto ¶m : parameters_list_) { | ||||
| ofs << "%" << param->id_ << "P" | |||||
| oss << "%" << param->id_ << "P" | |||||
| << "\t" | << "\t" | ||||
| << "#" << param->size_ << "S" | << "#" << param->size_ << "S" | ||||
| << "\t" | << "\t" | ||||
| @@ -1219,13 +1222,11 @@ void Somas::DumpParameters(std::ofstream &ofs) const { | |||||
| } | } | ||||
| } | } | ||||
| void Somas::DumpOfflineIR(const string filename) { | |||||
| MS_LOG(INFO) << "Printing somas-log-from-graph log: " << filename; | |||||
| void Somas::DumpSomasInfoIR(const string filename) { | |||||
| if (filename.size() > PATH_MAX) { | if (filename.size() > PATH_MAX) { | ||||
| MS_LOG(ERROR) << "File path " << filename << " is too long."; | MS_LOG(ERROR) << "File path " << filename << " is too long."; | ||||
| return; | return; | ||||
| } | } | ||||
| auto real_path = Common::GetRealPath(filename); | auto real_path = Common::GetRealPath(filename); | ||||
| if (!real_path.has_value()) { | if (!real_path.has_value()) { | ||||
| MS_LOG(ERROR) << "Get real path failed. path=" << filename; | MS_LOG(ERROR) << "Get real path failed. path=" << filename; | ||||
| @@ -1240,9 +1241,16 @@ void Somas::DumpOfflineIR(const string filename) { | |||||
| return; | return; | ||||
| } | } | ||||
| ofs << SomasInfo(); | |||||
| ofs.close(); | |||||
| } | |||||
| std::string Somas::Offline() { | |||||
| std::ostringstream oss; | |||||
| for (auto tensor : tensors_list_) { | for (auto tensor : tensors_list_) { | ||||
| if (tensor->IsOutputOnly() || tensor->type_ == TensorType::kRefNodeOutput) { | if (tensor->IsOutputOnly() || tensor->type_ == TensorType::kRefNodeOutput) { | ||||
| ofs << "Somas EDGE ERROR src=n" << tensor->GetSourceNode()->GetId() | |||||
| oss << "Somas EDGE ERROR src=n" << tensor->GetSourceNode()->GetId() | |||||
| << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=nc" | << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=nc" | ||||
| << ", dststm=nc" | << ", dststm=nc" | ||||
| << ", workspace=0, size=" << tensor->GetOriginalSize() | << ", workspace=0, size=" << tensor->GetOriginalSize() | ||||
| @@ -1255,7 +1263,7 @@ void Somas::DumpOfflineIR(const string filename) { | |||||
| } | } | ||||
| for (auto dest_info : dest_infos) { | for (auto dest_info : dest_infos) { | ||||
| ofs << "Somas EDGE src=n" << tensor->GetSourceNode()->GetId() | |||||
| oss << "Somas EDGE src=n" << tensor->GetSourceNode()->GetId() | |||||
| << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=n" << dest_info.first | << ", srcstm=" << tensor->GetSourceStream()->GetId() << ", dst=n" << dest_info.first | ||||
| << ", dststm=" << dest_info.second << ", workspace=" << static_cast<int>(tensor->type_ == kWorkspace) | << ", dststm=" << dest_info.second << ", workspace=" << static_cast<int>(tensor->type_ == kWorkspace) | ||||
| << ", size=" << tensor->GetOriginalSize() << ", lifelong=" << static_cast<int>(tensor->lifelong_value_) | << ", size=" << tensor->GetOriginalSize() << ", lifelong=" << static_cast<int>(tensor->lifelong_value_) | ||||
| @@ -1265,23 +1273,24 @@ void Somas::DumpOfflineIR(const string filename) { | |||||
| } | } | ||||
| } | } | ||||
| for (vector<size_t> tList : contiguous_tensors_list_) { | for (vector<size_t> tList : contiguous_tensors_list_) { | ||||
| ofs << "Somas CONTIGUOUS"; | |||||
| oss << "Somas CONTIGUOUS"; | |||||
| for (size_t tid : tList) { | for (size_t tid : tList) { | ||||
| ofs << " " << tid; | |||||
| oss << " " << tid; | |||||
| } | } | ||||
| ofs << std::endl; | |||||
| oss << std::endl; | |||||
| } | } | ||||
| for (const auto &group : streams_groups_) { | for (const auto &group : streams_groups_) { | ||||
| ofs << "Somas GROUP"; | |||||
| oss << "Somas GROUP"; | |||||
| for (int64_t sid : group) { | for (int64_t sid : group) { | ||||
| ofs << " " << sid; | |||||
| oss << " " << sid; | |||||
| } | } | ||||
| ofs << std::endl; | |||||
| oss << std::endl; | |||||
| } | } | ||||
| ofs.close(); | |||||
| return oss.str(); | |||||
| } | } | ||||
| void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| void Somas::DumpOfflineIR(const string filename) { | |||||
| MS_LOG(INFO) << "Printing somas-log-from-graph log: " << filename; | |||||
| if (filename.size() > PATH_MAX) { | if (filename.size() > PATH_MAX) { | ||||
| MS_LOG(ERROR) << "File path " << filename << " is too long."; | MS_LOG(ERROR) << "File path " << filename << " is too long."; | ||||
| return; | return; | ||||
| @@ -1301,6 +1310,13 @@ void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| return; | return; | ||||
| } | } | ||||
| ofs << Offline(); | |||||
| ofs.close(); | |||||
| } | |||||
| std::string Somas::SomasMemory() { | |||||
| std::ostringstream oss; | |||||
| std::map<size_t, size_t> mem_map; | std::map<size_t, size_t> mem_map; | ||||
| for (auto tensor : tensors_list_) { | for (auto tensor : tensors_list_) { | ||||
| mem_map[tensor->GetOffset()] = 0; | mem_map[tensor->GetOffset()] = 0; | ||||
| @@ -1325,7 +1341,7 @@ void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| } | } | ||||
| } | } | ||||
| ofs << "mem_id:" | |||||
| oss << "mem_id:" | |||||
| << "\tstart_offset:" | << "\tstart_offset:" | ||||
| << "\tend_offset:" | << "\tend_offset:" | ||||
| << "\ttensor_id:" | << "\ttensor_id:" | ||||
| @@ -1353,7 +1369,7 @@ void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| } | } | ||||
| std::string split_name = GetSplitName(scope_name); | std::string split_name = GetSplitName(scope_name); | ||||
| ofs << "#" << mem_map[tensor->GetOffset()] << "\t" << tensor->GetOffset() << "\t" | |||||
| oss << "#" << mem_map[tensor->GetOffset()] << "\t" << tensor->GetOffset() << "\t" | |||||
| << tensor->GetOffset() + tensor->GetAlignedSize() << "\t%" << tensor->GetId() << "T\t" | << tensor->GetOffset() + tensor->GetAlignedSize() << "\t%" << tensor->GetId() << "T\t" | ||||
| << tensor->GetOriginalSize() << "\t" << tensor->GetAlignedSize() << "\t&" | << tensor->GetOriginalSize() << "\t" << tensor->GetAlignedSize() << "\t&" | ||||
| << static_cast<void *>(tensor->GetOffset() + mem_base_addr_) << "\t&" | << static_cast<void *>(tensor->GetOffset() + mem_base_addr_) << "\t&" | ||||
| @@ -1362,6 +1378,31 @@ void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| << tensor->lifetime_.start_ << "\t" << tensor->lifetime_.end_ << "\n"; | << tensor->lifetime_.start_ << "\t" << tensor->lifetime_.end_ << "\n"; | ||||
| } | } | ||||
| } | } | ||||
| return oss.str(); | |||||
| } | |||||
| void Somas::DumpSomasMemoryIR(const string filename) { | |||||
| if (filename.size() > PATH_MAX) { | |||||
| MS_LOG(ERROR) << "File path " << filename << " is too long."; | |||||
| return; | |||||
| } | |||||
| auto real_path = Common::GetRealPath(filename); | |||||
| if (!real_path.has_value()) { | |||||
| MS_LOG(ERROR) << "Get real path failed. path=" << filename; | |||||
| return; | |||||
| } | |||||
| ChangeFileMode(real_path.value(), S_IRWXU); | |||||
| std::ofstream ofs(real_path.value()); | |||||
| if (!ofs.is_open()) { | |||||
| MS_LOG(ERROR) << "Open dump file '" << real_path.value() << "' failed!"; | |||||
| return; | |||||
| } | |||||
| ofs << SomasMemory(); | |||||
| ofs.close(); | |||||
| } | } | ||||
| size_t Somas::CalcLowerBound() const { | size_t Somas::CalcLowerBound() const { | ||||
| @@ -1,5 +1,5 @@ | |||||
| /** | /** | ||||
| * Copyright 2020 Huawei Technologies Co., Ltd | |||||
| * Copyright 2020-2021 Huawei Technologies Co., Ltd | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | * Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| * you may not use this file except in compliance with the License. | * you may not use this file except in compliance with the License. | ||||
| @@ -49,6 +49,8 @@ class Somas { | |||||
| uint8_t *GetNodeOutputPtr(const AnfNodePtr &node, size_t index) const; | uint8_t *GetNodeOutputPtr(const AnfNodePtr &node, size_t index) const; | ||||
| uint8_t *GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const; | uint8_t *GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const; | ||||
| std::string SomasInfo(); | |||||
| std::string SomasMemory(); | |||||
| void DumpSomasInfoIR(const string filename); | void DumpSomasInfoIR(const string filename); | ||||
| void DumpSomasMemoryIR(const string filename); | void DumpSomasMemoryIR(const string filename); | ||||
| @@ -121,8 +123,8 @@ class Somas { | |||||
| bool Assign(const session::KernelGraph *graph); | bool Assign(const session::KernelGraph *graph); | ||||
| std::string Offline(); | |||||
| void DumpOfflineIR(const string filename); | void DumpOfflineIR(const string filename); | ||||
| void DumpSomasMemoryPoolInfoIR(const string filename); | |||||
| std::string GetSplitName(const string &scope_name) const; | std::string GetSplitName(const string &scope_name) const; | ||||
| size_t CalcLowerBound() const; | size_t CalcLowerBound() const; | ||||
| void GenGraphStatisticInfo(); | void GenGraphStatisticInfo(); | ||||
| @@ -143,9 +145,9 @@ class Somas { | |||||
| void UpdateRefOverlapTensorsConflicts(); | void UpdateRefOverlapTensorsConflicts(); | ||||
| void UpdateRefTensorsOffset(); | void UpdateRefTensorsOffset(); | ||||
| void UpdateContiguousTensorsOffset(const std::map<size_t, size_t> &contiguous_ref_list_map); | void UpdateContiguousTensorsOffset(const std::map<size_t, size_t> &contiguous_ref_list_map); | ||||
| void DumpParameters(std::ofstream &ofs) const; | |||||
| void DumpTensors(std::ofstream &ofs) const; | |||||
| void DumpNodes(std::ofstream &ofs) const; | |||||
| void DumpParameters(std::ostringstream &oss) const; | |||||
| void DumpTensors(std::ostringstream &oss) const; | |||||
| void DumpNodes(std::ostringstream &oss) const; | |||||
| std::map<size_t, size_t> GetContiguousListContainRefTensor(); | std::map<size_t, size_t> GetContiguousListContainRefTensor(); | ||||
| std::map<size_t, size_t> GetRefTensorsInContiguousList(); | std::map<size_t, size_t> GetRefTensorsInContiguousList(); | ||||
| }; | }; | ||||
| @@ -8,7 +8,7 @@ set(_DEBUG_SRC_LIST | |||||
| "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc" | "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc" | ||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" | "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_exec_order_recorder.cc" | ||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc" | "${CMAKE_CURRENT_SOURCE_DIR}/rdr/graph_recorder.cc" | ||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/somas_recorder.cc" | |||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/string_recorder.cc" | |||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc" | "${CMAKE_CURRENT_SOURCE_DIR}/rdr/recorder_manager.cc" | ||||
| "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc" | "${CMAKE_CURRENT_SOURCE_DIR}/rdr/running_data_recorder.cc" | ||||
| ) | ) | ||||
| @@ -16,9 +16,9 @@ | |||||
| #include "debug/rdr/running_data_recorder.h" | #include "debug/rdr/running_data_recorder.h" | ||||
| #include <utility> | #include <utility> | ||||
| #include "debug/rdr/graph_recorder.h" | #include "debug/rdr/graph_recorder.h" | ||||
| #include "debug/rdr/somas_recorder.h" | |||||
| #include "debug/rdr/graph_exec_order_recorder.h" | #include "debug/rdr/graph_exec_order_recorder.h" | ||||
| #include "debug/rdr/recorder_manager.h" | #include "debug/rdr/recorder_manager.h" | ||||
| #include "debug/rdr/string_recorder.h" | |||||
| #include "mindspore/core/ir/func_graph.h" | #include "mindspore/core/ir/func_graph.h" | ||||
| #include "mindspore/core/ir/anf.h" | #include "mindspore/core/ir/anf.h" | ||||
| @@ -75,15 +75,16 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, | |||||
| return ans; | return ans; | ||||
| } | } | ||||
| bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { | |||||
| bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { | |||||
| std::string submodule_name = std::string(GetSubModuleName(module)); | std::string submodule_name = std::string(GetSubModuleName(module)); | ||||
| SomasRecorderPtr somas_recorder = std::make_shared<SomasRecorder>(submodule_name, tag, somas_ptr, graph_id); | |||||
| somas_recorder->GenString(); | |||||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(somas_recorder)); | |||||
| StringRecorderPtr string_recorder = std::make_shared<StringRecorder>(submodule_name, tag, data, filename); | |||||
| string_recorder->SetFilename(filename); | |||||
| bool ans = mindspore::RecorderManager::Instance().RecordObject(std::move(string_recorder)); | |||||
| return ans; | return ans; | ||||
| } | } | ||||
| void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } | void TriggerAll() { mindspore::RecorderManager::Instance().TriggerAll(); } | ||||
| #else | #else | ||||
| bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, | bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, | ||||
| const std::string &file_type, int graph_id) { | const std::string &file_type, int graph_id) { | ||||
| @@ -107,7 +108,7 @@ bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, std: | |||||
| return false; | return false; | ||||
| } | } | ||||
| bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id) { | |||||
| bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, const std::string &filename) { | |||||
| static bool already_printed = false; | static bool already_printed = false; | ||||
| if (already_printed) { | if (already_printed) { | ||||
| return false; | return false; | ||||
| @@ -116,6 +117,7 @@ bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const Som | |||||
| MS_LOG(WARNING) << "The RDR presently only support linux os."; | MS_LOG(WARNING) << "The RDR presently only support linux os."; | ||||
| return false; | return false; | ||||
| } | } | ||||
| void TriggerAll() { | void TriggerAll() { | ||||
| static bool already_printed = false; | static bool already_printed = false; | ||||
| if (already_printed) { | if (already_printed) { | ||||
| @@ -13,28 +13,27 @@ | |||||
| * See the License for the specific language governing permissions and | * See the License for the specific language governing permissions and | ||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #ifndef RUNNING_DATA_RECORDER_H_ | |||||
| #define RUNNING_DATA_RECORDER_H_ | |||||
| #ifndef MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ | |||||
| #define MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ | |||||
| #include <vector> | #include <vector> | ||||
| #include <string> | #include <string> | ||||
| #include <memory> | #include <memory> | ||||
| #include "backend/optimizer/somas/somas.h" | |||||
| #include "mindspore/core/utils/log_adapter.h" | #include "mindspore/core/utils/log_adapter.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| class FuncGraph; | class FuncGraph; | ||||
| class CNode; | class CNode; | ||||
| using FuncGraphPtr = std::shared_ptr<FuncGraph>; | using FuncGraphPtr = std::shared_ptr<FuncGraph>; | ||||
| using CNodePtr = std::shared_ptr<CNode>; | using CNodePtr = std::shared_ptr<CNode>; | ||||
| using SomasPtr = std::shared_ptr<somas::Somas>; | |||||
| namespace RDR { | namespace RDR { | ||||
| bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, | bool RecordAnfGraph(const SubModuleId module, const std::string &tag, const FuncGraphPtr &graph, | ||||
| const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0); | const std::string &file_type = ".ir;.pb;.dat", int graph_id = 0); | ||||
| bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, | bool RecordGraphExecOrder(const SubModuleId module, const std::string &tag, | ||||
| const std::vector<CNodePtr> &&final_exec_order); | const std::vector<CNodePtr> &&final_exec_order); | ||||
| bool RecordSomasInfo(const SubModuleId module, const std::string &tag, const SomasPtr &somas_ptr, int graph_id); | |||||
| bool RecordString(SubModuleId module, const std::string &tag, const std::string &data, | |||||
| const std::string &filename = ""); | |||||
| void TriggerAll(); | void TriggerAll(); | ||||
| } // namespace RDR | } // namespace RDR | ||||
| } // namespace mindspore | } // namespace mindspore | ||||
| #endif // RUNNING_DATA_RECORDER_H_ | |||||
| #endif // MINDSPORE_CCSRC_DEBUG_RDR_RUNNING_DATA_RECORDER_H_ | |||||
| @@ -1,30 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2021 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #include "debug/rdr/somas_recorder.h" | |||||
| #include "backend/optimizer/somas/somas.h" | |||||
| namespace mindspore { | |||||
| void SomasRecorder::Export() { | |||||
| if (filename_.empty()) { | |||||
| filename_ = directory_ + module_ + "_" + tag_; | |||||
| } | |||||
| std::string filename = filename_ + "_somas_after_allocate_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; | |||||
| somas_reuse_util_ptr_->DumpSomasInfoIR(filename); | |||||
| std::string mem_filename = filename_ + "_somas_mem_info_" + std::to_string(graph_id_) + "_" + timestamp_ + ".ir"; | |||||
| somas_reuse_util_ptr_->DumpSomasMemoryIR(mem_filename); | |||||
| } | |||||
| bool SomasRecorder::GenString() { return true; } | |||||
| } // namespace mindspore | |||||
| @@ -0,0 +1,51 @@ | |||||
| /** | |||||
| * Copyright 2021 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #include "debug/rdr/string_recorder.h" | |||||
| #include <sys/stat.h> | |||||
| #include <fstream> | |||||
| #include "debug/common.h" | |||||
| #include "utils/utils.h" | |||||
| #include "mindspore/core/utils/log_adapter.h" | |||||
| namespace mindspore { | |||||
| void StringRecorder::Export() { | |||||
| if (directory_.back() != '/') { | |||||
| directory_ += "/"; | |||||
| } | |||||
| if (filename_.empty()) { | |||||
| filename_ = module_ + "_" + tag_ + "_" + timestamp_ + ".txt"; | |||||
| } | |||||
| std::string file_path = directory_ + filename_; | |||||
| auto realpath = Common::GetRealPath(file_path); | |||||
| if (!realpath.has_value()) { | |||||
| MS_LOG(ERROR) << "Get real path failed. path=" << file_path; | |||||
| return; | |||||
| } | |||||
| ChangeFileMode(realpath.value(), S_IRWXU); | |||||
| std::ofstream fout(realpath.value(), std::ofstream::app); | |||||
| if (!fout.is_open()) { | |||||
| MS_LOG(WARNING) << "Open file for saving string failed."; | |||||
| return; | |||||
| } | |||||
| fout << data_; | |||||
| fout.close(); | |||||
| // set file mode to read only by user | |||||
| ChangeFileMode(realpath.value(), S_IRUSR); | |||||
| } | |||||
| } // namespace mindspore | |||||
| @@ -13,34 +13,28 @@ | |||||
| * See the License for the specific language governing permissions and | * See the License for the specific language governing permissions and | ||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #ifndef SOMAS_RECORDER_H_ | |||||
| #define SOMAS_RECORDER_H_ | |||||
| #ifndef MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ | |||||
| #define MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ | |||||
| #include <string> | #include <string> | ||||
| #include <sstream> | |||||
| #include <memory> | #include <memory> | ||||
| #include "debug/rdr/base_recorder.h" | |||||
| #include "debug/rdr/base_recorder.h" | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace somas { | |||||
| class Somas; | |||||
| using SomasPtr = std::shared_ptr<Somas>; | |||||
| } // namespace somas | |||||
| class SomasRecorder : public BaseRecorder { | |||||
| class StringRecorder : public BaseRecorder { | |||||
| public: | public: | ||||
| SomasRecorder() : BaseRecorder(), somas_reuse_util_ptr_(nullptr) {} | |||||
| SomasRecorder(const std::string &module, const std::string &tag, const somas::SomasPtr &somas_reuse_util_ptr, | |||||
| int graph_id) | |||||
| : BaseRecorder(module, tag), somas_reuse_util_ptr_(somas_reuse_util_ptr), graph_id_(graph_id) {} | |||||
| StringRecorder() : BaseRecorder() {} | |||||
| StringRecorder(const std::string &module, const std::string &tag, const std::string &data, | |||||
| const std::string &file_type) | |||||
| : BaseRecorder(module, tag), data_(data) {} | |||||
| ~StringRecorder() {} | |||||
| void SetModule(const std::string &module) { module_ = module; } | void SetModule(const std::string &module) { module_ = module; } | ||||
| void SetSomas(const somas::SomasPtr &somas_reuse_util_ptr) { somas_reuse_util_ptr_ = somas_reuse_util_ptr; } | |||||
| bool GenString(); | |||||
| void SetFilename(const std::string &filename) { filename_ = filename; } | |||||
| virtual void Export(); | virtual void Export(); | ||||
| private: | private: | ||||
| somas::SomasPtr somas_reuse_util_ptr_; | |||||
| int graph_id_{0}; | |||||
| std::string data_; | |||||
| }; | }; | ||||
| using SomasRecorderPtr = std::shared_ptr<SomasRecorder>; | |||||
| using StringRecorderPtr = std::shared_ptr<StringRecorder>; | |||||
| } // namespace mindspore | } // namespace mindspore | ||||
| #endif // SOMAS_RECORDER_H | |||||
| #endif // MINDSPORE_CCSRC_DEBUG_RDR_STRING_RECORDER_H_ | |||||
| @@ -17,6 +17,7 @@ | |||||
| #include "runtime/device/memory_manager.h" | #include "runtime/device/memory_manager.h" | ||||
| #include <string> | #include <string> | ||||
| #include "backend/session/anf_runtime_algorithm.h" | #include "backend/session/anf_runtime_algorithm.h" | ||||
| #include "debug/rdr/running_data_recorder.h" | |||||
| #include "utils/ms_context.h" | #include "utils/ms_context.h" | ||||
| using mindspore::memreuse::BestFitMemReuse; | using mindspore::memreuse::BestFitMemReuse; | ||||
| @@ -69,6 +70,16 @@ void MemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) { | |||||
| auto context_ptr = MsContext::GetInstance(); | auto context_ptr = MsContext::GetInstance(); | ||||
| MS_EXCEPTION_IF_NULL(context_ptr); | MS_EXCEPTION_IF_NULL(context_ptr); | ||||
| #ifdef ENABLE_DUMP_IR | |||||
| SubModuleId module = SubModuleId::SM_OPTIMIZER; | |||||
| std::string tag = "somas"; | |||||
| std::string filename = "somas_allocate_info_" + std::to_string(graph->graph_id()) + ".ir"; | |||||
| mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasInfo(), filename); | |||||
| filename = "somas_mem_info_" + std::to_string(graph->graph_id()) + ".ir"; | |||||
| mindspore::RDR::RecordString(module, tag, somas_reuse_util_ptr_->SomasMemory(), filename); | |||||
| #endif | |||||
| bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG); | ||||
| auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH); | auto save_graphs_path = context_ptr->get_param<std::string>(MS_CTX_SAVE_GRAPHS_PATH); | ||||
| if (save_graphs_path.empty()) { | if (save_graphs_path.empty()) { | ||||