From a2e398d44b598514bae7d770945b932fb232d42a Mon Sep 17 00:00:00 2001 From: zhou_chao1993 Date: Fri, 19 Feb 2021 16:03:30 +0800 Subject: [PATCH] inference dump --- ge/common/dump/dump_manager.cc | 43 ++++++++++++------- ge/common/dump/dump_manager.h | 11 ++--- ge/graph/build/model_builder.cc | 3 +- ge/graph/load/model_manager/model_manager.cc | 6 +-- ge/graph/manager/graph_manager.cc | 3 +- ge/hybrid/executor/hybrid_model_executor.cc | 3 +- .../hybrid_model_pipeline_executor.cc | 3 +- ge/session/inner_session.cc | 7 +-- ge/single_op/task/op_task.cc | 4 +- 9 files changed, 50 insertions(+), 33 deletions(-) diff --git a/ge/common/dump/dump_manager.cc b/ge/common/dump/dump_manager.cc index 17019c5a..a440958b 100644 --- a/ge/common/dump/dump_manager.cc +++ b/ge/common/dump/dump_manager.cc @@ -31,14 +31,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetIn FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf(const DumpConfig &dump_config) { std::lock_guard lock(mutex_); - dump_properties_.ClearDumpPropertyValue(); - dump_properties_.ClearDumpInfo(); + DumpProperties dump_properties; + dump_properties.ClearDumpPropertyValue(); + dump_properties.ClearDumpInfo(); std::string dump_status; std::string dump_path; std::string dump_mode; std::string dump_op_switch; if (dump_config.dump_status.empty()) { + AddDumpProperties(0,dump_properties); GELOGI("Dump does not open"); return SUCCESS; } @@ -46,14 +48,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf dump_status = dump_config.dump_status; GELOGI("Dump status is %s", dump_status.c_str()); if (dump_config.dump_status == kDumpoff || dump_config.dump_status == kDumpOFF) { - dump_properties_.ClearDumpPropertyValue(); + dump_properties.ClearDumpPropertyValue(); + AddDumpProperties(0,dump_properties); return SUCCESS; } - dump_properties_.SetDumpStatus(dump_status); + dump_properties.SetDumpStatus(dump_status); dump_op_switch = dump_config.dump_op_switch; - dump_properties_.SetDumpOpSwitch(dump_op_switch); + dump_properties.SetDumpOpSwitch(dump_op_switch); if (dump_op_switch == kDumpoff && dump_config.dump_list.empty()) { + AddDumpProperties(0,dump_properties); GELOGE(PARAM_INVALID, "Dump list is invalid,dump_op_switch is %s", dump_op_switch.c_str()); return PARAM_INVALID; } @@ -67,7 +71,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf GELOGI("Dump layer is %s in model", layer.c_str()); dump_layers.insert(layer); } - dump_properties_.AddPropertyValue(model_name, dump_layers); + dump_properties.AddPropertyValue(model_name, dump_layers); } if (dump_op_switch == kDumpOn) { GELOGI("Start to dump model and single op,dumo op switch is %s", dump_op_switch.c_str()); @@ -89,27 +93,34 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf } dump_path = dump_path + CurrentTimeInStr() + "/"; GELOGI("Dump path is %s", dump_path.c_str()); - dump_properties_.SetDumpPath(dump_path); + dump_properties.SetDumpPath(dump_path); dump_mode = dump_config.dump_mode; GELOGI("Dump mode is %s", dump_mode.c_str()); - dump_properties_.SetDumpMode(dump_mode); + dump_properties.SetDumpMode(dump_mode); + AddDumpProperties(0,dump_properties); return SUCCESS; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const DumpProperties &DumpManager::GetDumpProperties() { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &DumpManager::GetDumpProperties( + uint64_t session_id) { std::lock_guard lock(mutex_); - return dump_properties_; + // If session_id is not found in dump_properties_map_, operator[] will insert one. + return dump_properties_map_[session_id]; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::SetModelName(const std::string &model_name) { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::AddDumpProperties( + uint64_t session_id, const DumpProperties &dump_properties) { std::lock_guard lock(mutex_); - model_name_ = model_name; + dump_properties_map_.emplace(session_id, dump_properties); } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpManager::GetModelName() { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::RemoveDumpProperties(uint64_t session_id) { std::lock_guard lock(mutex_); - return model_name_; -} -} // namespace ge + auto iter = dump_properties_map_.find(session_id); + if (iter != dump_properties_map_.end()) { + dump_properties_map_.erase(iter); + } +} +} // namespace ge \ No newline at end of file diff --git a/ge/common/dump/dump_manager.h b/ge/common/dump/dump_manager.h index 53a643f9..5dc34b18 100644 --- a/ge/common/dump/dump_manager.h +++ b/ge/common/dump/dump_manager.h @@ -28,14 +28,15 @@ class DumpManager { static DumpManager &GetInstance(); Status SetDumpConf(const DumpConfig &dump_config); - const DumpProperties &GetDumpProperties(); - void SetModelName(const std::string &model_name); - const std::string &GetModelName(); + DumpProperties &GetDumpProperties(uint64_t session_id); + const std::map &GetDumpPropertiesMap() { return dump_properties_map_; } + void AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties); + void RemoveDumpProperties(uint64_t session_id); private: - DumpProperties dump_properties_; + //DumpProperties dump_properties_; std::mutex mutex_; - std::string model_name_; + std::map dump_properties_map_; }; } // namespace ge #endif // GE_COMMON_DUMP_DUMP_MANAGER_H_ diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index ec891f70..10e58460 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -27,6 +27,7 @@ #include "graph/common/omg_util.h" #include "graph/common/ge_call_wrapper.h" #include "graph/common/local_context.h" +#include "common/dump/dump_manager.h" #include "graph/debug/ge_attr_define.h" #include "graph/ge_attr_value.h" #include "graph/ge_context.h" @@ -429,7 +430,7 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_), GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed."); return FAILED); - const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(session_id_); + const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_); bool is_op_debug = dump_properties.IsOpDebugOpen(); if (is_op_debug) { if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) { diff --git a/ge/graph/load/model_manager/model_manager.cc b/ge/graph/load/model_manager/model_manager.cc index 16a6ac76..aedf24bd 100755 --- a/ge/graph/load/model_manager/model_manager.cc +++ b/ge/graph/load/model_manager/model_manager.cc @@ -335,7 +335,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptrSetId(model_id); davinci_model->SetDeviceId(GetContext().DeviceId()); - const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(GetContext().SessionId()); + const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(GetContext().SessionId()); davinci_model->SetDumpProperties(dump_properties); dump_properties_ = dump_properties; @@ -1101,8 +1101,8 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model } davinci_model->SetDeviceId(device_id); davinci_model->SetOmName(model.om_name); - if (DumpManager::GetInstance().GetDumpProperties().IsDumpOpen()) { - davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties()); + if (DumpManager::GetInstance().GetDumpProperties(0).IsDumpOpen()) { + davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties(0)); } else { davinci_model->SetDumpProperties(dump_properties_); } diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index f5102f62..a184b007 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -100,6 +100,7 @@ #include "graph/common/local_context.h" #include "graph/common/omg_util.h" #include "common/formats/utils/formats_trans_utils.h" +#include "common/dump/dump_manager.h" #include "register/custom_pass_helper.h" namespace { @@ -3122,7 +3123,7 @@ Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &comp } bool is_always_dump = false; - if (!PropertiesManager::Instance().GetDumpProperties(session_id).GetDumpPath().empty()) { + if (!DumpManager::GetInstance().GetDumpProperties(session_id).GetDumpPath().empty()) { is_always_dump = true; } diff --git a/ge/hybrid/executor/hybrid_model_executor.cc b/ge/hybrid/executor/hybrid_model_executor.cc index c4154abb..80b8983a 100755 --- a/ge/hybrid/executor/hybrid_model_executor.cc +++ b/ge/hybrid/executor/hybrid_model_executor.cc @@ -17,6 +17,7 @@ #include "hybrid_model_executor.h" #include "graph/ge_context.h" #include "graph/runtime_inference_context.h" +#include "common/dump/dump_manager.h" namespace ge { namespace hybrid { @@ -107,7 +108,7 @@ Status HybridModelExecutor::InitExecutionContext() { GE_CHECK_NOTNULL(context_.allocator); context_.callback_manager = std::unique_ptr(new(std::nothrow)CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); - context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id); + context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); const char *profiling_level = std::getenv(kEnvProfilingLevel); if (profiling_level != nullptr) { context_.profiling_level = std::strtol(profiling_level, nullptr, kIntBase); diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc index b043ca7f..2b7201f9 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc @@ -3,6 +3,7 @@ #include "common/math/math_util.h" #include "graph/ge_context.h" #include "graph/runtime_inference_context.h" +#include "common/dump/dump_manager.h" namespace ge { namespace hybrid { @@ -144,7 +145,7 @@ Status StageExecutor::InitExecutionContext() { GE_CHECK_NOTNULL(context_.allocator); context_.callback_manager = std::unique_ptr(new (std::nothrow) CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); - context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id); + context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); if (IsLogEnable(GE_MODULE_NAME, DLOG_DEBUG)) { context_.trace_enabled = true; } diff --git a/ge/session/inner_session.cc b/ge/session/inner_session.cc index 6a56fc05..7d29ad94 100755 --- a/ge/session/inner_session.cc +++ b/ge/session/inner_session.cc @@ -22,6 +22,7 @@ #include "analyzer/analyzer.h" #include "adx_datadump_server.h" +#include "common/dump/dump_manager.h" #include "common/dump/dump_properties.h" #include "common/util.h" #include "framework/common/debug/ge_log.h" @@ -374,13 +375,13 @@ Status InnerSession::AddDumpProperties(const DumpProperties &dump_properties) { is_dump_server_inited_ = true; } } - PropertiesManager::Instance().AddDumpProperties(session_id_, dump_properties); + DumpManager::GetInstance().AddDumpProperties(session_id_, dump_properties); return SUCCESS; } Status InnerSession::RemoveDumpProperties() { - PropertiesManager::Instance().RemoveDumpProperties(session_id_); - if (is_dump_server_inited_ && PropertiesManager::Instance().GetDumpPropertiesMap().empty()) { + DumpManager::GetInstance().RemoveDumpProperties(session_id_); + if (is_dump_server_inited_ && DumpManager::GetInstance().GetDumpPropertiesMap().empty()) { GE_IF_BOOL_EXEC(AdxDataDumpServerUnInit() != kDumpStatus, GELOGE(PARAM_INVALID, "Data dump server uninit failed"); return PARAM_INVALID) GELOGI("UnInit adx data dump server success"); diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index ff200806..5ac36e7a 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -44,7 +44,7 @@ void FreeHbm(void *var) { } // namespace Status OpTask::OpenDump(rtStream_t stream) { - if (DumpManager::GetInstance().GetDumpProperties().IsSingleOpNeedDump()) { + if (DumpManager::GetInstance().GetDumpProperties(0).IsSingleOpNeedDump()) { GELOGI("Dump is open in single op, start to set dump info"); std::vector input_addrs; std::vector output_adds; @@ -68,7 +68,7 @@ Status OpTask::OpenDump(rtStream_t stream) { uint64_t output_addr = arg_base[input_size + j]; output_adds.emplace_back(output_addr); } - dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(), op_desc_, input_addrs, output_adds, stream); + dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(0), op_desc_, input_addrs, output_adds, stream); auto status = dump_op_.LaunchDumpOp(); if (status != SUCCESS) { GELOGE(status, "Launch dump op failed in single op");