diff --git a/mindspore/ccsrc/profiler/device/ascend/ascend_profiling.h b/mindspore/ccsrc/profiler/device/ascend/ascend_profiling.h index 4980f1d8c8..6489e73d9c 100644 --- a/mindspore/ccsrc/profiler/device/ascend/ascend_profiling.h +++ b/mindspore/ccsrc/profiler/device/ascend/ascend_profiling.h @@ -53,7 +53,6 @@ class AscendProfiler { std::vector events_; std::atomic_int counter_; }; - } // namespace ascend } // namespace profiler } // namespace mindspore diff --git a/mindspore/ccsrc/profiler/device/ascend/profiling_context.h b/mindspore/ccsrc/profiler/device/ascend/profiling_context.h index 08be9e0e4e..e8e233615a 100644 --- a/mindspore/ccsrc/profiler/device/ascend/profiling_context.h +++ b/mindspore/ccsrc/profiler/device/ascend/profiling_context.h @@ -26,7 +26,6 @@ namespace mindspore { namespace profiler { namespace ascend { - inline pid_t GetTid() { thread_local static pid_t tid = syscall(__NR_gettid); return tid; diff --git a/mindspore/ccsrc/profiler/device/ascend/rt_callback_manager.cc b/mindspore/ccsrc/profiler/device/ascend/rt_callback_manager.cc index b36e57e399..48ddec7577 100644 --- a/mindspore/ccsrc/profiler/device/ascend/rt_callback_manager.cc +++ b/mindspore/ccsrc/profiler/device/ascend/rt_callback_manager.cc @@ -21,7 +21,6 @@ namespace mindspore { namespace profiler { namespace ascend { - CallbackManager::CallbackManager(rtStream_t stream) : stream_(stream) {} Status CallbackManager::Init() { diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc index 32b4a80fba..da66d338fb 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_stream_assign.cc @@ -155,72 +155,15 @@ void AscendStreamAssign::AssignAllNodesStream(const NotNull &gra MS_LOG(INFO) << "Common start from 0, common stream nums:" << resource_manager.get_cur_stream_num(); if (exit_hcom) { - std::map> graph_nodes_map; - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; - // node has been assigned stream before - if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { - continue; - } - - if (IsHcom(cur_cnode_ptr)) { - auto hcom_graph_id = AnfAlgo::GetGraphId(cur_cnode_ptr.get()); - auto it = graph_nodes_map.find(hcom_graph_id); - if (it == graph_nodes_map.end()) { - graph_nodes_map[hcom_graph_id] = {cur_cnode_ptr}; - } else { - it->second.emplace_back(cur_cnode_ptr); - } - } - } - MS_LOG(INFO) << "hcom diff graph id size:" << graph_nodes_map.size(); - for (const auto &item : graph_nodes_map) { - bool new_graph = true; - auto graph_id = item.first; - hcom_graph_map_[graph_id] = {}; - for (const auto &hcom_node_ptr : item.second) { - auto assigned_stream_id = AssignHcomStreamId(hcom_node_ptr, new_graph); - hcom_graph_map_[graph_id].emplace(assigned_stream_id); - new_graph = false; - } - } - MS_LOG(INFO) << "hcom stream nums : " << hcom_stream_map_.size(); + AssignHcom(graph_ptr); } if (exit_independent) { - std::map> graph_nodes_map; - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; - if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { - continue; - } - if (AnfAlgo::IsIndependentNode(cur_cnode_ptr)) { - auto independent_graph_id = AnfAlgo::GetGraphId(cur_cnode_ptr.get()); - auto it = graph_nodes_map.find(independent_graph_id); - if (it == graph_nodes_map.end()) { - graph_nodes_map[independent_graph_id] = {cur_cnode_ptr}; - } else { - it->second.emplace_back(cur_cnode_ptr); - } - } - } - - MS_LOG(INFO) << "independent diff graph id size:" << graph_nodes_map.size(); - for (const auto &item : graph_nodes_map) { - bool new_graph = true; - auto graph_id = item.first; - independent_graph_map_[graph_id] = {}; - for (const auto &independent_node_ptr : item.second) { - auto assigned_stream_id = AssignIndependentStreamId(independent_node_ptr, new_graph); - independent_graph_map_[graph_id].emplace(assigned_stream_id); - new_graph = false; - } - } - MS_LOG(INFO) << "stream nums:" << independent_stream_map_.size(); + AssignIndependent(graph_ptr); } MS_LOG(INFO) << "After stream assign, total stream nums:" << resource_manager.get_cur_stream_num(); -} // namespace ascend +} void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr) { MS_EXCEPTION_IF_NULL(cur_cnode_ptr); @@ -249,6 +192,40 @@ void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr) { } } +void AscendStreamAssign::AssignHcom(const NotNull &graph_ptr) { + auto cnode_ptr_list = graph_ptr->execution_order(); + std::map> graph_nodes_map; + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + // node has been assigned stream before + if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { + continue; + } + + if (IsHcom(cur_cnode_ptr)) { + auto hcom_graph_id = AnfAlgo::GetGraphId(cur_cnode_ptr.get()); + auto it = graph_nodes_map.find(hcom_graph_id); + if (it == graph_nodes_map.end()) { + graph_nodes_map[hcom_graph_id] = {cur_cnode_ptr}; + } else { + it->second.emplace_back(cur_cnode_ptr); + } + } + } + MS_LOG(INFO) << "hcom diff graph id size:" << graph_nodes_map.size(); + for (const auto &item : graph_nodes_map) { + bool new_graph = true; + auto graph_id = item.first; + hcom_graph_map_[graph_id] = {}; + for (const auto &hcom_node_ptr : item.second) { + auto assigned_stream_id = AssignHcomStreamId(hcom_node_ptr, new_graph); + hcom_graph_map_[graph_id].emplace(assigned_stream_id); + new_graph = false; + } + } + MS_LOG(INFO) << "hcom stream nums : " << hcom_stream_map_.size(); +} + uint32_t AscendStreamAssign::AssignHcomStreamId(const CNodePtr &cur_cnode_ptr, bool new_graph) { MS_EXCEPTION_IF_NULL(cur_cnode_ptr); AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); @@ -275,6 +252,39 @@ uint32_t AscendStreamAssign::AssignHcomStreamId(const CNodePtr &cur_cnode_ptr, b return cur_hcom_stream_id; } +void AscendStreamAssign::AssignIndependent(const NotNull &graph_ptr) { + auto cnode_ptr_list = graph_ptr->execution_order(); + std::map> graph_nodes_map; + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { + continue; + } + if (AnfAlgo::IsIndependentNode(cur_cnode_ptr)) { + auto independent_graph_id = AnfAlgo::GetGraphId(cur_cnode_ptr.get()); + auto it = graph_nodes_map.find(independent_graph_id); + if (it == graph_nodes_map.end()) { + graph_nodes_map[independent_graph_id] = {cur_cnode_ptr}; + } else { + it->second.emplace_back(cur_cnode_ptr); + } + } + } + + MS_LOG(INFO) << "independent diff graph id size:" << graph_nodes_map.size(); + for (const auto &item : graph_nodes_map) { + bool new_graph = true; + auto graph_id = item.first; + independent_graph_map_[graph_id] = {}; + for (const auto &independent_node_ptr : item.second) { + auto assigned_stream_id = AssignIndependentStreamId(independent_node_ptr, new_graph); + independent_graph_map_[graph_id].emplace(assigned_stream_id); + new_graph = false; + } + } + MS_LOG(INFO) << "stream nums:" << independent_stream_map_.size(); +} + uint32_t AscendStreamAssign::AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr, bool new_graph) { MS_EXCEPTION_IF_NULL(cur_cnode_ptr); AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); @@ -556,8 +566,8 @@ void AscendStreamAssign::InsertStreamActiveForIndependent(const NotNull update_cnode_list; @@ -568,28 +578,23 @@ void AscendStreamAssign::InsertStreamActiveForIndependent(const NotNullset_execution_order(update_cnode_list); - MS_LOG(INFO) << "End"; } void AscendStreamAssign::GetProcessedStream(const NotNull &graph_ptr) { @@ -637,28 +641,36 @@ void AscendStreamAssign::GetProcessedStream(const NotNull &graph } } -void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, - vector *orders) { +bool AscendStreamAssign::CheckStreamSwitch(const CNodePtr &switch_ptr) { if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, switch_ptr)) { - orders->emplace_back(switch_ptr); - return; + return false; } + auto need_active = AnfAlgo::GetNodeAttr(switch_ptr, kStreamNeedActivedFirst); if (!need_active) { - orders->emplace_back(switch_ptr); - return; + return false; } if (!AnfAlgo::HasNodeAttr(kAttrStreamSwitchKind, switch_ptr)) { - orders->emplace_back(switch_ptr); - return; + return false; } + auto kind = AnfAlgo::GetNodeAttr(switch_ptr, kAttrStreamSwitchKind); if (kind == kEosStreamSwitch || kind == kGetNextStreamSwitch) { + return false; + } + + return true; +} + +void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, + vector *orders) { + if (!CheckStreamSwitch(switch_ptr)) { orders->emplace_back(switch_ptr); return; } + auto kind = AnfAlgo::GetNodeAttr(switch_ptr, kAttrStreamSwitchKind); if (kind == kIndependentStreamSwitch) { bool independent_empty = independent_stream_map_.empty(); // if indepdent empty: delete independent streamswitch @@ -679,23 +691,18 @@ void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph for (const auto &item : independent_stream_map_) { processed_streams_.emplace(item.first); } - return; - } - - if (kind == kFpBpStreamSwitch) { - bool hcom_empty = hcom_stream_map_.empty(); - if (hcom_empty) { + } else if (kind == kFpBpStreamSwitch) { + if (hcom_stream_map_.empty()) { orders->emplace_back(switch_ptr); return; } if (!AnfAlgo::HasNodeAttr(kAttrTrueBranchStream, switch_ptr)) { + // FpBp StreamSwitch has no true branch attr orders->emplace_back(switch_ptr); - MS_LOG(WARNING) << "FpBp StreamSwitch has no true branch attr"; return; } auto true_stream_id = AnfAlgo::GetNodeAttr(switch_ptr, kAttrTrueBranchStream); - MS_LOG(INFO) << "Streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) - << "; active stream id:" << true_stream_id; + MS_LOG(INFO) << "Swtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) << "; active stream id:" << true_stream_id; CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); AnfAlgo::SetStreamId(true_stream_id, active_ptr.get()); vector active_ids; @@ -1633,8 +1640,7 @@ StreamActiveKind AscendStreamAssign::GetStreamActiveKind(const NotNull &graph_ptr); void AssignAllNodesStream(const NotNull &graph_ptr); void AssignCommonStreamId(const CNodePtr &cur_cnode_ptr); + void AssignHcom(const NotNull &graph_ptr); uint32_t AssignHcomStreamId(const CNodePtr &cur_cnode_ptr, bool new_graph); + void AssignIndependent(const NotNull &graph_ptr); uint32_t AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr, bool new_graph); void UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr); void FindHcomParallelStreams(const NotNull &graph_ptr); @@ -137,6 +139,7 @@ class AscendStreamAssign { std::map> other_graph); void UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, vector *orders); + bool CheckStreamSwitch(const CNodePtr &switch_ptr); void InsertEventForIndependentParallel(const NotNull &graph_ptr); void InsertCtrlForIndependentParallel(const NotNull &graph_ptr); void InsertEventForHcomParallel(const NotNull &graph_ptr);