From 3226e8407ac61ca4bff8b7bb5ca4406354d3dda4 Mon Sep 17 00:00:00 2001 From: Harshvardhan Gupta Date: Thu, 27 Aug 2020 17:41:50 -0400 Subject: [PATCH] fix gpu wp --- mindspore/ccsrc/debug/debug_services.cc | 61 +--------------------- mindspore/ccsrc/debug/debug_services.h | 6 +-- mindspore/ccsrc/debug/debugger/debugger.cc | 43 ++++----------- mindspore/ccsrc/debug/debugger/debugger.h | 3 +- 4 files changed, 16 insertions(+), 97 deletions(-) diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc index 4e23a1dd20..05abc6881c 100644 --- a/mindspore/ccsrc/debug/debug_services.cc +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -66,11 +66,9 @@ void DebugServices::RemoveWatchpoint(unsigned int id) { void DebugServices::CheckWatchpoints(std::vector *name, std::vector *slot, std::vector *condition, std::vector *watchpoint_id, - const std::vector &op_overflows) { + const std::vector &op_overflows, + const std::vector> &tensor_list) { std::lock_guard lg(lock_); - - std::vector> tensor_list = tensor_loader_->GetTensor(); - std::string current_tensor_name; std::unordered_map watchpoints_to_check_table; const size_t location = 0; @@ -198,61 +196,6 @@ void DebugServices::HandleWatchpointHits(const std::vector &hit_en } } -void DebugServices::CheckSingleWatchpoint(std::shared_ptr watchtensor, std::string *name, std::string *slot, - char **data_ptr, unsigned int *data_size, int *condition, - unsigned int *wacthpoint_id) { - std::lock_guard lg(lock_); - - std::string current_watchtensor_name; - current_watchtensor_name = watchtensor->GetName(); - mindspore::tensor::TensorPtr tensor_ptr = watchtensor->GetTensor(); - int tensor_data_type = tensor_ptr->data_type_c(); - watchpoint_t watchpoint_to_check; - - for (auto w_table_item : watchpoint_table) { - auto check_node_list = std::get<1>(w_table_item).check_node_list; - for (auto check_node : check_node_list) { - std::string w_name = std::get<0>(check_node); - bool w_type = std::get<1>(check_node); - // get current the full info including condition, id..., for current watchtensor - std::string current_node_name = current_watchtensor_name.substr(0, current_watchtensor_name.find_first_of(":")); - if ((w_type == true && (current_watchtensor_name.find(w_name) != string::npos || w_name == "*")) || - (w_type == false && current_node_name == w_name)) { - watchpoint_to_check = w_table_item.second; - // need to add support for float16 and float64, and other types when we support conditions beyond inf and nan - if (tensor_data_type != kNumberTypeFloat && tensor_data_type != kNumberTypeFloat32) { - return; - } - break; - } - } - } - - float *start_addr = reinterpret_cast(tensor_ptr->data_c()); - unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float); - - for (unsigned int index = 0; index < num_elements; index++) { - float x = start_addr[index]; - if (((watchpoint_to_check.conditions.inf.enabled || watchpoint_to_check.conditions.neg_inf.enabled) && isinf(x)) || - (watchpoint_to_check.conditions.nan.enabled && isnan(x))) { - std::string name_no_slot = current_watchtensor_name.substr(0, current_watchtensor_name.find_first_of(":")); - *name = name_no_slot; - *slot = std::to_string(watchtensor->GetSlot()); - *data_ptr = reinterpret_cast(tensor_ptr->data_c()); - *data_size = tensor_ptr->data().nbytes(); - int condition_item = -1; - if (watchpoint_to_check.conditions.nan.enabled) { - condition_item = 0; - } else if (watchpoint_to_check.conditions.inf.enabled || watchpoint_to_check.conditions.neg_inf.enabled) { - condition_item = 1; - } - *condition = condition_item; - - *wacthpoint_id = watchpoint_to_check.id; - } - } -} - void DebugServices::ReadNodesTensors(std::vector name, std::vector *ret_name, std::vector *data_ptr, std::vector *data_size, std::vector *dtype, std::vector> *shape) { diff --git a/mindspore/ccsrc/debug/debug_services.h b/mindspore/ccsrc/debug/debug_services.h index 69ba31463a..08840eef4a 100644 --- a/mindspore/ccsrc/debug/debug_services.h +++ b/mindspore/ccsrc/debug/debug_services.h @@ -76,10 +76,8 @@ class DebugServices { void RemoveWatchpoint(unsigned int id); void CheckWatchpoints(std::vector *name, std::vector *slot, std::vector *condition, - std::vector *watchpoint_id, const std::vector &op_overflows); - - void CheckSingleWatchpoint(std::shared_ptr watchnode, std::string *name, std::string *slot, - char **data_ptr, unsigned int *data_size, int *condition, unsigned int *wacthpoint_id); + std::vector *watchpoint_id, const std::vector &op_overflows, + const std::vector> &tensor_list); void ReadNodesTensors(std::vector name, std::vector *ret_name, std::vector *data_ptr, std::vector *data_size, diff --git a/mindspore/ccsrc/debug/debugger/debugger.cc b/mindspore/ccsrc/debug/debugger/debugger.cc index ee63f08a6b..d57537834f 100644 --- a/mindspore/ccsrc/debug/debugger/debugger.cc +++ b/mindspore/ccsrc/debug/debugger/debugger.cc @@ -254,7 +254,7 @@ void Debugger::PostExecuteNode() { // if kernel is watchpoint,and get hit. suspend. if (is_watchpoint) { - auto hits = CheckSingleWatchpoint(cur_name_); + auto hits = CheckWatchpoints(cur_name_); if (!hits.empty()) { SendWatchpointsAndSuspend(hits); } @@ -547,7 +547,7 @@ void Debugger::Exit() { std::exit(EXIT_FAILURE); } -std::list Debugger::CheckWatchpoints() { +std::list Debugger::CheckWatchpoints(const std::string &watchnode) { std::vector name; std::vector slot; std::vector condition; @@ -556,7 +556,15 @@ std::list Debugger::CheckWatchpoints() { #ifdef ENABLE_D overflow_ops = CheckOpOverflow(); #endif - debug_services_->CheckWatchpoints(&name, &slot, &condition, &watchpoint_id, overflow_ops); + auto tensor_loader = debug_services_->tensor_loader(); + std::vector> tensor_list; + if (watchnode.empty()) { + tensor_list = tensor_loader->GetTensor(); + } else { + tensor_list = tensor_loader->GetNodeTensorMap(watchnode); + } + + debug_services_->CheckWatchpoints(&name, &slot, &condition, &watchpoint_id, overflow_ops, tensor_list); std::list hits; for (unsigned int i = 0; i < name.size(); i++) { WatchpointHit hit; @@ -576,35 +584,6 @@ std::list Debugger::CheckWatchpoints() { return hits; } -std::list Debugger::CheckSingleWatchpoint(std::string watchnode) const { - auto tensor_loader = debug_services_->tensor_loader(); - auto tensors = tensor_loader->GetNodeTensorMap(watchnode); - std::list hits; - for (std::vector>::iterator it = tensors.begin(); it != tensors.end(); ++it) { - auto cur_tensor = *it; - std::string name = ""; - std::string slot = ""; - char *data_ptr = nullptr; - unsigned int data_size = 0; - int condition = -1; - unsigned int watchpoint_id = -1; - WatchpointHit hit; - debug_services_->CheckSingleWatchpoint(cur_tensor, &name, &slot, &data_ptr, &data_size, &condition, &watchpoint_id); - if (name != "") { - hit.set_id(watchpoint_id); - // here TensorProto act as a tensor indicator, not sending tensor content - TensorProto *tensor_item = hit.mutable_tensor(); - tensor_item->set_node_name(name); - tensor_item->set_slot(slot); - tensor_item->set_finished(true); - WatchCondition *condition_item = hit.mutable_watch_condition(); - condition_item->set_condition(debugger::WatchCondition_Condition(condition)); - hits.push_back(hit); - } - } - return hits; -} - void Debugger::SendWatchpointsAndSuspend(const std::list &points) { // send info about watchpoint if (!points.empty()) { diff --git a/mindspore/ccsrc/debug/debugger/debugger.h b/mindspore/ccsrc/debug/debugger/debugger.h index 0905985184..43ca582e96 100644 --- a/mindspore/ccsrc/debug/debugger/debugger.h +++ b/mindspore/ccsrc/debug/debugger/debugger.h @@ -137,8 +137,7 @@ class Debugger : public std::enable_shared_from_this { // analyze tensors and check watchpoint conditions // return names of tensors and what condition they hit - std::list CheckWatchpoints(); - std::list CheckSingleWatchpoint(std::string watchnode) const; + std::list CheckWatchpoints(const std::string &watchnode = std::string()); // send watchpoints that hit and enter command wait loop void SendWatchpointsAndSuspend(const std::list &points);