diff --git a/mindspore/lite/src/lite_session.cc b/mindspore/lite/src/lite_session.cc index 6ddebb5ed2..4c2386cded 100644 --- a/mindspore/lite/src/lite_session.cc +++ b/mindspore/lite/src/lite_session.cc @@ -290,15 +290,6 @@ void LiteSession::InitGraphOutputNodeMap(const lite::Model *model) { } } -void LiteSession::InitGraphOutputTensorNames(const lite::Model *model) { - MS_ASSERT(model != nullptr); - MS_ASSERT(this->output_tensor_names_.empty()); - auto out_size = model->sub_graphs_.front()->output_indices_.size(); - for (size_t i = 0; i < out_size; ++i) { - this->output_tensor_names_.emplace_back(std::to_string(model->sub_graphs_.front()->output_indices_[i])); - } -} - void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) { MS_ASSERT(model != nullptr); MS_ASSERT(this->output_tensor_map_.empty()); @@ -311,9 +302,12 @@ void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) { MS_LOG(ERROR) << "out_tensor is null!"; return; } - this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor)); if (!out_tensor->tensor_name().empty()) { this->output_tensor_map_.insert(std::make_pair(out_tensor->tensor_name(), out_tensor)); + this->output_tensor_names_.emplace_back(out_tensor->tensor_name()); + } else { + this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor)); + this->output_tensor_names_.emplace_back(std::to_string(graph_out_index)); } } } @@ -339,7 +333,6 @@ void LiteSession::InitGraphInOutTensors(const lite::Model *model) { InitGraphOutputTensors(model); InitGraphInputMap(model); InitGraphOutputNodeMap(model); - InitGraphOutputTensorNames(model); InitGraphOutputTensorMap(model); for (auto *tensor : this->inputs_) { tensor->set_category(Tensor::Category::GRAPH_INPUT); diff --git a/mindspore/lite/src/lite_session.h b/mindspore/lite/src/lite_session.h index a24e5ff297..6819abf175 100644 --- a/mindspore/lite/src/lite_session.h +++ b/mindspore/lite/src/lite_session.h @@ -88,8 +88,6 @@ class LiteSession : public session::LiteSession { void InitGraphOutputNodeMap(const lite::Model *model); - void InitGraphOutputTensorNames(const lite::Model *model); - void InitGraphOutputTensorMap(const lite::Model *model); void AdjustModelOutputTensorInitRefCount(const lite::Model *model); diff --git a/mindspore/lite/tools/benchmark/benchmark.cc b/mindspore/lite/tools/benchmark/benchmark.cc index 018760ea18..fd184ef761 100644 --- a/mindspore/lite/tools/benchmark/benchmark.cc +++ b/mindspore/lite/tools/benchmark/benchmark.cc @@ -159,8 +159,7 @@ int Benchmark::ReadInputFile() { } // calibData is FP32 -int Benchmark::ReadCalibData() { - const char *calib_data_path = flags_->benchmark_data_file_.c_str(); +int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) { // read calib data std::ifstream in_file(calib_data_path); if (!in_file.good()) { @@ -232,7 +231,11 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string & MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; return RET_ERROR; } - this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); + if (has_new_data_) { + this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); + } else { + this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); + } return RET_OK; } @@ -240,26 +243,51 @@ int Benchmark::CompareOutput() { std::cout << "================ Comparing Output data ================" << std::endl; float total_bias = 0; int total_size = 0; - for (const auto &calib_tensor : benchmark_data_) { - std::string node_or_tensor_name = calib_tensor.first; - tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; - return RET_ERROR; - } - int ret; - if (tensor->data_type() == kObjectTypeString) { - ret = CompareStringData(node_or_tensor_name, tensor); - } else { - ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size); + if (new_benchmark_data_.size() > benchmark_data_.size()) { + for (const auto &calib_tensor : new_benchmark_data_) { + std::string node_or_tensor_name = calib_tensor.first; + tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; + return RET_ERROR; + } + int ret; + if (tensor->data_type() == kObjectTypeString) { + ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_); + } else { + ret = + CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_); + } + if (ret != RET_OK) { + MS_LOG(ERROR) << "Error in CompareData"; + std::cerr << "Error in CompareData" << std::endl; + std::cout << "=======================================================" << std::endl << std::endl; + return ret; + } } - if (ret != RET_OK) { - MS_LOG(ERROR) << "Error in CompareData"; - std::cerr << "Error in CompareData" << std::endl; - std::cout << "=======================================================" << std::endl << std::endl; - return ret; + } else { + for (const auto &calib_tensor : benchmark_data_) { + std::string node_or_tensor_name = calib_tensor.first; + tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name; + return RET_ERROR; + } + int ret; + if (tensor->data_type() == kObjectTypeString) { + ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_); + } else { + ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_); + } + if (ret != RET_OK) { + MS_LOG(ERROR) << "Error in CompareData"; + std::cerr << "Error in CompareData" << std::endl; + std::cout << "=======================================================" << std::endl << std::endl; + return ret; + } } } + float mean_bias; if (total_size != 0) { mean_bias = total_bias / float_t(total_size) * 100; @@ -291,7 +319,8 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node return tensor; } -int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) { +int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor, + std::unordered_map benchmark_data) { auto iter = this->benchmark_data_.find(name); if (iter != this->benchmark_data_.end()) { std::vector calib_strings = iter->second->strings_data; @@ -314,7 +343,8 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens } int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, - int *total_size) { + int *total_size, + std::unordered_map benchmark_data) { float bias = 0; auto mutableData = tensor->MutableData(); if (mutableData == nullptr) { @@ -323,19 +353,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M } switch (msCalibDataType) { case TypeId::kNumberTypeFloat: { - bias = CompareData(name, tensor->shape(), mutableData); + bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); break; } case TypeId::kNumberTypeInt8: { - bias = CompareData(name, tensor->shape(), mutableData); + bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); break; } case TypeId::kNumberTypeUInt8: { - bias = CompareData(name, tensor->shape(), mutableData); + bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); break; } case TypeId::kNumberTypeInt32: { - bias = CompareData(name, tensor->shape(), mutableData); + bias = CompareData(name, tensor->shape(), mutableData, benchmark_data); break; } default: @@ -423,12 +453,20 @@ int Benchmark::MarkAccuracy() { std::cerr << "Inference error " << status << std::endl; return status; } - status = ReadCalibData(); + const char *calib_data_path = flags_->benchmark_data_file_.c_str(); + status = ReadCalibData(false, calib_data_path); if (status != RET_OK) { MS_LOG(ERROR) << "Read calib data error " << status; std::cerr << "Read calib data error " << status << std::endl; - return status; + has_new_data_ = true; + status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str()); + if (status != RET_OK) { + MS_LOG(ERROR) << "no new data."; + has_new_data_ = false; + return status; + } } + status = CompareOutput(); if (status != RET_OK) { MS_LOG(ERROR) << "Compare output error " << status; diff --git a/mindspore/lite/tools/benchmark/benchmark.h b/mindspore/lite/tools/benchmark/benchmark.h index cf8d97a954..df38d690a2 100644 --- a/mindspore/lite/tools/benchmark/benchmark.h +++ b/mindspore/lite/tools/benchmark/benchmark.h @@ -129,7 +129,7 @@ class MS_API Benchmark { int ReadInputFile(); - int ReadCalibData(); + int ReadCalibData(bool new_data, const char *calib_data_path); int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector &dims); @@ -137,10 +137,11 @@ class MS_API Benchmark { tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name); - int CompareStringData(const std::string &name, tensor::MSTensor *tensor); + int CompareStringData(const std::string &name, tensor::MSTensor *tensor, + std::unordered_map benchmark_data); int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias, - int *total_size); + int *total_size, std::unordered_map benchmark_data); int InitCallbackParameter(); @@ -150,10 +151,11 @@ class MS_API Benchmark { // tensorData need to be converter first template - float CompareData(const std::string &nodeName, const std::vector &msShape, const void *tensor_data) { + float CompareData(const std::string &nodeName, const std::vector &msShape, const void *tensor_data, + std::unordered_map benchmark_data) { const T *msTensorData = static_cast(tensor_data); - auto iter = this->benchmark_data_.find(nodeName); - if (iter != this->benchmark_data_.end()) { + auto iter = benchmark_data.find(nodeName); + if (iter != benchmark_data.end()) { std::vector castedMSShape; size_t shapeSize = 1; for (int64_t dim : msShape) { @@ -238,11 +240,13 @@ class MS_API Benchmark { int MarkAccuracy(); private: + bool has_new_data_ = false; BenchmarkFlags *flags_; session::LiteSession *session_{nullptr}; std::vector ms_inputs_; std::unordered_map> ms_outputs_; std::unordered_map benchmark_data_; + std::unordered_map new_benchmark_data_; std::unordered_map data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat}, {"INT8", TypeId::kNumberTypeInt8}, {"INT32", TypeId::kNumberTypeInt32},