6

mindspore/lite/src/lite_session.cc

@@ -290,15 +290,6 @@ void LiteSession::InitGraphOutputNodeMap(const lite::Model *model) {
   }
 }
 
-void LiteSession::InitGraphOutputTensorNames(const lite::Model *model) {
-  MS_ASSERT(model != nullptr);
-  MS_ASSERT(this->output_tensor_names_.empty());
-  auto out_size = model->sub_graphs_.front()->output_indices_.size();
-  for (size_t i = 0; i < out_size; ++i) {
-    this->output_tensor_names_.emplace_back(std::to_string(model->sub_graphs_.front()->output_indices_[i]));
-  }
-}
-
 void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) {
   MS_ASSERT(model != nullptr);
   MS_ASSERT(this->output_tensor_map_.empty());
@@ -311,9 +302,12 @@ void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) {
       MS_LOG(ERROR) << "out_tensor is null!";
       return;
     }
-    this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor));
     if (!out_tensor->tensor_name().empty()) {
       this->output_tensor_map_.insert(std::make_pair(out_tensor->tensor_name(), out_tensor));
+      this->output_tensor_names_.emplace_back(out_tensor->tensor_name());
+    } else {
+      this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor));
+      this->output_tensor_names_.emplace_back(std::to_string(graph_out_index));
     }
   }
 }
@@ -339,7 +333,6 @@ void LiteSession::InitGraphInOutTensors(const lite::Model *model) {
   InitGraphOutputTensors(model);
   InitGraphInputMap(model);
   InitGraphOutputNodeMap(model);
-  InitGraphOutputTensorNames(model);
   InitGraphOutputTensorMap(model);
   for (auto *tensor : this->inputs_) {
     tensor->set_category(Tensor::Category::GRAPH_INPUT);

mindspore/lite/src/lite_session.h

@@ -88,8 +88,6 @@ class LiteSession : public session::LiteSession {
 
   void InitGraphOutputNodeMap(const lite::Model *model);
 
-  void InitGraphOutputTensorNames(const lite::Model *model);
-
   void InitGraphOutputTensorMap(const lite::Model *model);
 
   void AdjustModelOutputTensorInitRefCount(const lite::Model *model);

mindspore/lite/tools/benchmark/benchmark.cc

@@ -159,8 +159,7 @@ int Benchmark::ReadInputFile() {
 }
 
 // calibData is FP32
-int Benchmark::ReadCalibData() {
-  const char *calib_data_path = flags_->benchmark_data_file_.c_str();
+int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) {
   // read calib data
   std::ifstream in_file(calib_data_path);
   if (!in_file.good()) {
@@ -232,7 +231,11 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string &
     MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name;
     return RET_ERROR;
   }
-  this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
+  if (has_new_data_) {
+    this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
+  } else {
+    this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
+  }
   return RET_OK;
 }
 
@@ -240,26 +243,51 @@ int Benchmark::CompareOutput() {
   std::cout << "================ Comparing Output data ================" << std::endl;
   float total_bias = 0;
   int total_size = 0;
-  for (const auto &calib_tensor : benchmark_data_) {
-    std::string node_or_tensor_name = calib_tensor.first;
-    tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
-    if (tensor == nullptr) {
-      MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
-      return RET_ERROR;
-    }
-    int ret;
-    if (tensor->data_type() == kObjectTypeString) {
-      ret = CompareStringData(node_or_tensor_name, tensor);
-    } else {
-      ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size);
+  if (new_benchmark_data_.size() > benchmark_data_.size()) {
+    for (const auto &calib_tensor : new_benchmark_data_) {
+      std::string node_or_tensor_name = calib_tensor.first;
+      tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
+      if (tensor == nullptr) {
+        MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
+        return RET_ERROR;
+      }
+      int ret;
+      if (tensor->data_type() == kObjectTypeString) {
+        ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_);
+      } else {
+        ret =
+          CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_);
+      }
+      if (ret != RET_OK) {
+        MS_LOG(ERROR) << "Error in CompareData";
+        std::cerr << "Error in CompareData" << std::endl;
+        std::cout << "=======================================================" << std::endl << std::endl;
+        return ret;
+      }
     }
-    if (ret != RET_OK) {
-      MS_LOG(ERROR) << "Error in CompareData";
-      std::cerr << "Error in CompareData" << std::endl;
-      std::cout << "=======================================================" << std::endl << std::endl;
-      return ret;
+  } else {
+    for (const auto &calib_tensor : benchmark_data_) {
+      std::string node_or_tensor_name = calib_tensor.first;
+      tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
+      if (tensor == nullptr) {
+        MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
+        return RET_ERROR;
+      }
+      int ret;
+      if (tensor->data_type() == kObjectTypeString) {
+        ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_);
+      } else {
+        ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_);
+      }
+      if (ret != RET_OK) {
+        MS_LOG(ERROR) << "Error in CompareData";
+        std::cerr << "Error in CompareData" << std::endl;
+        std::cout << "=======================================================" << std::endl << std::endl;
+        return ret;
+      }
     }
   }
+
   float mean_bias;
   if (total_size != 0) {
     mean_bias = total_bias / float_t(total_size) * 100;
@@ -291,7 +319,8 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node
   return tensor;
 }
 
-int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) {
+int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor,
+                                 std::unordered_map<std::string, CheckTensor *> benchmark_data) {
   auto iter = this->benchmark_data_.find(name);
   if (iter != this->benchmark_data_.end()) {
     std::vector<std::string> calib_strings = iter->second->strings_data;
@@ -314,7 +343,8 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens
 }
 
 int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
-                                              int *total_size) {
+                                              int *total_size,
+                                              std::unordered_map<std::string, CheckTensor *> benchmark_data) {
   float bias = 0;
   auto mutableData = tensor->MutableData();
   if (mutableData == nullptr) {
@@ -323,19 +353,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M
   }
   switch (msCalibDataType) {
     case TypeId::kNumberTypeFloat: {
-      bias = CompareData<float>(name, tensor->shape(), mutableData);
+      bias = CompareData<float>(name, tensor->shape(), mutableData, benchmark_data);
       break;
     }
     case TypeId::kNumberTypeInt8: {
-      bias = CompareData<int8_t>(name, tensor->shape(), mutableData);
+      bias = CompareData<int8_t>(name, tensor->shape(), mutableData, benchmark_data);
       break;
     }
     case TypeId::kNumberTypeUInt8: {
-      bias = CompareData<uint8_t>(name, tensor->shape(), mutableData);
+      bias = CompareData<uint8_t>(name, tensor->shape(), mutableData, benchmark_data);
       break;
     }
     case TypeId::kNumberTypeInt32: {
-      bias = CompareData<int32_t>(name, tensor->shape(), mutableData);
+      bias = CompareData<int32_t>(name, tensor->shape(), mutableData, benchmark_data);
       break;
     }
     default:
@@ -423,12 +453,20 @@ int Benchmark::MarkAccuracy() {
     std::cerr << "Inference error " << status << std::endl;
     return status;
   }
-  status = ReadCalibData();
+  const char *calib_data_path = flags_->benchmark_data_file_.c_str();
+  status = ReadCalibData(false, calib_data_path);
   if (status != RET_OK) {
     MS_LOG(ERROR) << "Read calib data error " << status;
     std::cerr << "Read calib data error " << status << std::endl;
-    return status;
+    has_new_data_ = true;
+    status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str());
+    if (status != RET_OK) {
+      MS_LOG(ERROR) << "no new data.";
+      has_new_data_ = false;
+      return status;
+    }
   }
+
   status = CompareOutput();
   if (status != RET_OK) {
     MS_LOG(ERROR) << "Compare output error " << status;

mindspore/lite/tools/benchmark/benchmark.h

@@ -129,7 +129,7 @@ class MS_API Benchmark {
 
   int ReadInputFile();
 
-  int ReadCalibData();
+  int ReadCalibData(bool new_data, const char *calib_data_path);
 
   int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector<size_t> &dims);
 
@@ -137,10 +137,11 @@ class MS_API Benchmark {
 
   tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name);
 
-  int CompareStringData(const std::string &name, tensor::MSTensor *tensor);
+  int CompareStringData(const std::string &name, tensor::MSTensor *tensor,
+                        std::unordered_map<std::string, CheckTensor *> benchmark_data);
 
   int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
-                                     int *total_size);
+                                     int *total_size, std::unordered_map<std::string, CheckTensor *> benchmark_data);
 
   int InitCallbackParameter();
 
@@ -150,10 +151,11 @@ class MS_API Benchmark {
 
   // tensorData need to be converter first
   template <typename T>
-  float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data) {
+  float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data,
+                    std::unordered_map<std::string, CheckTensor *> benchmark_data) {
     const T *msTensorData = static_cast<const T *>(tensor_data);
-    auto iter = this->benchmark_data_.find(nodeName);
-    if (iter != this->benchmark_data_.end()) {
+    auto iter = benchmark_data.find(nodeName);
+    if (iter != benchmark_data.end()) {
       std::vector<size_t> castedMSShape;
       size_t shapeSize = 1;
       for (int64_t dim : msShape) {
@@ -238,11 +240,13 @@ class MS_API Benchmark {
   int MarkAccuracy();
 
  private:
+  bool has_new_data_ = false;
   BenchmarkFlags *flags_;
   session::LiteSession *session_{nullptr};
   std::vector<mindspore::tensor::MSTensor *> ms_inputs_;
   std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_;
   std::unordered_map<std::string, CheckTensor *> benchmark_data_;
+  std::unordered_map<std::string, CheckTensor *> new_benchmark_data_;
   std::unordered_map<std::string, TypeId> data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat},
                                                          {"INT8", TypeId::kNumberTypeInt8},
                                                          {"INT32", TypeId::kNumberTypeInt32},