From 7d2e9e8dbd1e2426651e304e5bc593b302ce763e Mon Sep 17 00:00:00 2001 From: yangjie159 Date: Tue, 13 Apr 2021 14:04:21 +0800 Subject: [PATCH] add benchmark calibration for micro --- mindspore/lite/micro/cmake/file_list.cmake | 1 + .../component/const_blocks/benchmark.cc | 32 ++- .../component/const_blocks/calib_output.cc | 246 ++++++++++++++++++ .../component/const_blocks/calib_output.h | 27 ++ .../component/const_blocks/cmake_lists.cc | 1 + .../component/const_blocks/mstring.cc | 1 + .../lite/micro/coder/generator/generator.cc | 5 +- 7 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.cc create mode 100644 mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.h diff --git a/mindspore/lite/micro/cmake/file_list.cmake b/mindspore/lite/micro/cmake/file_list.cmake index 8777af0662..82ac783bc7 100644 --- a/mindspore/lite/micro/cmake/file_list.cmake +++ b/mindspore/lite/micro/cmake/file_list.cmake @@ -32,6 +32,7 @@ set(CODER_GENERATOR_SRC ${MICRO_DIR}/coder/generator/component/const_blocks/model.cc ${MICRO_DIR}/coder/generator/component/const_blocks/license.cc ${MICRO_DIR}/coder/generator/component/const_blocks/load_input.cc + ${MICRO_DIR}/coder/generator/component/const_blocks/calib_output.cc ${MICRO_DIR}/coder/generator/component/const_blocks/thread_pool.cc ${MICRO_DIR}/coder/generator/component/const_blocks/benchmark.cc ) diff --git a/mindspore/lite/micro/coder/generator/component/const_blocks/benchmark.cc b/mindspore/lite/micro/coder/generator/component/const_blocks/benchmark.cc index c036449f54..fa2776136f 100644 --- a/mindspore/lite/micro/coder/generator/component/const_blocks/benchmark.cc +++ b/mindspore/lite/micro/coder/generator/component/const_blocks/benchmark.cc @@ -44,6 +44,7 @@ const char *benchmark_source = R"RAW( #include "include/errorcode.h" #include "load_input.h" +#include "calib_output.h" using namespace mindspore; @@ -54,8 +55,9 @@ void usage() { "args[1]: inputs binary file\n" "args[2]: model weight binary file\n" "args[3]: loop count for performance test\n" - "args[4]: runtime thread num\n" - "args[5]: runtime thread bind mode\n\n"); + "args[4]: calibration file\n" + "args[5]: runtime thread num\n" + "args[6]: runtime thread bind mode\n\n"); } uint64_t GetTimeUs() { @@ -131,15 +133,15 @@ int main(int argc, const char **argv) { } lite::Context *context = nullptr; - if (argc >= 6) { + if (argc >= 7) { // config benchmark context context = new (std::nothrow) lite::Context(); if (context == nullptr) { return lite::RET_ERROR; } - context->thread_num_ = atoi(argv[4]); + context->thread_num_ = atoi(argv[5]); context->device_list_.resize(1); - context->device_list_[0] = {lite::DT_CPU, {{false, static_cast(atoi(argv[5]))}}}; + context->device_list_[0] = {lite::DT_CPU, {{false, static_cast(atoi(argv[6]))}}}; printf("context: ThreadNum: %d, BindMode: %d\n", context->thread_num_, context->device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_); } @@ -187,12 +189,29 @@ int main(int argc, const char **argv) { return lite::RET_ERROR; } - Vector outputs_name = session->GetOutputTensorNames(); printf("\noutputs: \n"); + Vector outputs_name = session->GetOutputTensorNames(); + Vector outputs; for (const auto &name : outputs_name) { auto output = session->GetOutputByTensorName(name); + outputs.push_back(output); TensorToString(output); } + if (argc >= 5) { + lite::Calibrator *calibrator = new (std::nothrow) lite::Calibrator(); + if (calibrator == nullptr) { + return lite::RET_NULL_PTR; + } + ret = calibrator->ReadCalibData(argv[4]); + if (ret != lite::RET_OK) { + return lite::RET_ERROR; + } + ret = calibrator->CompareOutputs(outputs); + if (ret != lite::RET_OK) { + return lite::RET_ERROR; + } + delete calibrator; + } printf("========run success=======\n"); delete session; session = nullptr; @@ -207,5 +226,4 @@ int main(int argc, const char **argv) { return lite::RET_OK; } )RAW"; - } // namespace mindspore::lite::micro diff --git a/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.cc b/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.cc new file mode 100644 index 0000000000..5955336325 --- /dev/null +++ b/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.cc @@ -0,0 +1,246 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "coder/generator/component/const_blocks/calib_output.h" + +namespace mindspore::lite::micro { + +const char *calib_header = R"RAW( +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_MICRO_CALIB_OUTPUT_H_ +#define MINDSPORE_LITE_MICRO_CALIB_OUTPUT_H_ + +#include "include/lite_utils.h" +#include "include/ms_tensor.h" +#include "include/errorcode.h" + +namespace mindspore { +namespace lite { + +class CalibTensor { + public: + CalibTensor(String name, size_t elements_num) : tensor_name_(name), elements_num_(elements_num) {} + ~CalibTensor() { + free(data_); + data_ = nullptr; + } + String tensor_name() const { return tensor_name_; } + int ElementsNum() const { return elements_num_; } + float *MutableData() { + if (data_ == nullptr) { + if (elements_num_ == 0 || elements_num_ > INT16_MAX) { + return nullptr; + } + data_ = static_cast(malloc(elements_num_ * sizeof(float))); + } + return data_; + } + + private: + String tensor_name_; + int elements_num_{0}; + float *data_{nullptr}; +}; + +class Calibrator { + public: + Calibrator() = default; + ~Calibrator() { + for (auto &calib : calib_outputs_) { + delete calib; + calib = nullptr; + } + calib_outputs_.clear(); + } + int ReadCalibData(const char *calib_data_path); + int CompareOutputs(const Vector &outputs) const; + + private: + Vector calib_outputs_; +}; + +} // namespace lite +} // namespace mindspore + +#endif // MINDSPORE_LITE_MICRO_CALIB_OUTPUT_H_ +)RAW"; + +const char *calib_source = R"RAW( +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "calib_output.h" +#include +#include +#include +#include +#include + +namespace mindspore { +namespace lite { +constexpr float kToleranceVal = 0.0001; + +#define MS_ERROR_IF_NULL(ptr) \ + do { \ + if ((ptr) == nullptr) { \ + return mindspore::lite::RET_ERROR; \ + } \ + } while (0) + +int Calibrator::ReadCalibData(const char *calib_data_path) { + std::ifstream in_file(calib_data_path); + if (!in_file.good()) { + printf("file is not exist, %s\n", calib_data_path); + return RET_ERROR; + } + if (!in_file.is_open()) { + printf("open file failed, %s\n", calib_data_path); + in_file.close(); + return RET_ERROR; + } + while (!in_file.eof()) { + std::string line; + getline(in_file, line); + if (line.empty()) { + continue; + } + std::stringstream name_line(line); + std::string tensor_name; + size_t dim = 0; + name_line >> tensor_name >> dim; + size_t elements = 1; + for (size_t i = 0; i < dim; i++) { + size_t tmp_dim; + name_line >> tmp_dim; + elements *= tmp_dim; + } + getline(in_file, line); + std::stringstream data_line(line); + String name(tensor_name.c_str()); + CalibTensor *output = new (std::nothrow) CalibTensor(name, elements); + MS_ERROR_IF_NULL(output); + float *data = output->MutableData(); + MS_ERROR_IF_NULL(data); + for (size_t i = 0; i < elements; i++) { + data_line >> data[i]; + } + calib_outputs_.push_back(output); + } + in_file.close(); + return RET_OK; +} + +template +float CompareData(const T *output, const float *calib, size_t elements_num) { + float error = 0.; + if (output == nullptr || calib == nullptr) { + printf("output or calib is nullptr\n"); + return error; + } + for (size_t i = 0; i < elements_num; ++i) { + if (std::isnan(output[i]) || std::isinf(output[i]) || std::isnan(calib[i]) || std::isinf(calib[i])) { + printf("error, output data is nan or inf\n"); + return error; + } + error += std::abs(output[i] - calib[i]); + } + return error; +} + +int Calibrator::CompareOutputs(const Vector &outputs) const { + if (outputs.size() != calib_outputs_.size()) { + printf("error, outputs and calibs size is mismatch\n"); + return RET_ERROR; + } + float total_error = 0; + size_t outputs_num = outputs.size(); + for (size_t i = 0; i < outputs_num; ++i) { + tensor::MSTensor *output = outputs[i]; + MS_ERROR_IF_NULL(output); + CalibTensor *calib = calib_outputs_[i]; + MS_ERROR_IF_NULL(calib); + if (output->tensor_name() != calib->tensor_name()) { + printf("error, output tensor name is not equal to calib\n"); + return RET_ERROR; + } + if (output->ElementsNum() != calib->ElementsNum()) { + printf("error, output elements num is not equal to calib\n"); + return RET_ERROR; + } + switch (output->data_type()) { + case TypeId::kNumberTypeFloat: + case TypeId::kNumberTypeFloat32: { + total_error += CompareData(static_cast(output->data()), calib->MutableData(), output->ElementsNum()); + break; + } + case TypeId::kNumberTypeInt8: { + total_error += CompareData(static_cast(output->data()), calib->MutableData(), output->ElementsNum()); + break; + } + case TypeId::kNumberTypeUInt8: { + total_error += CompareData(static_cast(output->data()), calib->MutableData(), output->ElementsNum()); + break; + } + case TypeId::kNumberTypeUInt: + case TypeId::kNumberTypeUInt32: { + total_error += CompareData(static_cast(output->data()), calib->MutableData(), output->ElementsNum()); + break; + } + default: { + printf("unsupported tensor data type\n"); + } + } + } + if (total_error > kToleranceVal) { + printf("compare outputs failed, total error: %f\n", total_error); + return RET_ERROR; + } + printf("compare outputs success, total error: %f\n", total_error); + return RET_OK; +} +} // namespace lite +} // namespace mindspore +)RAW"; + +} // namespace mindspore::lite::micro diff --git a/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.h b/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.h new file mode 100644 index 0000000000..b6e3cc4f46 --- /dev/null +++ b/mindspore/lite/micro/coder/generator/component/const_blocks/calib_output.h @@ -0,0 +1,27 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_MICRO_GENERATOR_CONST_BLOCK_CALIB_OUTPUT_H_ +#define MINDSPORE_LITE_MICRO_GENERATOR_CONST_BLOCK_CALIB_OUTPUT_H_ + +namespace mindspore::lite::micro { + +extern const char *calib_header; +extern const char *calib_source; + +} // namespace mindspore::lite::micro + +#endif // MINDSPORE_LITE_MICRO_GENERATOR_CONST_BLOCK_CALIB_OUTPUT_H_ diff --git a/mindspore/lite/micro/coder/generator/component/const_blocks/cmake_lists.cc b/mindspore/lite/micro/coder/generator/component/const_blocks/cmake_lists.cc index 7a44fe7456..3de5e3c4b8 100644 --- a/mindspore/lite/micro/coder/generator/component/const_blocks/cmake_lists.cc +++ b/mindspore/lite/micro/coder/generator/component/const_blocks/cmake_lists.cc @@ -72,6 +72,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${HEADER_PATH}) set(SRC_FILES benchmark/benchmark.cc + benchmark/calib_output.cc benchmark/load_input.c ) add_executable(benchmark ${SRC_FILES}) diff --git a/mindspore/lite/micro/coder/generator/component/const_blocks/mstring.cc b/mindspore/lite/micro/coder/generator/component/const_blocks/mstring.cc index 9e2c459cfc..87fd824a0c 100644 --- a/mindspore/lite/micro/coder/generator/component/const_blocks/mstring.cc +++ b/mindspore/lite/micro/coder/generator/component/const_blocks/mstring.cc @@ -307,6 +307,7 @@ String operator+(const char *lhs, const String &rhs) { return str; } +bool operator!=(const String &lhs, const String &rhs) { return lhs.compare(rhs) != 0; } bool operator==(const String &lhs, const String &rhs) { return lhs.compare(rhs) == 0; } bool operator==(const String &lhs, const char *rhs) { return lhs.compare(rhs) == 0; } bool operator==(const char *lhs, const String &rhs) { return rhs.compare(lhs) == 0; } diff --git a/mindspore/lite/micro/coder/generator/generator.cc b/mindspore/lite/micro/coder/generator/generator.cc index f8b4c16cb4..ed7ae28ec3 100644 --- a/mindspore/lite/micro/coder/generator/generator.cc +++ b/mindspore/lite/micro/coder/generator/generator.cc @@ -24,6 +24,7 @@ #include "coder/generator/component/const_blocks/cmake_lists.h" #include "coder/generator/component/const_blocks/debug_utils.h" #include "coder/generator/component/const_blocks/load_input.h" +#include "coder/generator/component/const_blocks/calib_output.h" #include "coder/generator/component/const_blocks/msession.h" #include "coder/generator/component/const_blocks/mtensor.h" #include "coder/generator/component/const_blocks/mstring.h" @@ -85,9 +86,11 @@ int Generator::CodeSourceCMakeFile() { int Generator::CodeStaticContent() { std::vector> const_blocks = { + {config_->code_path() + "/" + "CMakeLists.txt", bench_cmake_lists_txt}, + {net_main_file_path_ + "calib_output.h", calib_header}, + {net_main_file_path_ + "calib_output.cc", calib_source}, {net_main_file_path_ + "load_input.h", load_input_h}, {net_main_file_path_ + "load_input.c", load_input_c}, - {config_->code_path() + "/" + "CMakeLists.txt", bench_cmake_lists_txt}, {net_main_file_path_ + "benchmark.cc", benchmark_source}, {net_src_file_path_ + "CMakeLists.txt", src_cmake_lists_txt}, {net_src_file_path_ + "session.h", session_header},