diff --git a/build.sh b/build.sh index f9e8d8ad30..efed9fec89 100755 --- a/build.sh +++ b/build.sh @@ -116,7 +116,7 @@ checkopts() DEBUG_MODE="on" ;; n) - if [[ "X$OPTARG" == "Xoff" || "X$OPTARG" == "Xlite" || "X$OPTARG" == "Xfull" ]]; then + if [[ "X$OPTARG" == "Xoff" || "X$OPTARG" == "Xlite" || "X$OPTARG" == "Xfull" || "X$OPTARG" == "Xlite_cv" ]]; then COMPILE_MINDDATA_LITE="$OPTARG" else echo "Invalid value ${OPTARG} for option -n" diff --git a/cmake/package_lite.cmake b/cmake/package_lite.cmake index a8753f2204..600b039045 100644 --- a/cmake/package_lite.cmake +++ b/cmake/package_lite.cmake @@ -40,6 +40,18 @@ if (BUILD_MINDDATA STREQUAL "lite") endif () endif () + +if (BUILD_MINDDATA STREQUAL "lite_cv") + install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv DESTINATION ${INC_DIR} COMPONENT ${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h") + if (PLATFORM_ARM64) + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) + elseif (PLATFORM_ARM32) + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) + else () + install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${LIB_DIR_RUN_X86} COMPONENT ${RUN_X86_COMPONENT_NAME}) + endif () +endif () + if (PLATFORM_ARM64) install(FILES ${TOP_DIR}/mindspore/lite/build/src/libmindspore-lite.so DESTINATION ${LIB_DIR} COMPONENT ${COMPONENT_NAME}) install(FILES ${TOP_DIR}/mindspore/core/ir/dtype/type_id.h DESTINATION ${INC_DIR}/ir/dtype COMPONENT ${COMPONENT_NAME}) diff --git a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt index 759ca974b5..becdaeeccc 100644 --- a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt @@ -43,6 +43,8 @@ include_directories(${CMAKE_BINARY_DIR}) # for protobuf generated .h include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/mindrecord/include) include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include) +include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image) + ###################################################################### ####################### Flags ######################################## @@ -94,6 +96,7 @@ set(submodules $ $ $ + $ $ $ $ diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/image/CMakeLists.txt index 31027617c9..763bb71327 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/CMakeLists.txt @@ -1,6 +1,7 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_subdirectory(soft_dvpp) +add_subdirectory(lite_cv) add_library(kernels-image OBJECT affine_op.cc auto_contrast_op.cc @@ -49,4 +50,4 @@ add_library(kernels-image OBJECT random_resize_with_bbox_op.cc random_color_op.cc ) -add_dependencies(kernels-image kernels-soft-dvpp-image) +add_dependencies(kernels-image kernels-soft-dvpp-image ) diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/CMakeLists.txt new file mode 100644 index 0000000000..3b3170702a --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") +set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) +add_library(lite-cv OBJECT + image_process.cc + lite_mat.cc) \ No newline at end of file diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc new file mode 100644 index 0000000000..68d4a3f878 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc @@ -0,0 +1,604 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lite_cv/image_process.h" + +#include +#include +#include +#include + +namespace mindspore { +namespace dataset { + +static inline void InitBilinearWeight(int *data_ptr, int16_t *weight_ptr, double scale, int dst_length, int src_length, + int a) { + const int RESIZE_SCALE = 1 << 11; + if (data_ptr == nullptr || weight_ptr == nullptr) { + return; + } + + int *data_start_ptr = data_ptr; + int16_t *weight_start_ptr = weight_ptr; + + for (unsigned int i = 0; i < dst_length; i++) { + float src_f_x = static_cast((i + 0.5) * scale - 0.5); + int src_u_x = static_cast(floor(src_f_x)); + src_f_x -= src_u_x; + if (src_u_x < 0) { + src_u_x = 0; + src_f_x = 0.0f; + } + if (src_u_x >= src_length - 1) { + src_u_x = src_length - 2; + src_f_x = 1.0f; + } + data_start_ptr[i] = src_u_x * a; + int16_t t0 = INT16_CAST((1.0f - src_f_x) * RESIZE_SCALE); + int16_t t1 = INT16_CAST(src_f_x * RESIZE_SCALE); + + weight_start_ptr[i * 2] = t0; + weight_start_ptr[i * 2 + 1] = t1; + } +} + +static void ResizeBilinear3C(const unsigned char *src, int src_width, int src_height, unsigned char *dst, int dst_width, + int dst_height) { + double scale_width = static_cast(src_width) / dst_width; + double scale_height = static_cast(src_height) / dst_height; + + int *data_buf = new int[2 * dst_width + 2 * dst_height]; + + int *x_offset = data_buf; + int *y_offset = data_buf + dst_width; + + int16_t *x_weight = reinterpret_cast(data_buf + dst_width + dst_height); + int16_t *y_weight = reinterpret_cast(x_weight + dst_width); + + InitBilinearWeight(x_offset, x_weight, scale_width, dst_width, src_width, 3); + InitBilinearWeight(y_offset, y_weight, scale_height, dst_height, src_height, 1); + + LiteMat x_tmp_buf0(dst_width * 3 + 1, LDataType::UINT16); + LiteMat x_tmp_buf1(dst_width * 3 + 1, LDataType::UINT16); + int16_t *row0_ptr = reinterpret_cast(x_tmp_buf0.data_ptr_); + int16_t *row1_ptr = reinterpret_cast(x_tmp_buf1.data_ptr_); + + int prev_height = -2; + + for (int y = 0; y < dst_height; y++) { + int y_span = y_offset[y]; + + if (y_span == prev_height) { + } else if (y_span == prev_height + 1) { + int16_t *tmp = row0_ptr; + row0_ptr = row1_ptr; + row1_ptr = tmp; + const unsigned char *src_start = src + 3 * src_width * (y_span + 1); + const int16_t *x_weight_p = x_weight; + int16_t *row1_ptr1 = row1_ptr; + for (int x = 0; x < dst_width; x++) { + const unsigned char *src_start_p = src_start + x_offset[x]; + row1_ptr1[0] = (src_start_p[0] * x_weight_p[0] + src_start_p[3] * x_weight_p[1]) >> 4; + row1_ptr1[1] = (src_start_p[1] * x_weight_p[0] + src_start_p[4] * x_weight_p[1]) >> 4; + row1_ptr1[2] = (src_start_p[2] * x_weight_p[0] + src_start_p[5] * x_weight_p[1]) >> 4; + x_weight_p += 2; + row1_ptr1 += 3; + } + } else { + const unsigned char *src0 = src + 3 * src_width * (y_span); + const unsigned char *src1 = src + 3 * src_width * (y_span + 1); + + const int16_t *x_weight_ptr = x_weight; + int16_t *row0_ptr0 = row0_ptr; + int16_t *row1_ptr1 = row1_ptr; + for (int x = 0; x < dst_width; x++) { + const unsigned char *src0_ptr = src0 + x_offset[x]; + const unsigned char *src1_ptr = src1 + x_offset[x]; + + for (int c = 0; c < 3; c++) { + row0_ptr0[c] = (src0_ptr[c] * x_weight_ptr[0] + src0_ptr[c + 3] * x_weight_ptr[1]) >> 4; + row1_ptr1[c] = (src1_ptr[c] * x_weight_ptr[0] + src1_ptr[c + 3] * x_weight_ptr[1]) >> 4; + } + + x_weight_ptr += 2; + row0_ptr0 += 3; + row1_ptr1 += 3; + } + } + prev_height = y_span; + + int16_t *row0_ptr0 = row0_ptr; + int16_t *row1_ptr1 = row1_ptr; + unsigned char *dst_ptr = dst + dst_width * 3 * (y); + + for (int k = 0; k < dst_width * 3; k++) { + int16_t t0 = (int16_t)((y_weight[0] * (int16_t)(*row0_ptr0++)) >> 16); + int16_t t1 = (int16_t)((y_weight[1] * (int16_t)(*row1_ptr1++)) >> 16); + *dst_ptr++ = (unsigned char)((t0 + t1 + 2) >> 2); + } + y_weight += 2; + } + delete[] data_buf; +} + +static void ResizeBilinear1C(const unsigned char *src, int src_width, int src_height, unsigned char *dst, int dst_width, + int dst_height) { + double scale_width = static_cast(src_width) / dst_width; + double scale_height = static_cast(src_height) / dst_height; + + int *data_buf = new int[2 * dst_width + 2 * dst_height]; + + int *x_offset = data_buf; + int *y_offset = data_buf + dst_width; + + int16_t *x_weight = reinterpret_cast(data_buf + dst_width + dst_height); + int16_t *y_weight = reinterpret_cast(x_weight + dst_width); + + InitBilinearWeight(x_offset, x_weight, scale_width, dst_width, src_width, 1); + InitBilinearWeight(y_offset, y_weight, scale_height, dst_height, src_height, 1); + + LiteMat x_tmp_buf0(dst_width, LDataType::UINT16); + LiteMat x_tmp_buf1(dst_width, LDataType::UINT16); + int16_t *row0_ptr = reinterpret_cast(x_tmp_buf0.data_ptr_); + int16_t *row1_ptr = reinterpret_cast(x_tmp_buf1.data_ptr_); + + int prev_height = -2; + + for (int y = 0; y < dst_height; y++) { + int y_span = y_offset[y]; + + if (y_span == prev_height) { + } else if (y_span == prev_height + 1) { + int16_t *tmp = row0_ptr; + row0_ptr = row1_ptr; + row1_ptr = tmp; + const unsigned char *src_start = src + src_width * (y_span + 1); + const int16_t *x_weight_p = x_weight; + int16_t *row1_ptr1 = row1_ptr; + for (int x = 0; x < dst_width; x++) { + const unsigned char *src_start_p = src_start + x_offset[x]; + row1_ptr1[x] = (src_start_p[0] * x_weight_p[0] + src_start_p[3] * x_weight_p[1]) >> 4; + x_weight_p += 2; + } + } else { + const unsigned char *src0 = src + src_width * (y_span); + const unsigned char *src1 = src + src_width * (y_span + 1); + + const int16_t *x_weight_ptr = x_weight; + int16_t *row0_ptr0 = row0_ptr; + int16_t *row1_ptr1 = row1_ptr; + for (int x = 0; x < dst_width; x++) { + const unsigned char *src0_ptr = src0 + x_offset[x]; + const unsigned char *src1_ptr = src1 + x_offset[x]; + + row0_ptr0[x] = (src0_ptr[0] * x_weight_ptr[0] + src0_ptr[3] * x_weight_ptr[1]) >> 4; + row1_ptr1[x] = (src1_ptr[0] * x_weight_ptr[0] + src1_ptr[3] * x_weight_ptr[1]) >> 4; + + x_weight_ptr += 2; + } + } + prev_height = y_span; + + int16_t *row0_ptr0 = row0_ptr; + int16_t *row1_ptr1 = row1_ptr; + unsigned char *dst_ptr = dst + dst_width * (y); + + for (int k = 0; k < dst_width; k++) { + int16_t t0 = (int16_t)((y_weight[0] * (int16_t)(*row0_ptr0++)) >> 16); + int16_t t1 = (int16_t)((y_weight[1] * (int16_t)(*row1_ptr1++)) >> 16); + *dst_ptr++ = (unsigned char)((t0 + t1 + 2) >> 2); + } + + y_weight += 2; + } + delete[] data_buf; +} + +bool ResizeBilinear(const LiteMat &src, LiteMat &dst, int dst_w, int dst_h) { + if (src.data_type_ != LDataType::UINT8) { + return false; + } + if (src.channel_ == 3) { + (void)dst.Init(dst_w, dst_h, 3, LDataType::UINT8); + const unsigned char *src_start_p = src; + unsigned char *dst_start_p = dst; + (void)ResizeBilinear3C(src_start_p, src.width_, src.height_, dst_start_p, dst_w, dst_h); + } else if (src.channel_ == 1) { + (void)dst.Init(dst_w, dst_h, 1, LDataType::UINT8); + const unsigned char *src_start_p = src; + unsigned char *dst_start_p = dst; + (void)ResizeBilinear1C(src_start_p, src.width_, src.height_, dst_start_p, dst_w, dst_h); + } else { + return false; + } + return true; +} + +static bool ConvertRGBAToBGR(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) { + if (data_type == LDataType::UINT8) { + mat.Init(w, h, 3, LDataType::UINT8); + unsigned char *ptr = mat; + const unsigned char *data_ptr = data; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + ptr[0] = data_ptr[2]; + ptr[1] = data_ptr[1]; + ptr[2] = data_ptr[0]; + ptr += 3; + data_ptr += 4; + } + } + } else { + return false; + } + return true; +} + +static bool ConvertRGBAToGRAY(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) { + if (data_type == LDataType::UINT8) { + mat.Init(w, h, 1, LDataType::UINT8); + if (mat.IsEmpty()) { + return false; + } + unsigned char *ptr = mat; + const unsigned char *data_ptr = data; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + *ptr = (data_ptr[2] * B2GRAY + data_ptr[1] * G2GRAY + data_ptr[0] * R2GRAY) >> GRAYSHIFT; + ptr++; + data_ptr += 4; + } + } + } else { + return false; + } + return true; +} + +bool InitFromPixel(const unsigned char *data, LPixelType pixel_type, LDataType data_type, int w, int h, LiteMat &m) { + if (pixel_type == LPixelType::RGBA2BGR) { + return ConvertRGBAToBGR(data, data_type, w, h, m); + } else if (pixel_type == LPixelType::RGBA2GRAY) { + return ConvertRGBAToGRAY(data, data_type, w, h, m); + } else { + return false; + } + return true; +} + +bool ConvertTo(const LiteMat &src, LiteMat &dst, double scale) { + if (src.data_type_ != LDataType::UINT8) { + return false; + } + + (void)dst.Init(src.width_, src.height_, src.channel_, LDataType::FLOAT32); + const unsigned char *src_start_p = src; + float *dst_start_p = dst; + for (int h = 0; h < src.height_; h++) { + for (int w = 0; w < src.width_; w++) { + for (int c = 0; c < src.channel_; c++) { + int index = (h * src.width_ + w) * src.channel_; + dst_start_p[index + c] = (static_cast(src_start_p[index + c] * scale)); + } + } + } + return true; +} + +template +static void CropInternal(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h) { + int dst_h = h; + int dst_w = w; + int dst_c = src.channel_; + dst.Init(dst_w, dst_h, dst_c, src.data_type_); + const T *src_start_p = src; + T *dst_start_p = dst; + for (int i_h = 0; i_h < dst_h; i_h++) { + const T *src_index_p = src_start_p + (y + i_h) * src.width_ * dst_c + x * dst_c; + T *dst_index_p = dst_start_p + i_h * dst_w * dst_c; + (void)memcpy(dst_index_p, src_index_p, dst_w * dst_c * sizeof(T)); + } +} + +bool Crop(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h) { + if (y < 0 || y + h > src.height_ || x < 0 || x + w > src.width_) { + return false; + } + + if (src.data_type_ == LDataType::UINT8) { + CropInternal(src, dst, x, y, w, h); + } else if (src.data_type_ == LDataType::FLOAT32) { + CropInternal(src, dst, x, y, w, h); + } else { + return false; + } + return true; +} + +bool SubStractMeanNormalize(const LiteMat &src, LiteMat &dst, float *mean, float *norm) { + if (src.data_type_ != LDataType::FLOAT32) { + return false; + } + + dst.Init(src.width_, src.height_, src.channel_, LDataType::FLOAT32); + + const float *src_start_p = src; + float *dst_start_p = dst; + if (mean && !norm) { + for (int h = 0; h < src.height_; h++) { + for (int w = 0; w < src.width_; w++) { + for (int c = 0; c < src.channel_; c++) { + int index = (h * src.width_ + w) * src.channel_ + c; + dst_start_p[index] = src_start_p[index] - mean[c]; + } + } + } + } else if (!mean && norm) { + for (int h = 0; h < src.height_; h++) { + for (int w = 0; w < src.width_; w++) { + for (int c = 0; c < src.channel_; c++) { + int index = (h * src.width_ + w) * src.channel_ + c; + dst_start_p[index] = src_start_p[index] * norm[c]; + } + } + } + } else if (mean && norm) { + for (int h = 0; h < src.height_; h++) { + for (int w = 0; w < src.width_; w++) { + for (int c = 0; c < src.channel_; c++) { + int index = (h * src.width_ + w) * src.channel_ + c; + dst_start_p[index] = (src_start_p[index] - mean[c]) * norm[c]; + } + } + } + } else { + return false; + } + return true; +} + +template +static void PaddWithConstant(const LiteMat &src, LiteMat &dst, const int top, const int bottom, const int left, + const int right, const PaddBorderType pad_type, uint8_t fill_b_or_gray, uint8_t fill_g, + uint8_t fill_r) { + dst.Init(src.width_ + left + right, src.height_ + top + bottom, src.channel_, src.data_type_); + const T *src_start_p = src; + T *dst_start_p = dst; + // padd top + for (int h = 0; h < top; h++) { + for (int w = 0; w < dst.width_; w++) { + int index = (h * dst.width_ + w) * dst.channel_; + if (dst.channel_ == 1) { + dst_start_p[index] = fill_b_or_gray; + } else if (dst.channel_ == 3) { + dst_start_p[index] = fill_b_or_gray; + dst_start_p[index + 1] = fill_g; + dst_start_p[index + 2] = fill_r; + } else { + } + } + } + // padd bottom + for (int h = dst.height_ - bottom; h < dst.height_; h++) { + for (int w = 0; w < dst.width_; w++) { + int index = (h * dst.width_ + w) * dst.channel_; + if (dst.channel_ == 1) { + dst_start_p[index] = fill_b_or_gray; + } else if (dst.channel_ == 3) { + dst_start_p[index] = fill_b_or_gray; + dst_start_p[index + 1] = fill_g; + dst_start_p[index + 2] = fill_r; + } else { + } + } + } + + // padd left + for (int h = top; h < dst.height_ - bottom; h++) { + for (int w = 0; w < left; w++) { + int index = (h * dst.width_ + w) * dst.channel_; + if (dst.channel_ == 1) { + dst_start_p[index] = fill_b_or_gray; + } else if (dst.channel_ == 3) { + dst_start_p[index] = fill_b_or_gray; + dst_start_p[index + 1] = fill_g; + dst_start_p[index + 2] = fill_r; + } else { + } + } + } + + // padd right + for (int h = top; h < dst.height_ - bottom; h++) { + for (int w = dst.width_ - right; w < dst.width_; w++) { + int index = (h * dst.width_ + w) * dst.channel_; + if (dst.channel_ == 1) { + dst_start_p[index] = fill_b_or_gray; + } else if (dst.channel_ == 3) { + dst_start_p[index] = fill_b_or_gray; + dst_start_p[index + 1] = fill_g; + dst_start_p[index + 2] = fill_r; + } else { + } + } + } + // image data + dst_start_p = dst_start_p + (top * dst.width_ + left) * dst.channel_; + for (int i_h = 0; i_h < src.height_; i_h++) { + const T *src_index_p = src_start_p + i_h * src.width_ * src.channel_; + T *dst_index_p = dst_start_p + i_h * dst.width_ * dst.channel_; + (void)memcpy(dst_index_p, src_index_p, src.width_ * src.channel_ * sizeof(T)); + } +} + +bool Padd(const LiteMat &src, LiteMat &dst, int top, int bottom, int left, int right, PaddBorderType pad_type, + uint8_t fill_b_or_gray, uint8_t fill_g, uint8_t fill_r) { + if (pad_type == PADD_BORDER_CONSTANT && src.data_type_ == LDataType::FLOAT32) { + PaddWithConstant(src, dst, top, bottom, left, right, pad_type, fill_b_or_gray, fill_g, fill_r); + } else if (pad_type == PADD_BORDER_CONSTANT && src.data_type_ == LDataType::UINT8) { + PaddWithConstant(src, dst, top, bottom, left, right, pad_type, fill_b_or_gray, fill_g, fill_r); + } else { + return false; + } + return true; +} + +std::vector> GetDefaultBoxes(BoxesConfig config) { + std::vector fk; + float num = static_cast(config.img_shape[0]); + for (int i = 0; i < config.steps.size(); i++) { + fk.push_back(num / config.steps[i]); + } + float scale_rate = (config.max_scale - config.min_scale) / (config.num_default.size() - 1); + std::vector scales(config.num_default.size()); + for (int i = 0; i < scales.size(); i++) { + scales[i] = config.min_scale + scale_rate * i; + } + scales.push_back(1.0f); + std::vector> default_boxes; + for (int i = 0; i < config.feature_size.size(); i++) { + float sk1 = scales[i]; + float sk2 = scales[i + 1]; + float sk3 = sqrt(sk1 * sk2); + std::vector> all_sizes; + float w, h; + if (i == 0) { + w = sk1 * sqrt(2); + h = sk1 / sqrt(2); + all_sizes = {{0.1, 0.1}, {w, h}, {h, w}}; + } else { + all_sizes = {{sk1, sk1}}; + for (int j = 0; j < config.aspect_rations[i].size(); j++) { + w = sk1 * sqrt(config.aspect_rations[i][j]); + h = sk1 / sqrt(config.aspect_rations[i][j]); + all_sizes.push_back({w, h}); + all_sizes.push_back({h, w}); + } + all_sizes.push_back({sk3, sk3}); + } + + for (int j = 0; j < config.feature_size[i]; j++) { + for (int k = 0; k < config.feature_size[i]; k++) { + for (int m = 0; m < all_sizes.size(); m++) { + float cx = (k + 0.5) / fk[i]; + float cy = (j + 0.5) / fk[i]; + default_boxes.push_back({cy, cx, all_sizes[m][1], all_sizes[m][0]}); + } + } + } + } + return default_boxes; +} + +void ConvertBoxes(std::vector> &boxes, const std::vector> &default_boxes, + const BoxesConfig config) { + for (int i = 0; i < default_boxes.size(); i++) { + boxes[i][0] = boxes[i][0] * config.prior_scaling[0] * default_boxes[i][2] + default_boxes[i][0]; + boxes[i][1] = boxes[i][1] * config.prior_scaling[0] * default_boxes[i][3] + default_boxes[i][1]; + boxes[i][2] = exp(boxes[i][2] * config.prior_scaling[1]) * default_boxes[i][2]; + boxes[i][3] = exp(boxes[i][3] * config.prior_scaling[1]) * default_boxes[i][3]; + } +} + +std::vector ApplyNms(const std::vector> &all_boxes, std::vector &all_scores, float thres, + int max_boxes) { + int boxes_num = all_boxes.size(); + std::vector y1(boxes_num); + std::vector x1(boxes_num); + std::vector y2(boxes_num); + std::vector x2(boxes_num); + std::vector areas(boxes_num); + std::vector order(boxes_num); + for (int i = 0; i < boxes_num; i++) { + y1[i] = all_boxes[i][0]; + x1[i] = all_boxes[i][1]; + y2[i] = all_boxes[i][2]; + x2[i] = all_boxes[i][3]; + areas[i] = (x2[i] - x1[i] + 1) * (y2[i] - y1[i] + 1); + order[i] = i; + } + + std::sort(order.begin(), order.end(), + [&all_scores](int pos1, int pos2) { return (all_scores[pos1] > all_scores[pos2]); }); + std::vector keep; + while (order.size() > 0) { + int i = order[0]; + keep.push_back(i); + if (keep.size() >= max_boxes) { + break; + } + int len = order.size() - 1; + std::vector ovr(len); + for (int j = 0; j < len; j++) { + float xx1 = std::max(x1[i], x1[order[j + 1]]); + float yy1 = std::max(y1[i], y1[order[j + 1]]); + float xx2 = std::min(x2[i], x2[order[j + 1]]); + float yy2 = std::min(y2[i], y2[order[j + 1]]); + + float w = std::max(0.0f, xx2 - xx1 + 1); + float h = std::max(0.0f, yy2 - yy1 + 1); + float inter = w * h; + ovr[j] = inter / (areas[i] + areas[order[j + 1]] - inter); + } + std::vector inds; + for (int j = 0; j < len; j++) { + if (ovr[j] <= thres) { + inds.push_back(j + 1); + } + } + std::vector new_order; + for (int k = 0; k < inds.size(); k++) { + new_order.push_back(order[inds[k]]); + } + order = new_order; + } + return keep; +} + +void WarpAffine(LiteMat &src, LiteMat &out_img, double M[6], std::vector dsize, UINT8_C3 borderValue) { + double IM[6]; + for (int i = 0; i < 6; i++) { + IM[i] = M[i]; + } + + double D = IM[0] * IM[4] - IM[1] * IM[3]; + D = D != 0 ? 1. / D : 0; + double A11 = IM[4] * D, A22 = IM[0] * D; + IM[0] = A11; + IM[1] *= -D; + IM[3] *= -D; + IM[4] = A22; + double b1 = -IM[0] * IM[2] - IM[1] * IM[5]; + double b2 = -IM[3] * IM[2] - IM[4] * IM[5]; + IM[2] = b1; + IM[5] = b2; + + out_img.Init(dsize[0], dsize[1]); + for (int y = 0; y < out_img.height_; y++) { + for (int x = 0; x < out_img.width_; x++) { + int src_x = IM[0] * x + IM[1] * y + IM[2]; + int src_y = IM[3] * x + IM[4] * y + IM[5]; + if (src_x >= 0 && src_y >= 0 && src_x < src.width_ && src_y < src.height_) { + UINT8_C3 src_pixel = static_cast(src.data_ptr_)[src_y * src.width_ + src_x]; + static_cast(out_img.data_ptr_)[y * src.width_ + x] = src_pixel; + } else { + static_cast(out_img.data_ptr_)[y * src.width_ + x] = borderValue; + } + } + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h new file mode 100644 index 0000000000..297c24f5a4 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h @@ -0,0 +1,83 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_PROCESS_H_ +#define IMAGE_PROCESS_H_ + +#include +#include +#include + +#include "lite_cv/lite_mat.h" + +namespace mindspore { +namespace dataset { + +#define INT16_CAST(X) \ + static_cast(::std::min(::std::max(static_cast(X + (X >= 0.f ? 0.5f : -0.5f)), -32768), 32767)); + +#define R2GRAY 77 +#define G2GRAY 150 +#define B2GRAY 29 +#define GRAYSHIFT 8 + +enum PaddBorderType { PADD_BORDER_CONSTANT = 0, PADD_BORDER_REPLICATE = 1 }; + +struct BoxesConfig { + public: + std::vector img_shape; + std::vector num_default; + std::vector feature_size; + float min_scale; + float max_scale; + std::vector> aspect_rations; + std::vector steps; + std::vector prior_scaling; +}; + +/// \brief resizing image by bilinear algorithm, the data type of currently only supports is uint8, +/// the channel of currently supports is 3 and 1 +bool ResizeBilinear(const LiteMat &src, LiteMat &dst, int dst_w, int dst_h); + +/// \brief Init Lite Mat from pixel, the conversion of currently supports is rbgaTorgb and rgbaTobgr +bool InitFromPixel(const unsigned char *data, LPixelType pixel_type, LDataType data_type, int w, int h, LiteMat &m); + +/// \brief convert the data type, the conversion of currently supports is uint8 to float +bool ConvertTo(const LiteMat &src, LiteMat &dst, double scale = 1.0); + +/// \brief crop image, the channel supports is 3 and 1 +bool Crop(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h); + +/// \brief normalize image, currently the supports data type is float +bool SubStractMeanNormalize(const LiteMat &src, LiteMat &dst, float *mean, float *norm); + +/// \brief padd image, the channel supports is 3 and 1 +bool Padd(const LiteMat &src, LiteMat &dst, int top, int bottom, int left, int right, PaddBorderType pad_type, + uint8_t fill_b_or_gray, uint8_t fill_g, uint8_t fill_r); + +void WarpAffine(const LiteMat &src, LiteMat &out_img, double M[6], std::vector dsize, uint8_t borderValue[3]); + +std::vector> GetDefaultBoxes(const BoxesConfig config); + +void ConvertBoxes(std::vector> &boxes, const std::vector> &default_boxes, + const BoxesConfig config); + +std::vector ApplyNms(const std::vector> &all_boxes, std::vector &all_scores, float thres, + int max_boxes); + +} // namespace dataset +} // namespace mindspore +#endif // IMAGE_PROCESS_H_ diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc new file mode 100644 index 0000000000..f35d65bc39 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc @@ -0,0 +1,207 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "lite_cv/lite_mat.h" + +namespace mindspore { +namespace dataset { + +LiteMat::LiteMat() { + data_ptr_ = 0; + elem_size_ = 0; + width_ = 0; + height_ = 0; + channel_ = 0; + c_step_ = 0; + dims_ = 0; + data_type_ = LDataType::UINT8; + ref_count_ = 0; +} + +LiteMat::LiteMat(int width, LDataType data_type) { + data_ptr_ = 0; + elem_size_ = 0; + width_ = 0; + height_ = 0; + channel_ = 0; + c_step_ = 0; + dims_ = 0; + data_type_ = LDataType::UINT8; + ref_count_ = 0; + Init(width, data_type); +} + +LiteMat::LiteMat(int width, int height, LDataType data_type) { + data_ptr_ = 0; + elem_size_ = 0; + width_ = 0; + height_ = 0; + channel_ = 0; + c_step_ = 0; + dims_ = 0; + data_type_ = LDataType::UINT8; + ref_count_ = 0; + Init(width, height, data_type); +} + +LiteMat::LiteMat(int width, int height, int channel, LDataType data_type) { + data_ptr_ = 0; + elem_size_ = 0; + width_ = 0; + height_ = 0; + channel_ = 0; + c_step_ = 0; + dims_ = 0; + data_type_ = LDataType::UINT8; + ref_count_ = 0; + Init(width, height, channel, data_type); +} + +LiteMat::~LiteMat() { Release(); } + +int LiteMat::addRef(int *p, int value) { + int v = *p; + *p += value; + return v; +} + +LiteMat::LiteMat(const LiteMat &m) { + data_ptr_ = m.data_ptr_; + elem_size_ = m.elem_size_; + width_ = m.width_; + height_ = m.height_; + channel_ = m.channel_; + c_step_ = m.c_step_; + dims_ = m.dims_; + data_type_ = m.data_type_; + ref_count_ = m.ref_count_; + if (ref_count_) { + addRef(ref_count_, 1); + } +} + +LiteMat &LiteMat::operator=(const LiteMat &m) { + if (this == &m) { + return *this; + } + + if (m.ref_count_) { + addRef(m.ref_count_, 1); + } + + Release(); + data_ptr_ = m.data_ptr_; + elem_size_ = m.elem_size_; + width_ = m.width_; + height_ = m.height_; + channel_ = m.channel_; + c_step_ = m.c_step_; + dims_ = m.dims_; + data_type_ = m.data_type_; + ref_count_ = m.ref_count_; + return *this; +} + +void LiteMat::Init(int width, LDataType data_type) { + Release(); + data_type_ = data_type; + InitElemSize(data_type); + width_ = width; + dims_ = 1; + height_ = 1; + channel_ = 1; + c_step_ = width; + size_ = c_step_ * elem_size_; + data_ptr_ = AlignMalloc(size_); + ref_count_ = new int[1]; + *ref_count_ = 1; +} + +void LiteMat::Init(int width, int height, LDataType data_type) { + Release(); + data_type_ = data_type; + InitElemSize(data_type); + + width_ = width; + height_ = height; + dims_ = 2; + channel_ = 1; + c_step_ = width_ * height_; + size_ = c_step_ * elem_size_; + data_ptr_ = AlignMalloc(size_); + ref_count_ = new int[1]; + *ref_count_ = 1; +} + +void LiteMat::Init(int width, int height, int channel, LDataType data_type) { + Release(); + data_type_ = data_type; + InitElemSize(data_type); + width_ = width; + height_ = height; + dims_ = 3; + channel_ = channel; + c_step_ = ((height_ * width_ * elem_size_ + ALIGN - 1) & (-ALIGN)) / elem_size_; + size_ = c_step_ * channel_ * elem_size_; + + data_ptr_ = AlignMalloc(size_); + + ref_count_ = new int[1]; + *ref_count_ = 1; +} + +bool LiteMat::IsEmpty() const { return data_ptr_ == 0 || data_ptr_ == nullptr || c_step_ * channel_ == 0; } + +void LiteMat::Release() { + if (ref_count_ && (addRef(ref_count_, -1) == 1)) { + if (data_ptr_) { + AlignFree(data_ptr_); + } + if (ref_count_) { + delete[] ref_count_; + } + } + data_ptr_ = 0; + elem_size_ = 0; + width_ = 0; + height_ = 0; + channel_ = 0; + c_step_ = 0; + ref_count_ = 0; +} + +void *LiteMat::AlignMalloc(unsigned int size) { + unsigned int length = sizeof(void *) + ALIGN - 1; + void *p_raw = reinterpret_cast(malloc(size + length)); + if (p_raw) { + void **p_algin = reinterpret_cast(((size_t)(p_raw) + length) & ~(ALIGN - 1)); + p_algin[-1] = p_raw; + return p_algin; + } + return nullptr; +} +void LiteMat::AlignFree(void *ptr) { (void)free(reinterpret_cast(ptr)[-1]); } +inline void LiteMat::InitElemSize(LDataType data_type) { + if (data_type == LDataType::UINT8) { + elem_size_ = 1; + } else if (data_type == LDataType::UINT16) { + elem_size_ = 2; + } else if (data_type == LDataType::FLOAT32) { + elem_size_ = 4; + } else { + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h new file mode 100644 index 0000000000..1e902e89c5 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.h @@ -0,0 +1,219 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINI_MAT_H_ +#define MINI_MAT_H_ + +#include +#include + +namespace mindspore { +namespace dataset { + +#define ALIGN 16 + +template +struct Chn1 { + T c1; +}; + +template +struct Chn2 { + T c1; + T c2; +}; + +template +struct Chn3 { + T c1; + T c2; + T c3; +}; + +template +struct Chn4 { + T c1; + T c2; + T c3; + T c4; +}; + +using UINT8_C1 = Chn1; +using UINT8_C2 = Chn2; +using UINT8_C3 = Chn3; +using UINT8_C4 = Chn4; + +using INT8_C1 = Chn1; +using INT8_C2 = Chn2; +using INT8_C3 = Chn3; +using INT8_C4 = Chn4; + +using UINT32_C1 = Chn1; +using UINT32_C2 = Chn2; +using UINT32_C3 = Chn3; +using UINT32_C4 = Chn4; + +using INT32_C1 = Chn1; +using INT32_C2 = Chn2; +using INT32_C3 = Chn3; +using INT32_C4 = Chn4; + +using FLOAT32_C1 = Chn1; +using FLOAT32_C2 = Chn2; +using FLOAT32_C3 = Chn3; +using FLOAT32_C4 = Chn4; + +using FLOAT64_C1 = Chn1; +using FLOAT64_C2 = Chn2; +using FLOAT64_C3 = Chn3; +using FLOAT64_C4 = Chn4; + +enum LPixelType { + BGR = 0, + RGB = 1, + RGBA = 2, + RGBA2GRAY = 3, + RGBA2BGR = 4, +}; + +class LDataType { + public: + enum Type : uint8_t { + UNKNOWN = 0, + BOOL, + INT8, + UINT8, + INT16, + UINT16, + INT32, + UINT32, + INT64, + UINT64, + FLOAT16, + FLOAT32, + FLOAT64, + NUM_OF_TYPES + }; + + LDataType() : type_(UNKNOWN) {} + + LDataType(Type d) : type_(d) {} + + ~LDataType() = default; + + inline Type Value() const { return type_; } + inline bool operator==(const LDataType &ps) const { + if (this->type_ == ps.type_) return true; + return false; + } + + inline bool operator!=(const LDataType &ps) const { + if (this->type_ != ps.type_) return true; + return false; + } + + uint8_t SizeInBytes() const { + if (type_ < LDataType::NUM_OF_TYPES) + return SIZE_IN_BYTES[type_]; + else + return 0; + } + + public: + static inline const uint8_t SIZE_IN_BYTES[] = { + 0, // UNKNOWN + 1, // BOOL + 1, // INT8 + 1, // UINT8 + 2, // INT16 + 2, // UINT16 + 4, // INT32 + 4, // UINT32 + 8, // INT64 + 8, // UINT64 + 2, // FLOAT16 + 4, // FLOAT32 + 8, // FLOAT64 + }; + + Type type_; +}; + +class LiteMat { + // Class that represents a lite Mat of a Image. + // -# The pixel type of Lite Mat is RGBRGB...RGB. + public: + LiteMat(); + + explicit LiteMat(int width, LDataType data_type = LDataType::UINT8); + + LiteMat(int width, int height, LDataType data_type = LDataType::UINT8); + + LiteMat(int width, int height, int channel, LDataType data_type = LDataType::UINT8); + + ~LiteMat(); + + LiteMat(const LiteMat &m); + + void Init(int width, LDataType data_type = LDataType::UINT8); + + void Init(int width, int height, LDataType data_type = LDataType::UINT8); + + void Init(int width, int height, int channel, LDataType data_type = LDataType::UINT8); + + bool IsEmpty() const; + + void Release(); + + LiteMat &operator=(const LiteMat &m); + + template + operator T *() { + return reinterpret_cast(data_ptr_); + } + + template + operator const T *() const { + return reinterpret_cast(data_ptr_); + } + + private: + /// \brief apply for memory alignment + void *AlignMalloc(unsigned int size); + + /// \brief free memory + void AlignFree(void *ptr); + + void InitElemSize(LDataType data_type); + + /// \brief add reference + int addRef(int *p, int value); + + public: + void *data_ptr_ = nullptr; + int elem_size_; + int width_; + int height_; + int channel_; + int c_step_; + int dims_; + size_t size_; + LDataType data_type_; + int *ref_count_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINI_MAT_H_ diff --git a/mindspore/lite/CMakeLists.txt b/mindspore/lite/CMakeLists.txt index 32030375f0..351cf6f6ae 100644 --- a/mindspore/lite/CMakeLists.txt +++ b/mindspore/lite/CMakeLists.txt @@ -76,7 +76,7 @@ option(BUILD_CONVERTER "if build converter" on) option(ENABLE_FP16 "if build fp16 ops" off) option(SUPPORT_GPU "if support gpu" off) option(OFFLINE_COMPILE "if offline compile OpenCL kernel" off) -set(BUILD_MINDDATA "off" CACHE STRING "off, lite, or full") +set(BUILD_MINDDATA "off" CACHE STRING "off, lite, lite_cv or full") option(BUILD_MINDDATA_EXAMPLE "" on) set(CMAKE_VERBOSE_MAKEFILE on) @@ -215,6 +215,13 @@ if (BUILD_MINDDATA STREQUAL "lite" OR BUILD_MINDDATA STREQUAL "full") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/minddata) endif () +if (BUILD_MINDDATA STREQUAL "lite_cv") + # TODO: add sentencepiece dependency + #include(${TOP_DIR}/cmake/external_libs/sentencepiece.cmake) + add_compile_definitions(ENABLE_ANDROID) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/minddata) +endif () + if (BUILD_DEVICE) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/nnacl) diff --git a/mindspore/lite/minddata/CMakeLists.txt b/mindspore/lite/minddata/CMakeLists.txt index 46abb56b9f..bea627be3c 100644 --- a/mindspore/lite/minddata/CMakeLists.txt +++ b/mindspore/lite/minddata/CMakeLists.txt @@ -70,6 +70,9 @@ AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/text/kernels MINDDATA_TEXT_KERNELS_SRC_FILE AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/util MINDDATA_UTIL_SRC_FILES) +AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/image/lite_cv MINDDATA_KERNELS_IMAGE_LITE_CV_FILES) + + if (BUILD_MINDDATA STREQUAL "full") list(REMOVE_ITEM MINDDATA_API_SRC_FILES "${MINDDATA_DIR}/api/text.cc" @@ -175,7 +178,6 @@ if (BUILD_MINDDATA STREQUAL "full") COMMAND cp -rf ${CMAKE_CURRENT_SOURCE_DIR}/example/testCifar10Data ${CMAKE_BINARY_DIR}/minddata ) endif() - elseif (BUILD_MINDDATA STREQUAL "lite") list(REMOVE_ITEM MINDDATA_CORE_SRC_FILES "${MINDDATA_DIR}/core/client.cc") list(REMOVE_ITEM MINDDATA_KERNELS_SRC_FILES "${MINDDATA_DIR}/kernels/py_func_op.cc") @@ -207,6 +209,21 @@ elseif (BUILD_MINDDATA STREQUAL "lite") mindspore::json ) + # ref: https://github.com/android/ndk/issues/1202 + if (PLATFORM_ARM32) + file(GLOB_RECURSE LIBCLANG_RT_LIB $ENV{ANDROID_NDK}/libclang_rt.builtins-arm-android.a) + if (LIBCLANG_RT_LIB STREQUAL "") + MESSAGE(FATAL_ERROR "Cannot find libclang_rt.builtins-arm-androi2d.a in $ENV{ANDROID_NDK}") + endif() + target_link_libraries(minddata-lite ${LIBCLANG_RT_LIB}) + endif() +elseif (BUILD_MINDDATA STREQUAL "lite_cv") + include_directories(${MINDDATA_DIR}/kernels/image) + message(STATUS ${MINDDATA_DIR}/kernels/image) + add_library(minddata-lite SHARED + ${MINDDATA_KERNELS_IMAGE_LITE_CV_FILES} + ) + # ref: https://github.com/android/ndk/issues/1202 if (PLATFORM_ARM32) file(GLOB_RECURSE LIBCLANG_RT_LIB $ENV{ANDROID_NDK}/libclang_rt.builtins-arm-android.a) diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 4e2e390052..f6d2deac28 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -33,12 +33,14 @@ if(ENABLE_MINDDATA) endif() # fetch ut test files if(ENABLE_MINDDATA) + include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image) file(GLOB_RECURSE UT_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ./stub/*.cc ./common/*.cc ./abstract/*.cc ./base/*.cc ./dataset/*.cc + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/*.cc ./debug/*.cc ./device/*.cc ./ir/*.cc @@ -55,6 +57,7 @@ if(ENABLE_MINDDATA) ./utils/*.cc ./vm/*.cc ) + if(NOT ENABLE_PYTHON) set(PYTHON_RELATED_SRCS dataset/filter_op_test.cc @@ -159,6 +162,11 @@ endif() if (CMAKE_SYSTEM_NAME MATCHES "Linux") target_link_libraries(ut_tests PRIVATE mindspore::gtest mindspore::event mindspore::event_pthreads mindspore_gvar ${PYTHON_LIBRARIES} pthread util dl) if (ENABLE_MINDDATA) + + # AUX_SOURCE_DIRECTORY(LITE_CV_FILES) + # message(STATUS "xxxxxxxxxxxxxxxxx"${LITE_CV_FILES} ) + # add_library(_live_cv OBJECT ${LITE_CV_FILES}) + target_link_libraries(ut_tests PRIVATE _c_dataengine _c_mindrecord) endif() else() diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 8e79490a25..63d08e3163 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -123,6 +123,7 @@ SET(DE_UT_SRCS swap_red_blue_test.cc distributed_sampler_test.cc data_helper_test.cc + image_process_test.cc ) if (ENABLE_PYTHON) diff --git a/tests/ut/cpp/dataset/image_process_test.cc b/tests/ut/cpp/dataset/image_process_test.cc new file mode 100644 index 0000000000..3f8bbd14e3 --- /dev/null +++ b/tests/ut/cpp/dataset/image_process_test.cc @@ -0,0 +1,252 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common.h" +#include "lite_cv/lite_mat.h" +#include "lite_cv/image_process.h" +#include +#include +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +class MindDataImageProcess : public UT::Common { + public: + MindDataImageProcess() {} + + void SetUp() {} +}; + +void CompareMat(cv::Mat cv_mat, LiteMat lite_mat) { + int cv_h = cv_mat.rows; + int cv_w = cv_mat.cols; + int cv_c = cv_mat.channels(); + int lite_h = lite_mat.height_; + int lite_w = lite_mat.width_; + int lite_c = lite_mat.channel_; + ASSERT_TRUE(cv_h == lite_h); + ASSERT_TRUE(cv_w == lite_w); + ASSERT_TRUE(cv_c == lite_c); +} + +LiteMat Lite3CImageProcess(LiteMat &lite_mat_bgr) { + bool ret; + LiteMat lite_mat_resize; + ret = ResizeBilinear(lite_mat_bgr, lite_mat_resize, 256, 256); + if (!ret) { + MS_LOG(ERROR) << "ResizeBilinear error"; + } + LiteMat lite_mat_convert_float; + ret = ConvertTo(lite_mat_resize, lite_mat_convert_float, 1.0 ); + if (!ret) { + MS_LOG(ERROR) << "ConvertTo error"; + } + + LiteMat lite_mat_crop; + ret = Crop(lite_mat_convert_float, lite_mat_crop, 16, 16, 224, 224); + if (!ret) { + MS_LOG(ERROR) << "Crop error"; + } + + float means[3] = {0.485, 0.456, 0.406}; + float vars[3] = {1.0 / 0.229, 1.0 / 0.224, 1.0 / 0.225}; + + LiteMat lite_norm_mat_cut; + SubStractMeanNormalize(lite_mat_crop, lite_norm_mat_cut, means, vars); + + return lite_norm_mat_cut; +} + +cv::Mat cv3CImageProcess(cv::Mat &image) { + cv::Mat resize_256_image; + cv::resize(image, resize_256_image, cv::Size(256, 256), CV_INTER_LINEAR); + cv::Mat float_256_image; + resize_256_image.convertTo(float_256_image, CV_32FC3); + + cv::Mat roi_224_image; + cv::Rect roi; + roi.x = 16; + roi.y = 16; + roi.width = 224; + roi.height = 224; + + float_256_image(roi).copyTo(roi_224_image); + + float meanR = 0.485; + float meanG = 0.456; + float meanB = 0.406; + float varR = 0.229; + float varG = 0.224; + float varB = 0.225; + cv::Scalar mean = cv::Scalar(meanR, meanG, meanB); + cv::Scalar var = cv::Scalar(varR, varG, varB); + + cv::Mat imgMean(roi_224_image.size(), CV_32FC3, mean); + cv::Mat imgVar(roi_224_image.size(), CV_32FC3, var); + + cv::Mat imgR1 = roi_224_image - imgMean; + cv::Mat imgR2 = imgR1 / imgVar; + return imgR2; +} + +TEST_F(MindDataImageProcess, test3C) { + std::string filename = "data/dataset/apple.jpg"; + cv::Mat image = cv::imread(filename, cv::ImreadModes::IMREAD_COLOR); + cv::Mat cv_image = cv3CImageProcess(image); + // cv::imwrite("/home/xlei/test_3cv.jpg", cv_image); + + // convert to RGBA for Android bitmap(rgba) + cv::Mat rgba_mat; + cv::cvtColor(image, rgba_mat, CV_BGR2RGBA); + + bool ret = false; + LiteMat lite_mat_bgr; + ret = + InitFromPixel(rgba_mat.data, LPixelType::RGBA2BGR, LDataType::UINT8, rgba_mat.cols, rgba_mat.rows, lite_mat_bgr); + if (!ret) { + MS_LOG(ERROR) << "Init From RGBA error"; + } + LiteMat lite_norm_mat_cut = Lite3CImageProcess(lite_mat_bgr); + + cv::Mat dst_image(lite_norm_mat_cut.height_, lite_norm_mat_cut.width_, CV_32FC3, lite_norm_mat_cut.data_ptr_); + // cv::imwrite("/home/xlei/test_3clite.jpg", dst_image); + + CompareMat(cv_image, lite_norm_mat_cut); +} + +LiteMat Lite1CImageProcess(LiteMat &lite_mat_bgr) { + LiteMat lite_mat_resize; + ResizeBilinear(lite_mat_bgr, lite_mat_resize, 256, 256); + LiteMat lite_mat_convert_float; + ConvertTo(lite_mat_resize, lite_mat_convert_float); + + LiteMat lite_mat_cut; + + Crop(lite_mat_convert_float, lite_mat_cut, 16, 16, 224, 224); + + float means[1] = {0.485}; + float vars[1] = {1.0 / 0.229}; + + LiteMat lite_norm_mat_cut; + + SubStractMeanNormalize(lite_mat_cut, lite_norm_mat_cut, means, vars); + return lite_norm_mat_cut; +} + +cv::Mat cv1CImageProcess(cv::Mat &image) { + cv::Mat gray_image; + cv::cvtColor(image, gray_image, CV_BGR2GRAY); + + cv::Mat resize_256_image; + cv::resize(gray_image, resize_256_image, cv::Size(256, 256), CV_INTER_LINEAR); + cv::Mat float_256_image; + resize_256_image.convertTo(float_256_image, CV_32FC3); + + cv::Mat roi_224_image; + cv::Rect roi; + roi.x = 16; + roi.y = 16; + roi.width = 224; + roi.height = 224; + + float_256_image(roi).copyTo(roi_224_image); + + float meanR = 0.485; + float varR = 0.229; + cv::Scalar mean = cv::Scalar(meanR); + cv::Scalar var = cv::Scalar(varR); + + cv::Mat imgMean(roi_224_image.size(), CV_32FC1, mean); + cv::Mat imgVar(roi_224_image.size(), CV_32FC1, var); + + cv::Mat imgR1 = roi_224_image - imgMean; + cv::Mat imgR2 = imgR1 / imgVar; + return imgR2; +} + +TEST_F(MindDataImageProcess, test1C) { + std::string filename = "data/dataset/apple.jpg"; + cv::Mat image = cv::imread(filename, cv::ImreadModes::IMREAD_COLOR); + cv::Mat cv_image = cv1CImageProcess(image); + + // cv::imwrite("/home/xlei/test_c1v.jpg", cv_image); + + // convert to RGBA for Android bitmap(rgba) + cv::Mat rgba_mat; + cv::cvtColor(image, rgba_mat, CV_BGR2RGBA); + + LiteMat lite_mat_bgr; + InitFromPixel(rgba_mat.data, LPixelType::RGBA2GRAY, LDataType::UINT8, rgba_mat.cols, rgba_mat.rows, lite_mat_bgr); + LiteMat lite_norm_mat_cut = Lite1CImageProcess(lite_mat_bgr); + cv::Mat dst_image(lite_norm_mat_cut.height_, lite_norm_mat_cut.width_, CV_32FC1, lite_norm_mat_cut.data_ptr_); + // cv::imwrite("/home/xlei/test_c1lite.jpg", dst_image); + + CompareMat(cv_image, lite_norm_mat_cut); +} + +TEST_F(MindDataImageProcess, TestPadd) { + std::string filename = "data/dataset/apple.jpg"; + cv::Mat image = cv::imread(filename, cv::ImreadModes::IMREAD_COLOR); + + cv::Mat resize_256_image; + cv::resize(image, resize_256_image, cv::Size(256, 256), CV_INTER_LINEAR); + int left = 10; + int right = 10; + int top = 10; + int bottom = 10; + cv::Mat b_image; + cv::Scalar color = cv::Scalar(255, 255, 255); + cv::copyMakeBorder(resize_256_image, b_image, top, bottom, left, right, cv::BORDER_CONSTANT, color); + // cv::imwrite("/home/xlei/test_ccc.jpg", b_image); + cv::Mat rgba_mat; + cv::cvtColor(image, rgba_mat, CV_BGR2RGBA); + + LiteMat lite_mat_bgr; + InitFromPixel(rgba_mat.data, LPixelType::RGBA2BGR, LDataType::UINT8, rgba_mat.cols, rgba_mat.rows, lite_mat_bgr); + + LiteMat lite_mat_resize; + ResizeBilinear(lite_mat_bgr, lite_mat_resize, 256, 256); + + LiteMat makeborder; + Padd(lite_mat_resize, makeborder, top, bottom, left, right, PaddBorderType::PADD_BORDER_CONSTANT, 255, 255, 255); + + cv::Mat dst_image(256 + top + bottom, 256 + left + right, CV_8UC3, makeborder.data_ptr_); + + // cv::imwrite("/home/xlei/test_liteccc.jpg", dst_image); +} + +TEST_F(MindDataImageProcess, TestGetDefaultBoxes) { + BoxesConfig config; + config.img_shape = {300, 300}; + config.num_default = {3, 6, 6, 6, 6, 6}; + config.feature_size = {19, 10, 5, 3, 2, 1}; + config.min_scale = 0.2; + config.max_scale = 0.95; + config.aspect_rations = {{2}, {2, 3}, {2, 3}, {2, 3}, {2, 3}, {2, 3}}; + config.steps = {16, 32, 64, 100, 150, 300}; + config.prior_scaling = {0.1, 0.2}; + + std::vector> default_boxes = GetDefaultBoxes(config); + ASSERT_TRUE(default_boxes.size() == 1917); +} + +TEST_F(MindDataImageProcess, TestApplyNms) { + std::vector> all_boxes = {{1, 1, 2, 2}, {3, 3, 4, 4}, {5, 5, 6, 6}, {5, 5, 6, 6}}; + std::vector all_scores = {0.6, 0.5, 0.4, 0.9}; + std::vector keep = ApplyNms(all_boxes, all_scores, 0.5, 10); + ASSERT_TRUE(keep[0] == 3); + ASSERT_TRUE(keep[1] == 0); + ASSERT_TRUE(keep[2] == 1); +}