From 2b056b7a28a58d1f0479a349a9d4a8e33a7db72a Mon Sep 17 00:00:00 2001 From: fuzhiye Date: Wed, 23 Sep 2020 16:48:05 +0800 Subject: [PATCH] replace 8x8 block with 12x8 block in common conv --- mindspore/lite/nnacl/fp32/conv.c | 294 +-------- mindspore/lite/nnacl/fp32/conv.h | 29 +- mindspore/lite/nnacl/pack.c | 89 +-- mindspore/lite/nnacl/pack.h | 3 - mindspore/lite/nnacl/winograd_transform.c | 605 ------------------ mindspore/lite/nnacl/winograd_transform.h | 15 - mindspore/lite/nnacl/winograd_utils.c | 5 + .../runtime/kernel/arm/fp32/convolution.cc | 67 +- .../src/runtime/kernel/arm/fp32/convolution.h | 12 +- .../kernel/arm/fp32/convolution_3x3.cc | 245 ------- .../runtime/kernel/arm/fp32/convolution_3x3.h | 80 --- .../arm/fp32/convolution_slidewindow.cc | 201 ------ .../kernel/arm/fp32/convolution_slidewindow.h | 70 -- .../runtime/kernel/arm/common/pack_tests.cc | 29 - 14 files changed, 69 insertions(+), 1675 deletions(-) delete mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.cc delete mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.h delete mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.cc delete mode 100644 mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.h diff --git a/mindspore/lite/nnacl/fp32/conv.c b/mindspore/lite/nnacl/fp32/conv.c index 9db14f8b33..2778ec8f85 100644 --- a/mindspore/lite/nnacl/fp32/conv.c +++ b/mindspore/lite/nnacl/fp32/conv.c @@ -20,192 +20,9 @@ #include "nnacl/winograd_transform.h" #include "nnacl/fp32/matmul.h" -void SWBorderPixel(float *dst, const float *src, const float *weight, const float *bias, int height, int width, - int in_kh_step, int in_kw_step, int kernel_h, int kernel_w, int ic4, bool is_relu, bool is_relu6) { - for (int c = 0; c < C4NUM; c++) { - dst[c] = 0; - } - const float *weight_oc = weight; - for (int oc = 0; oc < C4NUM; ++oc) { - const float *weight_kh = weight_oc; - const float *src_kh = src; - for (int kh = 0; kh < height; kh++) { - const float *src_kw = src_kh; - const float *weight_kw = weight_kh; - for (int kw = 0; kw < width; kw++) { - const float *src_ic4 = src_kw; - const float *weight_ic4 = weight_kw; - for (int ic = 0; ic < ic4; ++ic) { - for (int c = 0; c < C4NUM; c++) { - dst[oc] += src_ic4[c] * weight_ic4[c]; - } - src_ic4 += C4NUM; - weight_ic4 += C4NUM; - } // ic4 loop - src_kw += in_kw_step; - weight_kw += ic4 * C4NUM; - } // kernel_w loop - src_kh += in_kh_step; - weight_kh += kernel_w * ic4 * C4NUM; - } // kernel_h loop - dst[oc] += bias[oc]; - dst[oc] = (is_relu) ? (MSMAX(0, dst[oc])) : (dst[oc]); - dst[oc] = (is_relu6) ? (MSMIN(6, MSMAX(0, dst[oc]))) : (dst[oc]); - weight_oc += kernel_h * kernel_w * ic4 * C4NUM; - } // oc loop -} - -void SWBorder(float *dst, const float *src, const float *weight, const float *bias, int top, int bottom, int left, - int right, const ConvParameter *conv_param, const SlidingWindowParam *sliding) { - int ic4 = sliding->ic4_channel_ / C4NUM; - bool relu = conv_param->act_type_ == ActType_Relu; - bool relu6 = conv_param->act_type_ == ActType_Relu6; - float *dst_h = dst + top * sliding->out_h_step_; - for (int oh = top; oh < bottom; oh++) { - int ih = oh * conv_param->stride_h_ - conv_param->pad_u_; - int start_kh = MSMAX(0, UP_DIV(-ih, conv_param->dilation_h_)); - int end_kh = MSMIN(conv_param->kernel_h_, UP_DIV(conv_param->input_h_ - ih, conv_param->dilation_h_)); - const float *src_h = src + ih * sliding->in_h_step_; - - float *dst_kernel = dst_h + left * sliding->block_channel_; - for (int ow = left; ow < right; ow++) { - int iw = ow * conv_param->stride_w_ - conv_param->pad_l_; - int start_kw = MSMAX(0, UP_DIV(-iw, conv_param->dilation_w_)); - int end_kw = MSMIN(conv_param->kernel_w_, UP_DIV(conv_param->input_w_ - iw, conv_param->dilation_w_)); - const float *src_w = src_h + iw * sliding->ic4_channel_; - - const float *src_kernel = src_w + start_kh * sliding->in_kh_step_ + start_kw * sliding->in_kw_step_; - const float *weight_kernel = weight + (start_kh * conv_param->kernel_w_ + start_kw) * sliding->ic4_channel_; - - SWBorderPixel(dst_kernel, src_kernel, weight_kernel, bias, end_kh - start_kh, end_kw - start_kw, - sliding->in_kh_step_, sliding->in_kw_step_, conv_param->kernel_h_, conv_param->kernel_w_, ic4, relu, - relu6); - - dst_kernel += sliding->block_channel_; - } // width loop - dst_h += sliding->out_h_step_; - } // height loop -} - -#ifndef ENABLE_ARM64 -void SWCenter(float *dst, const float *src, const float *weight, const float *bias, int height, int width, int kernel_h, - int kernel_w, int out_h_step, int block_channel, int ic4, int in_sh_step, int in_sw_step, int in_kh_step, - int in_kw_step, bool is_relu, bool is_relu6) { - float *dst_h = dst; - const float *src_h = src; - for (int oh = 0; oh < height; oh++) { - float *dst_w = dst_h; - const float *src_w = src_h; - for (int ow = 0; ow < width; ow++) { - const float *weight_oc = weight; - for (int c = 0; c < C4NUM; c++) { - dst_w[c] = 0; - } - - for (int oc = 0; oc < C4NUM; oc++) { - const float *weight_kh = weight_oc; - const float *src_kh = src_w; - for (int kh = 0; kh < kernel_h; kh++) { - const float *src_kw = src_kh; - const float *weight_kw = weight_kh; - for (int kw = 0; kw < kernel_w; kw++) { - const float *src_ic4 = src_kw; - const float *weight_ic4 = weight_kw; - for (int ic = 0; ic < ic4; ++ic) { - for (int c = 0; c < C4NUM; c++) { - dst_w[oc] += src_ic4[c] * weight_ic4[c]; - } - - src_ic4 += C4NUM; - weight_ic4 += C4NUM; - } // ic4 loop - src_kw += in_kw_step; - weight_kw += ic4 * C4NUM; - } // kernel_w loop - src_kh += in_kh_step; - weight_kh += kernel_w * ic4 * C4NUM; - } // kernel_h loop - // add biad relu - - dst_w[oc] += bias[oc]; - dst_w[oc] = (is_relu) ? (MSMAX(0, dst_w[oc])) : (dst_w[oc]); - dst_w[oc] = (is_relu6) ? (MSMIN(6, MSMAX(0, dst_w[oc]))) : (dst_w[oc]); - weight_oc += kernel_h * kernel_w * ic4 * C4NUM; - } // oc block - - dst_w += block_channel; - src_w += in_sw_step; - } // dst_width loop - dst_h += out_h_step; - src_h += in_sh_step; - } // dst_height loop -} -#endif - -// fp32 sliding window -void ConvSWFp32(const float *input_data, const float *packed_weight, const float *bias_data, float *tmp_out_block, - float *output_data, int task_id, ConvParameter *conv_param, SlidingWindowParam *slidingWindow_param) { - int ic4 = slidingWindow_param->ic4_channel_ / C4NUM; - int oc4_res = conv_param->output_channel_ % C4NUM; - bool relu = conv_param->act_type_ == ActType_Relu; - bool relu6 = conv_param->act_type_ == ActType_Relu6; - const float *src = input_data; - float *dst = NULL; - if (oc4_res == 0) { - dst = output_data; - } else { - dst = tmp_out_block; - } - - for (int b = 0; b < conv_param->output_batch_; b++) { - for (int oc = task_id; oc < slidingWindow_param->c_block_; oc += conv_param->thread_num_) { - const float *src_data = src; - float *dst_data = dst + oc * C4NUM; - const float *weight = packed_weight + oc * slidingWindow_param->kernel_step_; - const float *bias = bias_data + oc * C4NUM; - SWBorder(dst_data, src_data, weight, bias, 0, slidingWindow_param->top_, 0, conv_param->output_w_, conv_param, - slidingWindow_param); - SWBorder(dst_data, src_data, weight, bias, slidingWindow_param->bottom_, conv_param->output_h_, 0, - conv_param->output_w_, conv_param, slidingWindow_param); - SWBorder(dst_data, src_data, weight, bias, slidingWindow_param->top_, slidingWindow_param->bottom_, 0, - slidingWindow_param->left_, conv_param, slidingWindow_param); - SWBorder(dst_data, src_data, weight, bias, slidingWindow_param->top_, slidingWindow_param->bottom_, - slidingWindow_param->right_, conv_param->output_w_, conv_param, slidingWindow_param); - - if (slidingWindow_param->right_ > slidingWindow_param->left_ && - slidingWindow_param->bottom_ > slidingWindow_param->top_) { - int in_h_start = slidingWindow_param->top_ * conv_param->stride_h_ - conv_param->pad_u_; - int in_w_start = slidingWindow_param->left_ * conv_param->stride_w_ - conv_param->pad_l_; - const float *in_t = - src_data + in_h_start * slidingWindow_param->in_h_step_ + in_w_start * slidingWindow_param->ic4_channel_; - float *out_t = dst_data + slidingWindow_param->top_ * slidingWindow_param->out_h_step_ + - slidingWindow_param->left_ * slidingWindow_param->block_channel_; -#ifdef ENABLE_ARM64 - ConvSwFp32Center( - out_t, in_t, weight, bias, slidingWindow_param->bottom_ - slidingWindow_param->top_, - slidingWindow_param->right_ - slidingWindow_param->left_, conv_param->kernel_h_, conv_param->kernel_w_, - slidingWindow_param->out_h_step_ * sizeof(float), slidingWindow_param->block_channel_ * sizeof(float), ic4, - slidingWindow_param->in_sh_step_ * sizeof(float), slidingWindow_param->in_sw_step_ * sizeof(float), - slidingWindow_param->in_kh_step_ * sizeof(float), slidingWindow_param->in_kw_step_ * sizeof(float), relu, - relu6); -#else - SWCenter(out_t, in_t, weight, bias, slidingWindow_param->bottom_ - slidingWindow_param->top_, - slidingWindow_param->right_ - slidingWindow_param->left_, conv_param->kernel_h_, conv_param->kernel_w_, - slidingWindow_param->out_h_step_, slidingWindow_param->block_channel_, ic4, - slidingWindow_param->in_sh_step_, slidingWindow_param->in_sw_step_, slidingWindow_param->in_kh_step_, - slidingWindow_param->in_kw_step_, relu, relu6); -#endif - } - } // output C4 loop - src += slidingWindow_param->in_step_; - dst += slidingWindow_param->out_step_; - } // batch loop -} - // fp32 conv common void ConvFp32(float *input_data, float *packed_input, float *packed_weight, const float *bias_data, - float *tmp_out_block, float *output_data, int task_id, ConvParameter *conv_param, - GEMM_FUNC_FP32 gemm_func) { + float *col_major_input, float *output_data, int task_id, ConvParameter *conv_param) { int kernel_h = conv_param->kernel_h_; int kernel_w = conv_param->kernel_w_; int in_batch = conv_param->input_batch_; @@ -217,42 +34,38 @@ void ConvFp32(float *input_data, float *packed_input, float *packed_weight, cons int out_channel = conv_param->output_channel_; int thread_count = conv_param->thread_num_; int output_count = out_h * out_w; - int output_tile_count = UP_DIV(output_count, TILE_NUM); - int ic4 = UP_DIV(in_channel, C4NUM); +#ifdef ENABLE_ARM32 + const int cal_num = C4NUM; +#else + const int cal_num = C12NUM; +#endif + int output_tile_count = UP_DIV(output_count, cal_num); int kernel_plane = kernel_h * kernel_w; - int unit_size = kernel_plane * ic4 * C4NUM; - bool relu = conv_param->act_type_ == ActType_Relu; - bool relu6 = conv_param->act_type_ == ActType_Relu6; - - // we accumulate 4 channels per time for input blocks - int conv_depth = kernel_h * kernel_w; - // bytes from one output's i-th channel to the next output's i-th channel - // we write 32 bytes per st1 instruction, after which the pointer in register will step 32B forward - size_t output_offset = out_channel * sizeof(float); + int unit_size = kernel_plane * in_channel; + int deep = in_channel * kernel_plane; for (int b = 0; b < in_batch; b++) { int in_batch_offset = b * in_channel * in_h * in_w; int out_batch_offset = b * out_channel * out_h * out_w; for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) { - int start_index = thread_id * TILE_NUM; - int real_cal_num = (output_count - start_index) < TILE_NUM ? (output_count - start_index) : TILE_NUM; - float *gemm_input = packed_input + task_id * unit_size * TILE_NUM; - size_t packed_input_size = unit_size * TILE_NUM * sizeof(float); + int start_index = thread_id * cal_num; + int real_cal_num = (output_count - start_index) < cal_num ? (output_count - start_index) : cal_num; + float *gemm_input = packed_input + task_id * unit_size * cal_num; + float *col_major_gemm_input = col_major_input + task_id * unit_size * cal_num; + size_t packed_input_size = unit_size * cal_num * sizeof(float); memset(gemm_input, 0, packed_input_size); + memset(col_major_gemm_input, 0, packed_input_size); Im2ColPackUnitFp32(input_data + in_batch_offset, conv_param, gemm_input, real_cal_num, start_index); - int out_offset = thread_id * TILE_NUM * out_channel + out_batch_offset; - if (real_cal_num == TILE_NUM) { - float *gemm_output = output_data + out_offset; - gemm_func(gemm_output, gemm_input, packed_weight, bias_data, conv_depth, ic4, out_channel, output_offset, 0, 0, - relu, relu6); - } else { - // res part - float *tmp_out_ptr = tmp_out_block + task_id * TILE_NUM * out_channel; - gemm_func(tmp_out_ptr, gemm_input, packed_weight, bias_data, conv_depth, ic4, out_channel, output_offset, 0, 0, - relu, relu6); - memcpy(output_data + out_offset, tmp_out_ptr, real_cal_num * out_channel * sizeof(float)); - } + int out_offset = thread_id * cal_num * out_channel + out_batch_offset; + float *gemm_output = output_data + out_offset; +#ifdef ENABLE_ARM32 + RowMajor2Col4Major(gemm_input, col_major_gemm_input, cal_num, deep); +#else + RowMajor2Col12Major(gemm_input, col_major_gemm_input, cal_num, deep); +#endif + MatMulOpt(col_major_gemm_input, packed_weight, gemm_output, bias_data, conv_param->act_type_, deep, real_cal_num, + out_channel, out_channel, OutType_Nhwc); } } } @@ -321,62 +134,3 @@ void ConvWinogardFp32(float *input_data, float *trans_weight, const float *bias_ } } } - -// fp32 conv3x3 -void Conv3x3Fp32(float *input_data, float *transed_weight, const float *bias_data, TmpBufferAddress *buffer_list, - int task_id, ConvParameter *conv_param) { - int thread_count = conv_param->thread_num_; - int ic4 = UP_DIV(conv_param->input_channel_, C4NUM); - int output_channel = conv_param->output_channel_; - int oc4 = UP_DIV(output_channel, C4NUM); - int oc8 = UP_DIV(output_channel, C8NUM); - int out_w_block = UP_DIV(conv_param->output_w_, OUPUT_UNIT); - int out_h_block = UP_DIV(conv_param->output_h_, OUPUT_UNIT); - int output_count = out_w_block * out_h_block; -#ifdef ENABLE_ARM32 - const int tile_num = 4; -#else - const int tile_num = 12; -#endif - int output_tile_count = UP_DIV(output_count, tile_num); - const int input_unit_square = 4 * 4; - - float *tile_buffer = buffer_list[0]; - float *block_unit_buffer = buffer_list[1]; - float *tmp_dst_buffer = buffer_list[2]; - float *nc4hw4_out = buffer_list[3]; - float *col_buffer = buffer_list[4]; - int tile_buffer_offset = tile_num * input_unit_square * ic4 * C4NUM; - int block_unit_buffer_offset = input_unit_square * C4NUM; - int tmp_dst_buffer_offset = tile_num * input_unit_square * oc8 * C8NUM; - int col_buffer_offset = tile_num * ic4 * C4NUM; - - int input_batch = conv_param->input_batch_; - for (int batch = 0; batch < input_batch; batch++) { - int in_batch_offset = batch * conv_param->input_channel_ * conv_param->input_h_ * conv_param->input_w_; - int nc4hw4_buffer_offset = batch * oc4 * C4NUM * conv_param->output_h_ * conv_param->output_w_; - - for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) { - int start_index = thread_id * tile_num; - int real_cal_num = (output_count - start_index) < tile_num ? (output_count - start_index) : tile_num; - Conv3x3Fp32InputTransform(input_data + in_batch_offset, tile_buffer + task_id * tile_buffer_offset, - block_unit_buffer + task_id * block_unit_buffer_offset, start_index, real_cal_num, - out_w_block, conv_param); - - float *src_ptr = tile_buffer + task_id * tile_buffer_offset; - float *tmp_col_ptr = col_buffer + task_id * col_buffer_offset; - float *dst_ptr = tmp_dst_buffer + task_id * tmp_dst_buffer_offset; - for (int i = 0; i < input_unit_square; ++i) { -#ifdef ENABLE_ARM32 - RowMajor2Col4Major(src_ptr + i * C4NUM * ic4 * C4NUM, tmp_col_ptr, C4NUM, ic4 * C4NUM); -#else - RowMajor2Col12Major(src_ptr + i * C12NUM * ic4 * C4NUM, tmp_col_ptr, C12NUM, ic4 * C4NUM); -#endif - MatMulOpt(tmp_col_ptr, transed_weight + i * ic4 * C4NUM * oc8 * C8NUM, dst_ptr + i * C8NUM, NULL, 0, - ic4 * C4NUM, real_cal_num, oc8 * C8NUM, input_unit_square, 2); - } - Conv3x3Fp32OutputTransform(dst_ptr, nc4hw4_out + nc4hw4_buffer_offset, bias_data, start_index, real_cal_num, - out_w_block, conv_param); - } - } -} diff --git a/mindspore/lite/nnacl/fp32/conv.h b/mindspore/lite/nnacl/fp32/conv.h index 5aeba9fb1a..274456e557 100644 --- a/mindspore/lite/nnacl/fp32/conv.h +++ b/mindspore/lite/nnacl/fp32/conv.h @@ -28,46 +28,19 @@ #include "nnacl/fp32/conv_depthwise.h" typedef float *TmpBufferAddress; -typedef float *Matrices; -typedef void (*GEMM_FUNC_FP32)(float *output, const float *input, const float *weight, const float *bias, size_t step, - size_t ic4, size_t output_channel, size_t offset, size_t mode, size_t writeC4, - size_t relu, size_t relu6); #ifdef __cplusplus extern "C" { #endif -void SWBorder(float *dst, const float *src, const float *weight, const float *bias, int top, int bottom, int left, - int right, const ConvParameter *conv_param, const SlidingWindowParam *sliding); - -void SWCenter(float *dst, const float *src, const float *weight, const float *bias, int height, int width, int kernel_h, - int kernel_w, int out_h_step, int block_channel, int ic4, int in_sh_step, int in_sw_step, int in_kh_step, - int in_kw_step, bool is_relu, bool is_relu6); - -// fp32 sliding window -void ConvSWFp32(const float *input_data, const float *packed_weight, const float *bias_data, float *tmp_out_block, - float *output_data, int task_id, ConvParameter *conv_param, SlidingWindowParam *slidingWindow_param); // fp32 convolution common (im2col+gemm) void ConvFp32(float *input_data, float *packed_input, float *packed_weight, const float *bias_data, - float *tmp_out_block, float *output_data, int task_id, ConvParameter *conv_param, - GEMM_FUNC_FP32 gemm_func); + float *col_major_input, float *output_data, int task_id, ConvParameter *conv_param); // fp32 convolution winograd void ConvWinogardFp32(float *input_data, float *trans_weight, const float *bias_data, float *output_data, TmpBufferAddress *buffer_list, int task_id, ConvParameter *conv_param, InputTransFunc in_func, OutputTransFunc out_func); - -void UnPackWinogradOutput(const float *src, float *dst, int batch, int height, int width, int channel, int output_unit); - -void UnPackWinogradReluOutput(const float *src, float *dst, int batch, int height, int width, int channel, - int output_unit); - -void UnPackWinogradRelu6Output(const float *src, float *dst, int batch, int height, int width, int channel, - int output_unit); - -// fp32 conv3x3 -void Conv3x3Fp32(float *input_data, float *transed_weight, const float *bias_data, TmpBufferAddress *buffer_list, - int task_id, ConvParameter *conv_param); #ifdef __cplusplus } #endif diff --git a/mindspore/lite/nnacl/pack.c b/mindspore/lite/nnacl/pack.c index 04fd260a6d..70db272187 100644 --- a/mindspore/lite/nnacl/pack.c +++ b/mindspore/lite/nnacl/pack.c @@ -18,50 +18,6 @@ #include #include -void PackWeightFp32(float *weight_data, ConvParameter *conv_param, float *packed_weight, int oc_block, - int oc_block_num) { - // original weight format : ohwi - if (oc_block_num == 0) { - return; - } - int kernel_h = conv_param->kernel_h_; - int kernel_w = conv_param->kernel_w_; - int in_channel = conv_param->input_channel_; - int out_channel = conv_param->output_channel_; - int ic4 = UP_DIV(in_channel, C4NUM); - int kernel_plane = kernel_h * kernel_w; - int pack_weight_size = oc_block * oc_block_num * ic4 * C4NUM * kernel_plane; - - int unit_size = oc_block * C4NUM; - const int block_size = pack_weight_size / oc_block_num; - - for (int m = 0; m < kernel_plane; m++) { - int kernel_plane_stride = m * in_channel; - int packed_kernel_plane_stride = m * unit_size * ic4; - for (int i = 0; i < ic4; i++) { - int channel_block_stride = kernel_plane_stride + i * C4NUM; - int packed_channel_block_size = packed_kernel_plane_stride + i * unit_size; - int ic_remainder = in_channel - i * C4NUM; - int real_ic_num = ic_remainder < C4NUM ? ic_remainder : C4NUM; - for (int h = 0; h < real_ic_num; h++) { - int block_stride = channel_block_stride + h; - int packed_block_stride = packed_channel_block_size + h * oc_block; - for (int j = 0; j < oc_block_num; j++) { - int kernel_block_stride = block_stride + j * oc_block * kernel_plane * in_channel; - int packed_kernel_block_size = packed_block_stride + j * block_size; - int oc_remainder = out_channel - j * oc_block; - int real_oc_num = oc_remainder < oc_block ? oc_remainder : oc_block; - for (int k = 0; k < real_oc_num; k++) { - float *origin_data_ptr = weight_data + kernel_block_stride + k * kernel_plane * in_channel; - float *packed_data_ptr = packed_weight + packed_kernel_block_size + k; - *packed_data_ptr = *origin_data_ptr; - } - } // kernel block loop - } // inchannel block loop - } // channel block loop - } // kernel plane loop -} - void PackWeightKHWToHWKFp32(const void *src, void *dst, int plane, int channel) { return PackNCHWToNHWCFp32(src, dst, 1, plane, channel); } @@ -301,6 +257,7 @@ void Im2ColPackUnitFp32(const float *input_data, ConvParameter *conv_param, floa // input format : nhwc int kernel_h = conv_param->kernel_h_; int kernel_w = conv_param->kernel_w_; + int kernel_plane = kernel_h * kernel_w; int stride_h = conv_param->stride_h_; int stride_w = conv_param->stride_w_; int pad_h = conv_param->pad_u_; @@ -311,8 +268,6 @@ void Im2ColPackUnitFp32(const float *input_data, ConvParameter *conv_param, floa int in_h = conv_param->input_h_; int in_w = conv_param->input_w_; int out_w = conv_param->output_w_; - int ic4_minus = in_channel / C4NUM; - int ic4 = UP_DIV(in_channel, C4NUM); for (int i = 0; i < real_cal_num; i++) { int block_start = block_index + i; @@ -323,31 +278,25 @@ void Im2ColPackUnitFp32(const float *input_data, ConvParameter *conv_param, floa int kh_e = MSMIN(kernel_h, UP_DIV(in_h - input_h, dilation_h)); int kw_s = MSMAX(0, UP_DIV(-input_w, dilation_w)); int kw_e = MSMIN(kernel_w, UP_DIV(in_w - input_w, dilation_w)); - for (int j = kh_s; j < kh_e; j++) { - int input_y_stride = j * dilation_h * in_w * in_channel + input_stride; - for (int n = kw_s; n < kw_e; n++) { - int input_x_stride = input_y_stride + n * dilation_w * in_channel; - int input_plane_offset = (j * kernel_w + n) * C8NUM * C4NUM * ic4 + i * C4NUM; - for (int m = 0; m < ic4_minus; m++) { - int channel_block_stride = input_x_stride + m * C4NUM; - int channel_block_offset = input_plane_offset + m * C8NUM * C4NUM; -#ifdef ENABLE_NEON - vst1q_f32(packed_input + channel_block_offset, vld1q_f32(input_data + channel_block_stride)); -#else - for (int k = 0; k < C4NUM; ++k) { - (packed_input + channel_block_offset)[k] = (input_data + channel_block_stride)[k]; - } -#endif - } // channel_block loop - int ic_res = conv_param->input_channel_ - ic4_minus * C4NUM; - for (int l = 0; l < ic_res; ++l) { - int channel_block_stride = input_x_stride + ic4_minus * C4NUM + l; - int channel_block_offset = input_plane_offset + ic4_minus * C8NUM * C4NUM + l; - packed_input[channel_block_offset] = input_data[channel_block_stride]; + if (dilation_w == 1 && dilation_h == 1) { + for (int j = kh_s; j < kh_e; j++) { + int input_y_stride = j * in_w * in_channel + input_stride; + int input_x_stride = input_y_stride + kw_s * in_channel; + int input_plane_offset = (j * kernel_w + kw_s) * in_channel + i * in_channel * kernel_plane; + memcpy(packed_input + input_plane_offset, input_data + input_x_stride, + (kw_e - kw_s) * in_channel * sizeof(float)); + } // kernel_h loop + } else { + for (int j = kh_s; j < kh_e; j++) { + int input_y_stride = j * dilation_h * in_w * in_channel + input_stride; + for (int k = kw_s; k < kw_e; ++k) { + int input_x_stride = input_y_stride + k * dilation_w * in_channel; + int input_plane_offset = (j * kernel_w + k) * in_channel + i * in_channel * kernel_plane; + memcpy(packed_input + input_plane_offset, input_data + input_x_stride, in_channel * sizeof(float)); } - } // kernel_w loop - } // kernel_h loop - } // tile num loop + } // kernel_h loop + } + } // tile num loop } void Im2ColPackUnitInt8(const int8_t *input_data, int8_t *packed_input, int real_cal_num, int block_index, diff --git a/mindspore/lite/nnacl/pack.h b/mindspore/lite/nnacl/pack.h index 9d30e426f2..a72cac90b6 100644 --- a/mindspore/lite/nnacl/pack.h +++ b/mindspore/lite/nnacl/pack.h @@ -51,9 +51,6 @@ void MatrixPack(const float *src, float *dst, int row, int ic4, int stride); void PackInputToC8Int8(const int8_t *input_data, int16_t *packed_input, ConvParameter *conv_param); -void PackWeightFp32(float *weight_data, ConvParameter *conv_param, float *packed_weight, int oc_block, - int oc_block_num); - void PackWeightKHWToHWKFp32(const void *src, void *dst, int plane, int channel); void PackWeightInt8(int8_t *weight_data, ConvParameter *conv_param, int8_t *packed_weight, int32_t *weight_sum); diff --git a/mindspore/lite/nnacl/winograd_transform.c b/mindspore/lite/nnacl/winograd_transform.c index c8a7eb9851..6b185395c0 100644 --- a/mindspore/lite/nnacl/winograd_transform.c +++ b/mindspore/lite/nnacl/winograd_transform.c @@ -142,611 +142,6 @@ void WinogradOutputTransform(const float *gemm_out, float *out_data, const float } } -// fp32 conv3x3 -void Conv3x3Fp32InputUnit(const float *tmp_data, float *trans_input_data, size_t step) { -#ifdef ENABLE_ARM - float32x4_t d00 = vld1q_f32(tmp_data); - float32x4_t d01 = vld1q_f32(tmp_data + 4); - float32x4_t d02 = vld1q_f32(tmp_data + 2 * 4); - float32x4_t d03 = vld1q_f32(tmp_data + 3 * 4); - - float32x4_t d10 = vld1q_f32(tmp_data + 4 * 4); - float32x4_t d11 = vld1q_f32(tmp_data + 5 * 4); - float32x4_t d12 = vld1q_f32(tmp_data + 6 * 4); - float32x4_t d13 = vld1q_f32(tmp_data + 7 * 4); - - float32x4_t d20 = vld1q_f32(tmp_data + 8 * 4); - float32x4_t d21 = vld1q_f32(tmp_data + 9 * 4); - float32x4_t d22 = vld1q_f32(tmp_data + 10 * 4); - float32x4_t d23 = vld1q_f32(tmp_data + 11 * 4); - - float32x4_t d30 = vld1q_f32(tmp_data + 12 * 4); - float32x4_t d31 = vld1q_f32(tmp_data + 13 * 4); - float32x4_t d32 = vld1q_f32(tmp_data + 14 * 4); - float32x4_t d33 = vld1q_f32(tmp_data + 15 * 4); - - float32x4_t t00 = vsubq_f32(d00, d20); - float32x4_t t01 = vsubq_f32(d01, d21); - float32x4_t t02 = vsubq_f32(d02, d22); - float32x4_t t03 = vsubq_f32(d03, d23); - - float32x4_t t10 = vaddq_f32(d10, d20); - float32x4_t t11 = vaddq_f32(d11, d21); - float32x4_t t12 = vaddq_f32(d12, d22); - float32x4_t t13 = vaddq_f32(d13, d23); - - float32x4_t t20 = vsubq_f32(d20, d10); - float32x4_t t21 = vsubq_f32(d21, d11); - float32x4_t t22 = vsubq_f32(d22, d12); - float32x4_t t23 = vsubq_f32(d23, d13); - - float32x4_t t30 = vsubq_f32(d10, d30); - float32x4_t t31 = vsubq_f32(d11, d31); - float32x4_t t32 = vsubq_f32(d12, d32); - float32x4_t t33 = vsubq_f32(d13, d33); - - float32x4_t m00 = vsubq_f32(t00, t02); - float32x4_t m01 = vaddq_f32(t01, t02); - float32x4_t m02 = vsubq_f32(t02, t01); - float32x4_t m03 = vsubq_f32(t01, t03); - - float32x4_t m10 = vsubq_f32(t10, t12); - float32x4_t m11 = vaddq_f32(t11, t12); - float32x4_t m12 = vsubq_f32(t12, t11); - float32x4_t m13 = vsubq_f32(t11, t13); - - float32x4_t m20 = vsubq_f32(t20, t22); - float32x4_t m21 = vaddq_f32(t21, t22); - float32x4_t m22 = vsubq_f32(t22, t21); - float32x4_t m23 = vsubq_f32(t21, t23); - - float32x4_t m30 = vsubq_f32(t30, t32); - float32x4_t m31 = vaddq_f32(t31, t32); - float32x4_t m32 = vsubq_f32(t32, t31); - float32x4_t m33 = vsubq_f32(t31, t33); - - vst1q_f32(trans_input_data, m00); - vst1q_f32(trans_input_data + step, m01); - vst1q_f32(trans_input_data + 2 * step, m02); - vst1q_f32(trans_input_data + 3 * step, m03); - - vst1q_f32(trans_input_data + 4 * step, m10); - vst1q_f32(trans_input_data + 5 * step, m11); - vst1q_f32(trans_input_data + 6 * step, m12); - vst1q_f32(trans_input_data + 7 * step, m13); - - vst1q_f32(trans_input_data + 8 * step, m20); - vst1q_f32(trans_input_data + 9 * step, m21); - vst1q_f32(trans_input_data + 10 * step, m22); - vst1q_f32(trans_input_data + 11 * step, m23); - - vst1q_f32(trans_input_data + 12 * step, m30); - vst1q_f32(trans_input_data + 13 * step, m31); - vst1q_f32(trans_input_data + 14 * step, m32); - vst1q_f32(trans_input_data + 15 * step, m33); -#else - for (int i = 0; i < C4NUM; i++) { - const float *local_ptr = tmp_data + i; - float d00 = local_ptr[0]; - float d01 = (local_ptr + C4NUM)[0]; - float d02 = (local_ptr + 2 * C4NUM)[0]; - float d03 = (local_ptr + 3 * C4NUM)[0]; - - float d10 = (local_ptr + 4 * C4NUM)[0]; - float d11 = (local_ptr + 5 * C4NUM)[0]; - float d12 = (local_ptr + 6 * C4NUM)[0]; - float d13 = (local_ptr + 7 * C4NUM)[0]; - - float d20 = (local_ptr + 8 * C4NUM)[0]; - float d21 = (local_ptr + 9 * C4NUM)[0]; - float d22 = (local_ptr + 10 * C4NUM)[0]; - float d23 = (local_ptr + 11 * C4NUM)[0]; - - float d30 = (local_ptr + 12 * C4NUM)[0]; - float d31 = (local_ptr + 13 * C4NUM)[0]; - float d32 = (local_ptr + 14 * C4NUM)[0]; - float d33 = (local_ptr + 15 * C4NUM)[0]; - - float t00 = d00 - d20; - float t01 = d01 - d21; - float t02 = d02 - d22; - float t03 = d03 - d23; - - float t10 = d10 + d20; - float t11 = d11 + d21; - float t12 = d12 + d22; - float t13 = d13 + d23; - - float t20 = d20 - d10; - float t21 = d21 - d11; - float t22 = d22 - d12; - float t23 = d23 - d13; - - float t30 = d10 - d30; - float t31 = d11 - d31; - float t32 = d12 - d32; - float t33 = d13 - d33; - - float m00 = t00 - t02; - float m01 = t01 + t02; - float m02 = t02 - t01; - float m03 = t01 - t03; - - float m10 = t10 - t12; - float m11 = t11 + t12; - float m12 = t12 - t11; - float m13 = t11 - t13; - - float m20 = t20 - t22; - float m21 = t21 + t22; - float m22 = t22 - t21; - float m23 = t21 - t23; - - float m30 = t30 - t32; - float m31 = t31 + t32; - float m32 = t32 - t31; - float m33 = t31 - t33; - - (trans_input_data + i)[0] = m00; - (trans_input_data + i + step)[0] = m01; - (trans_input_data + i + 2 * step)[0] = m02; - (trans_input_data + i + 3 * step)[0] = m03; - - (trans_input_data + i + 4 * step)[0] = m10; - (trans_input_data + i + 5 * step)[0] = m11; - (trans_input_data + i + 6 * step)[0] = m12; - (trans_input_data + i + 7 * step)[0] = m13; - - (trans_input_data + i + 8 * step)[0] = m20; - (trans_input_data + i + 9 * step)[0] = m21; - (trans_input_data + i + 10 * step)[0] = m22; - (trans_input_data + i + 11 * step)[0] = m23; - - (trans_input_data + i + 12 * step)[0] = m30; - (trans_input_data + i + 13 * step)[0] = m31; - (trans_input_data + i + 14 * step)[0] = m32; - (trans_input_data + i + 15 * step)[0] = m33; - } -#endif -} - -void Conv3x3Fp32InputTransform(const float *input_data, float *trans_input, float *tmp_data, int start_index, - int real_cal_num, int out_w_block, ConvParameter *conv_param) { - // input data format : nhwc - int input_channel = conv_param->input_channel_; - int input_width = conv_param->input_w_; - int input_height = conv_param->input_h_; - int pad_w = conv_param->pad_l_; - int pad_h = conv_param->pad_u_; - int ic4 = UP_DIV(input_channel, C4NUM); - const int input_unit = 4; - if (out_w_block == 0) { - return; - } - for (int cal_id = 0; cal_id < real_cal_num; cal_id++) { - int x_id = start_index + cal_id; - int origin_x = (x_id % out_w_block) * OUPUT_UNIT - pad_w; - int origin_y = (x_id / out_w_block) * OUPUT_UNIT - pad_h; - int real_x_start = origin_x > 0 ? 0 : -origin_x; - int real_x_end = (origin_x + input_unit) < input_width ? input_unit : (input_width - origin_x); - int real_y_start = origin_y > 0 ? 0 : -origin_y; - int real_y_end = (origin_y + input_unit) < input_height ? input_unit : (input_height - origin_y); - - int src_plane_offset = input_channel * (origin_y * input_width + origin_x); - int dst_plane_offset = cal_id * C4NUM * ic4; - for (int ic = 0; ic < ic4; ic++) { - // clear tmp buffer - memset(tmp_data, 0, input_unit * input_unit * C4NUM * sizeof(float)); - int real_c = input_channel - ic * C4NUM; - real_c = real_c > C4NUM ? C4NUM : real_c; - - // get real input block with padding - int src_ic4_offset = src_plane_offset + ic * C4NUM; - if (real_c == C4NUM) { - for (int interval = real_y_start; interval < real_y_end; interval++) { - int src_y_offset = src_ic4_offset + (interval * input_width + real_x_start) * input_channel; - int dst_y_offset = interval * input_unit * C4NUM + real_x_start * C4NUM; - for (int j = 0; j < (real_x_end - real_x_start); j++) { - int src_x_offset = src_y_offset + j * input_channel; - int dst_x_offset = dst_y_offset + j * C4NUM; - float *src_addr = (float *)(input_data) + src_x_offset; - float *dst_addr = tmp_data + dst_x_offset; -#ifdef ENABLE_NEON - vst1q_f32(dst_addr, vld1q_f32(src_addr)); -#else - for (int k = 0; k < C4NUM; k++) { - dst_addr[k] = src_addr[k]; - } -#endif - } - } - } else { - for (int interval = real_y_start; interval < real_y_end; interval++) { - int src_y_offset = src_ic4_offset + (interval * input_width + real_x_start) * input_channel; - int dst_y_offset = interval * input_unit * C4NUM + real_x_start * C4NUM; - for (int j = 0; j < (real_x_end - real_x_start); j++) { - int src_x_offset = src_y_offset + j * input_channel; - int dst_x_offset = dst_y_offset + j * C4NUM; - float *src_addr = (float *)(input_data) + src_x_offset; - float *dst_addr = tmp_data + dst_x_offset; - for (int k = 0; k < real_c; k++) { - dst_addr[k] = src_addr[k]; - } - } - } - } - - // input transform -#ifdef ENABLE_ARM32 - const int tile_num = 4; -#else - const int tile_num = 12; -#endif - int dst_ic4_offset = dst_plane_offset + ic * C4NUM; - size_t dst_step = tile_num * ic4 * C4NUM; - float *trans_input_ptr = trans_input + dst_ic4_offset; - Conv3x3Fp32InputUnit(tmp_data, trans_input_ptr, dst_step); - } - } -} - -void Conv3x3Fp32FilterTransform(float *weight_data, float *trans_weight, int iC4, int output_channel, int kernel_plane, - int oc_block) { - if (oc_block == 0) { - return; - } - int oc_plane_block = UP_DIV(output_channel, oc_block); - int dst_step = iC4 * C4NUM * oc_block * oc_plane_block; - for (int o = 0; o < output_channel; o++) { - int oc_block_num = o / oc_block; - int oc_block_rem = o % oc_block; - int src_oc_offset = o * iC4 * C4NUM * kernel_plane; - int dst_oc_offset = oc_block_num * oc_block * iC4 * C4NUM + oc_block_rem; - for (int i = 0; i < iC4; i++) { - float *src_ic4_ptr = weight_data + src_oc_offset + i * kernel_plane * C4NUM; - float *dst_ic4_ptr = trans_weight + dst_oc_offset + i * oc_block * C4NUM; -#ifdef ENABLE_ARM - float32x4_t g00 = vld1q_f32(src_ic4_ptr); - float32x4_t g01 = vld1q_f32(src_ic4_ptr + 4); - float32x4_t g02 = vld1q_f32(src_ic4_ptr + 2 * 4); - float32x4_t g10 = vld1q_f32(src_ic4_ptr + 3 * 4); - float32x4_t g11 = vld1q_f32(src_ic4_ptr + 4 * 4); - float32x4_t g12 = vld1q_f32(src_ic4_ptr + 5 * 4); - float32x4_t g20 = vld1q_f32(src_ic4_ptr + 6 * 4); - float32x4_t g21 = vld1q_f32(src_ic4_ptr + 7 * 4); - float32x4_t g22 = vld1q_f32(src_ic4_ptr + 8 * 4); - - float32x4_t dst00 = g00; - float32x4_t dst01 = g01; - float32x4_t dst02 = g02; - - float32x4_t dst10 = vaddq_f32(vaddq_f32(g00, g10), g20); - dst10 = vmulq_n_f32(dst10, 0.5); - float32x4_t dst11 = vaddq_f32(vaddq_f32(g01, g11), g21); - dst11 = vmulq_n_f32(dst11, 0.5); - float32x4_t dst12 = vaddq_f32(vaddq_f32(g02, g12), g22); - dst12 = vmulq_n_f32(dst12, 0.5); - - float32x4_t dst20 = vaddq_f32(vsubq_f32(g00, g10), g20); - dst20 = vmulq_n_f32(dst20, 0.5); - float32x4_t dst21 = vaddq_f32(vsubq_f32(g01, g11), g21); - dst21 = vmulq_n_f32(dst21, 0.5); - float32x4_t dst22 = vaddq_f32(vsubq_f32(g02, g12), g22); - dst22 = vmulq_n_f32(dst22, 0.5); - - float32x4_t dst30 = g20; - float32x4_t dst31 = g21; - float32x4_t dst32 = g22; - - float32x4_t m00 = dst00; - float32x4_t m01 = vaddq_f32(vaddq_f32(dst00, dst01), dst02); - m01 = vmulq_n_f32(m01, 0.5); - float32x4_t m02 = vaddq_f32(vsubq_f32(dst00, dst01), dst02); - m02 = vmulq_n_f32(m02, 0.5); - float32x4_t m03 = dst02; - - float32x4_t m10 = dst10; - float32x4_t m11 = vaddq_f32(vaddq_f32(dst10, dst11), dst12); - m11 = vmulq_n_f32(m11, 0.5); - float32x4_t m12 = vaddq_f32(vsubq_f32(dst10, dst11), dst12); - m12 = vmulq_n_f32(m12, 0.5); - float32x4_t m13 = dst12; - - float32x4_t m20 = dst20; - float32x4_t m21 = vaddq_f32(vaddq_f32(dst20, dst21), dst22); - m21 = vmulq_n_f32(m21, 0.5); - float32x4_t m22 = vaddq_f32(vsubq_f32(dst20, dst21), dst22); - m22 = vmulq_n_f32(m22, 0.5); - float32x4_t m23 = dst22; - - float32x4_t m30 = dst30; - float32x4_t m31 = vaddq_f32(vaddq_f32(dst30, dst31), dst32); - m31 = vmulq_n_f32(m31, 0.5); - float32x4_t m32 = vaddq_f32(vsubq_f32(dst30, dst31), dst32); - m32 = vmulq_n_f32(m32, 0.5); - float32x4_t m33 = dst32; - - dst_ic4_ptr[0] = m00[0]; - dst_ic4_ptr[8] = m00[1]; - dst_ic4_ptr[16] = m00[2]; - dst_ic4_ptr[24] = m00[3]; - - dst_ic4_ptr[0 + dst_step] = m01[0]; - dst_ic4_ptr[8 + dst_step] = m01[1]; - dst_ic4_ptr[16 + dst_step] = m01[2]; - dst_ic4_ptr[24 + dst_step] = m01[3]; - - dst_ic4_ptr[0 + 2 * dst_step] = m02[0]; - dst_ic4_ptr[8 + 2 * dst_step] = m02[1]; - dst_ic4_ptr[16 + 2 * dst_step] = m02[2]; - dst_ic4_ptr[24 + 2 * dst_step] = m02[3]; - - dst_ic4_ptr[0 + 3 * dst_step] = m03[0]; - dst_ic4_ptr[8 + 3 * dst_step] = m03[1]; - dst_ic4_ptr[16 + 3 * dst_step] = m03[2]; - dst_ic4_ptr[24 + 3 * dst_step] = m03[3]; - - dst_ic4_ptr[0 + 4 * dst_step] = m10[0]; - dst_ic4_ptr[8 + 4 * dst_step] = m10[1]; - dst_ic4_ptr[16 + 4 * dst_step] = m10[2]; - dst_ic4_ptr[24 + 4 * dst_step] = m10[3]; - - dst_ic4_ptr[0 + 5 * dst_step] = m11[0]; - dst_ic4_ptr[8 + 5 * dst_step] = m11[1]; - dst_ic4_ptr[16 + 5 * dst_step] = m11[2]; - dst_ic4_ptr[24 + 5 * dst_step] = m11[3]; - - dst_ic4_ptr[0 + 6 * dst_step] = m12[0]; - dst_ic4_ptr[8 + 6 * dst_step] = m12[1]; - dst_ic4_ptr[16 + 6 * dst_step] = m12[2]; - dst_ic4_ptr[24 + 6 * dst_step] = m12[3]; - - dst_ic4_ptr[0 + 7 * dst_step] = m13[0]; - dst_ic4_ptr[8 + 7 * dst_step] = m13[1]; - dst_ic4_ptr[16 + 7 * dst_step] = m13[2]; - dst_ic4_ptr[24 + 7 * dst_step] = m13[3]; - - dst_ic4_ptr[0 + 8 * dst_step] = m20[0]; - dst_ic4_ptr[8 + 8 * dst_step] = m20[1]; - dst_ic4_ptr[16 + 8 * dst_step] = m20[2]; - dst_ic4_ptr[24 + 8 * dst_step] = m20[3]; - - dst_ic4_ptr[0 + 9 * dst_step] = m21[0]; - dst_ic4_ptr[8 + 9 * dst_step] = m21[1]; - dst_ic4_ptr[16 + 9 * dst_step] = m21[2]; - dst_ic4_ptr[24 + 9 * dst_step] = m21[3]; - - dst_ic4_ptr[0 + 10 * dst_step] = m22[0]; - dst_ic4_ptr[8 + 10 * dst_step] = m22[1]; - dst_ic4_ptr[16 + 10 * dst_step] = m22[2]; - dst_ic4_ptr[24 + 10 * dst_step] = m22[3]; - - dst_ic4_ptr[0 + 11 * dst_step] = m23[0]; - dst_ic4_ptr[8 + 11 * dst_step] = m23[1]; - dst_ic4_ptr[16 + 11 * dst_step] = m23[2]; - dst_ic4_ptr[24 + 11 * dst_step] = m23[3]; - - dst_ic4_ptr[0 + 12 * dst_step] = m30[0]; - dst_ic4_ptr[8 + 12 * dst_step] = m30[1]; - dst_ic4_ptr[16 + 12 * dst_step] = m30[2]; - dst_ic4_ptr[24 + 12 * dst_step] = m30[3]; - - dst_ic4_ptr[0 + 13 * dst_step] = m31[0]; - dst_ic4_ptr[8 + 13 * dst_step] = m31[1]; - dst_ic4_ptr[16 + 13 * dst_step] = m31[2]; - dst_ic4_ptr[24 + 13 * dst_step] = m31[3]; - - dst_ic4_ptr[0 + 14 * dst_step] = m32[0]; - dst_ic4_ptr[8 + 14 * dst_step] = m32[1]; - dst_ic4_ptr[16 + 14 * dst_step] = m32[2]; - dst_ic4_ptr[24 + 14 * dst_step] = m32[3]; - - dst_ic4_ptr[0 + 15 * dst_step] = m33[0]; - dst_ic4_ptr[8 + 15 * dst_step] = m33[1]; - dst_ic4_ptr[16 + 15 * dst_step] = m33[2]; - dst_ic4_ptr[24 + 15 * dst_step] = m33[3]; -#else - for (int j = 0; j < C4NUM; j++) { - float *local_ptr = src_ic4_ptr + j; - float dst00 = local_ptr[0]; - float dst01 = (local_ptr + 4)[0]; - float dst02 = (local_ptr + 8)[0]; - - const float dst10 = 0.5f * local_ptr[0] + 0.5f * (local_ptr + 12)[0] + 0.5f * (local_ptr + 24)[0]; - const float dst11 = 0.5f * (local_ptr + 4)[0] + 0.5f * (local_ptr + 16)[0] + 0.5f * (local_ptr + 28)[0]; - const float dst12 = 0.5f * (local_ptr + 8)[0] + 0.5f * (local_ptr + 20)[0] + 0.5f * (local_ptr + 32)[0]; - - const float dst20 = 0.5f * local_ptr[0] - 0.5f * (local_ptr + 12)[0] + 0.5f * (local_ptr + 24)[0]; - const float dst21 = 0.5f * (local_ptr + 4)[0] - 0.5f * (local_ptr + 16)[0] + 0.5f * (local_ptr + 28)[0]; - const float dst22 = 0.5f * (local_ptr + 8)[0] - 0.5f * (local_ptr + 20)[0] + 0.5f * (local_ptr + 32)[0]; - - float dst30 = (local_ptr + 24)[0]; - float dst31 = (local_ptr + 28)[0]; - float dst32 = (local_ptr + 32)[0]; - - float m00 = dst00; - const float m01 = 0.5f * dst00 + 0.5f * dst01 + 0.5f * dst02; - const float m02 = 0.5f * dst00 - 0.5f * dst01 + 0.5f * dst02; - float m03 = dst02; - - float m10 = dst10; - const float m11 = 0.5f * dst10 + 0.5f * dst11 + 0.5f * dst12; - const float m12 = 0.5f * dst10 - 0.5f * dst11 + 0.5f * dst12; - float m13 = dst12; - - float m20 = dst20; - const float m21 = 0.5f * dst20 + 0.5f * dst21 + 0.5f * dst22; - const float m22 = 0.5f * dst20 - 0.5f * dst21 + 0.5f * dst22; - float m23 = dst22; - - float m30 = dst30; - const float m31 = 0.5f * dst30 + 0.5f * dst31 + 0.5f * dst32; - const float m32 = 0.5f * dst30 - 0.5f * dst31 + 0.5f * dst32; - float m33 = dst32; - - *(dst_ic4_ptr + j * 8) = m00; - *(dst_ic4_ptr + j * 8 + dst_step) = m01; - *(dst_ic4_ptr + j * 8 + 2 * dst_step) = m02; - *(dst_ic4_ptr + j * 8 + 3 * dst_step) = m03; - - *(dst_ic4_ptr + j * 8 + 4 * dst_step) = m10; - *(dst_ic4_ptr + j * 8 + 5 * dst_step) = m11; - *(dst_ic4_ptr + j * 8 + 6 * dst_step) = m12; - *(dst_ic4_ptr + j * 8 + 7 * dst_step) = m13; - - *(dst_ic4_ptr + j * 8 + 8 * dst_step) = m20; - *(dst_ic4_ptr + j * 8 + 9 * dst_step) = m21; - *(dst_ic4_ptr + j * 8 + 10 * dst_step) = m22; - *(dst_ic4_ptr + j * 8 + 11 * dst_step) = m23; - - *(dst_ic4_ptr + j * 8 + 12 * dst_step) = m30; - *(dst_ic4_ptr + j * 8 + 13 * dst_step) = m31; - *(dst_ic4_ptr + j * 8 + 14 * dst_step) = m32; - *(dst_ic4_ptr + j * 8 + 15 * dst_step) = m33; - } -#endif - } - } -} - -void Conv3x3Fp32OutputUnit(const float *gemm_out, const float *bias_data, float *output_data, bool h_not_bound, - bool w_not_bound, int output_w) { -#ifdef ENABLE_ARM - float32x4_t bias_ptr = vld1q_f32(bias_data); - - float32x4_t s00 = vld1q_f32(gemm_out); - float32x4_t s01 = vld1q_f32(gemm_out + 8); - float32x4_t s02 = vld1q_f32(gemm_out + 16); - float32x4_t s03 = vld1q_f32(gemm_out + 24); - - float32x4_t s10 = vld1q_f32(gemm_out + 32); - float32x4_t s11 = vld1q_f32(gemm_out + 40); - float32x4_t s12 = vld1q_f32(gemm_out + 48); - float32x4_t s13 = vld1q_f32(gemm_out + 56); - - float32x4_t s20 = vld1q_f32(gemm_out + 64); - float32x4_t s21 = vld1q_f32(gemm_out + 72); - float32x4_t s22 = vld1q_f32(gemm_out + 80); - float32x4_t s23 = vld1q_f32(gemm_out + 88); - - float32x4_t s30 = vld1q_f32(gemm_out + 96); - float32x4_t s31 = vld1q_f32(gemm_out + 104); - float32x4_t s32 = vld1q_f32(gemm_out + 112); - float32x4_t s33 = vld1q_f32(gemm_out + 120); - - float32x4_t t00 = vaddq_f32(vaddq_f32(s00, s10), s20); - float32x4_t t01 = vaddq_f32(vaddq_f32(s01, s11), s21); - float32x4_t t02 = vaddq_f32(vaddq_f32(s02, s12), s22); - float32x4_t t03 = vaddq_f32(vaddq_f32(s03, s13), s23); - - float32x4_t t10 = vsubq_f32(vsubq_f32(s10, s20), s30); - float32x4_t t11 = vsubq_f32(vsubq_f32(s11, s21), s31); - float32x4_t t12 = vsubq_f32(vsubq_f32(s12, s22), s32); - float32x4_t t13 = vsubq_f32(vsubq_f32(s13, s23), s33); - - float32x4_t d00 = vaddq_f32(vaddq_f32(vaddq_f32(t00, t01), t02), bias_ptr); - float32x4_t d01 = vaddq_f32(vsubq_f32(vsubq_f32(t01, t02), t03), bias_ptr); - float32x4_t d10 = vaddq_f32(vaddq_f32(vaddq_f32(t10, t11), t12), bias_ptr); - float32x4_t d11 = vaddq_f32(vsubq_f32(vsubq_f32(t11, t12), t13), bias_ptr); - - vst1q_f32(output_data, d00); - if (w_not_bound) { - vst1q_f32(output_data + 4, d01); - } - if (h_not_bound) { - vst1q_f32(output_data + output_w * 4, d10); - if (w_not_bound) { - vst1q_f32(output_data + output_w * 4 + 4, d11); - } - } -#else - for (int i = 0; i < C4NUM; i++) { - const float *local_ptr = gemm_out + i; - const float *bias_ptr = bias_data + i; - - float s00 = local_ptr[0]; - float s01 = (local_ptr + 8)[0]; - float s02 = (local_ptr + 16)[0]; - float s03 = (local_ptr + 24)[0]; - - float s10 = (local_ptr + 32)[0]; - float s11 = (local_ptr + 40)[0]; - float s12 = (local_ptr + 48)[0]; - float s13 = (local_ptr + 56)[0]; - - float s20 = (local_ptr + 64)[0]; - float s21 = (local_ptr + 72)[0]; - float s22 = (local_ptr + 80)[0]; - float s23 = (local_ptr + 88)[0]; - - float s30 = (local_ptr + 96)[0]; - float s31 = (local_ptr + 104)[0]; - float s32 = (local_ptr + 112)[0]; - float s33 = (local_ptr + 120)[0]; - - float t00 = s00 + s10 + s20; - float t01 = s01 + s11 + s21; - float t02 = s02 + s12 + s22; - float t03 = s03 + s13 + s23; - - float t10 = s10 - s20 - s30; - float t11 = s11 - s21 - s31; - float t12 = s12 - s22 - s32; - float t13 = s13 - s23 - s33; - - float d00 = t00 + t01 + t02 + bias_ptr[0]; - float d01 = t01 - t02 - t03 + bias_ptr[0]; - float d10 = t10 + t11 + t12 + bias_ptr[0]; - float d11 = t11 - t12 - t13 + bias_ptr[0]; - - (output_data + i)[0] = d00; - if (w_not_bound) { - (output_data + i + C4NUM)[0] = d01; - } - if (h_not_bound) { - (output_data + i + output_w * C4NUM)[0] = d10; - if (w_not_bound) { - (output_data + i + output_w * C4NUM + C4NUM)[0] = d11; - } - } - } -#endif -} - -void Conv3x3Fp32OutputTransform(const float *gemm_out, float *out_data, const float *bias_data, int start_index, - int real_cal_num, int out_w_block, ConvParameter *conv_param) { - int output_channel = conv_param->output_channel_; - int output_w = conv_param->output_w_; - int output_h = conv_param->output_h_; - int oc4 = UP_DIV(output_channel, C4NUM); - int oc8 = UP_DIV(output_channel, C8NUM); - const int input_unit = 4; - if (out_w_block == 0) { - return; - } - for (int i = 0; i < real_cal_num; i++) { - int out_w_index = (start_index + i) % out_w_block; - int out_h_index = (start_index + i) / out_w_block; - int src_tile_offset = i * oc8 * C8NUM * input_unit * input_unit; - int dst_tile_offset = C4NUM * (out_w_index * OUPUT_UNIT + out_h_index * OUPUT_UNIT * output_w); - - for (int j = 0; j < oc4; j++) { - int c8_block = j / 2; - int c8_res = j % 2; - int src_oc4_offset = src_tile_offset + c8_block * input_unit * input_unit * C8NUM + c8_res * C4NUM; - int dst_oc4_offset = dst_tile_offset + j * C4NUM * output_h * output_w; - const float *src_ptr = gemm_out + src_oc4_offset; - const float *bias_ptr = bias_data + j * C4NUM; - float *dst_ptr = out_data + dst_oc4_offset; - - // output transform - bool w_not_bound = out_w_index * OUPUT_UNIT + 1 < output_w; - bool h_not_bound = out_h_index * OUPUT_UNIT + 1 < output_h; - Conv3x3Fp32OutputUnit(src_ptr, bias_ptr, dst_ptr, h_not_bound, w_not_bound, output_w); - } - } -} - // int8 conv3x3 void Conv3x3Int8InputUnit(int16_t *tmp_data, int16_t *trans_input_data, size_t step, int input_zp) { #ifdef ENABLE_ARM diff --git a/mindspore/lite/nnacl/winograd_transform.h b/mindspore/lite/nnacl/winograd_transform.h index 21cfb38ed0..4d28eaa037 100644 --- a/mindspore/lite/nnacl/winograd_transform.h +++ b/mindspore/lite/nnacl/winograd_transform.h @@ -38,21 +38,6 @@ void WinogradInputTransform(const float *input_data, float *trans_input, float * void WinogradOutputTransform(const float *gemm_out, float *out_data, const float *bias_data, int cal_num, int out_tile_index, int output_unit_num, ConvParameter *conv_param, OutputTransFunc func); -// for fp32 convolution 3x3 filter/input/output transform -void Conv3x3Fp32InputUnit(const float *tmp_data, float *trans_input_data, size_t step); - -void Conv3x3Fp32InputTransform(const float *input_data, float *trans_input, float *tmp_data, int start_index, - int real_cal_num, int out_w_block, ConvParameter *conv_param); - -void Conv3x3Fp32FilterTransform(float *weight_data, float *trans_weight, int iC4, int output_channel, int kernel_plane, - int oc_block); - -void Conv3x3Fp32OutputUnit(const float *gemm_out, const float *bias_data, float *output_data, bool h_not_bound, - bool w_not_bound, int output_w); - -void Conv3x3Fp32OutputTransform(const float *gemm_out, float *out_data, const float *bias_data, int start_index, - int real_cal_num, int out_w_block, ConvParameter *conv_param); - // for int8 convolution 3x3 filter/input/output transform void Conv3x3Int8InputUnit(int16_t *tmp_data, int16_t *trans_input_data, size_t step, int input_zp); diff --git a/mindspore/lite/nnacl/winograd_utils.c b/mindspore/lite/nnacl/winograd_utils.c index 564d3e4aae..b9010b7956 100644 --- a/mindspore/lite/nnacl/winograd_utils.c +++ b/mindspore/lite/nnacl/winograd_utils.c @@ -25,6 +25,7 @@ static InputTransFunc InputTransFuncList[] = { NULL, NULL, NULL, NULL, InputTransform4x4Unit, NULL, InputTransform6x6Unit, NULL, InputTransform8x8Unit}; static OutputTransFunc OutputTransFuncList4[] = {NULL, NULL, OutputTransform4x2Unit, OutputTransform4x3Unit}; + static OutputTransFunc OutputTransFuncReluList4[] = {NULL, NULL, OutputTransform4x2ReluUnit, OutputTransform4x3ReluUnit}; static OutputTransFunc OutputTransFuncRelu6List4[] = {NULL, NULL, OutputTransform4x2Relu6Unit, @@ -32,12 +33,14 @@ static OutputTransFunc OutputTransFuncRelu6List4[] = {NULL, NULL, OutputTransfor static OutputTransFunc OutputTransFuncList6[] = { NULL, NULL, OutputTransform6x2Unit, OutputTransform6x3Unit, OutputTransform6x4Unit, OutputTransform6x5Unit}; + static OutputTransFunc OutputTransFuncReluList6[] = {NULL, NULL, OutputTransform6x2ReluUnit, OutputTransform6x3ReluUnit, OutputTransform6x4ReluUnit, OutputTransform6x5ReluUnit}; + static OutputTransFunc OutputTransFuncRelu6List6[] = {NULL, NULL, OutputTransform6x2Relu6Unit, @@ -53,6 +56,7 @@ static OutputTransFunc OutputTransFuncList8[] = {NULL, OutputTransform8x5Unit, OutputTransform8x6Unit, OutputTransform8x7Unit}; + static OutputTransFunc OutputTransFuncReluList8[] = {NULL, NULL, OutputTransform8x2ReluUnit, @@ -61,6 +65,7 @@ static OutputTransFunc OutputTransFuncReluList8[] = {NULL, OutputTransform8x5ReluUnit, OutputTransform8x6ReluUnit, OutputTransform8x7ReluUnit}; + static OutputTransFunc OutputTransFuncRelu6List8[] = {NULL, NULL, OutputTransform8x2Relu6Unit, diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.cc index b2fdca3ceb..221257171a 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.cc @@ -15,9 +15,7 @@ */ #include "src/runtime/kernel/arm/fp32/convolution.h" -#include "src/runtime/kernel/arm/fp32/convolution_slidewindow.h" #include "src/runtime/kernel/arm/fp32/convolution_1x1.h" -#include "src/runtime/kernel/arm/fp32/convolution_3x3.h" #include "src/runtime/kernel/arm/fp32/convolution_winograd.h" #include "nnacl/fp32/conv.h" #include "nnacl/common_func.h" @@ -42,17 +40,10 @@ int ConvolutionCPUKernel::InitWeightBias() { int out_channel = filter_tensor->Batch(); conv_param_->input_channel_ = in_channel; conv_param_->output_channel_ = out_channel; - int ic4 = UP_DIV(in_channel, C4NUM); int kernel_plane = kernel_h * kernel_w; - int oc_block, oc_block_num; -#ifdef ENABLE_ARM32 - oc_block = C4NUM; - oc_block_num = UP_DIV(out_channel, C4NUM); -#else - oc_block = C8NUM; - oc_block_num = UP_DIV(out_channel, C8NUM); -#endif - int pack_weight_size = oc_block_num * oc_block * ic4 * C4NUM * kernel_plane; + const int oc_block = C8NUM; + int oc_block_num = UP_DIV(out_channel, C8NUM); + int pack_weight_size = oc_block_num * oc_block * in_channel * kernel_plane; auto origin_weight = reinterpret_cast(filter_tensor->MutableData()); packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float))); @@ -61,7 +52,7 @@ int ConvolutionCPUKernel::InitWeightBias() { return RET_ERROR; } memset(packed_weight_, 0, pack_weight_size * sizeof(float)); - PackWeightFp32(origin_weight, conv_param_, packed_weight_, oc_block, oc_block_num); + RowMajor2Col8Major(origin_weight, packed_weight_, out_channel, in_channel * kernel_plane); bias_data_ = reinterpret_cast(malloc(oc_block_num * oc_block * sizeof(float))); if (bias_data_ == nullptr) { @@ -80,38 +71,28 @@ int ConvolutionCPUKernel::InitWeightBias() { } int ConvolutionCPUKernel::InitTmpBuffer() { - int out_channel = conv_param_->output_channel_; + int in_channel = conv_param_->input_channel_; MS_ASSERT(ctx_->allocator != nullptr); - int ic4 = UP_DIV(conv_param_->input_channel_, C4NUM); - int unit_size = conv_param_->kernel_h_ * conv_param_->kernel_w_ * ic4 * C4NUM * TILE_NUM * thread_count_; +#ifdef ENABLE_ARM32 + int unit_size = conv_param_->kernel_h_ * conv_param_->kernel_w_ * in_channel * C4NUM * thread_count_; +#else + int unit_size = conv_param_->kernel_h_ * conv_param_->kernel_w_ * in_channel * C12NUM * thread_count_; +#endif packed_input_ = reinterpret_cast(ctx_->allocator->Malloc(unit_size * sizeof(float))); if (packed_input_ == nullptr) { MS_LOG(ERROR) << "malloc packed input failed."; return RET_ERROR; } - tmp_output_block_ = - reinterpret_cast(ctx_->allocator->Malloc(thread_count_ * TILE_NUM * out_channel * sizeof(float))); - if (tmp_output_block_ == nullptr) { - MS_LOG(ERROR) << "malloc tmp output block failed."; + col_major_input_ = reinterpret_cast(ctx_->allocator->Malloc(unit_size * sizeof(float))); + if (col_major_input_ == nullptr) { + MS_LOG(ERROR) << "malloc col_major_input_ failed."; return RET_ERROR; } return RET_OK; } -void ConvolutionCPUKernel::ConfigInputOutput() { - // set output format - auto output_tensor = out_tensors_.at(kOutputIndex); - output_tensor->SetFormat(schema::Format::Format_NHWC); - -#ifdef ENABLE_ARM32 - gemm_func_ = IndirectGemmFp32_8x4; -#else - gemm_func_ = IndirectGemmFp32_8x8; -#endif -} - int ConvolutionCPUKernel::Init() { auto ret = InitWeightBias(); if (ret != RET_OK) { @@ -121,7 +102,6 @@ int ConvolutionCPUKernel::Init() { if (!InferShapeDone()) { return RET_OK; } - ConfigInputOutput(); return ReSize(); } @@ -141,15 +121,11 @@ int ConvolutionCPUKernel::ReSize() { } int ConvolutionCPUKernel::RunImpl(int task_id) { - if (gemm_func_ == nullptr) { - MS_LOG(ERROR) << "gemm_func is nullptr."; - return RET_ERROR; - } auto input_tensor = in_tensors_.at(kInputIndex); auto ori_input_data = reinterpret_cast(input_tensor->MutableData()); auto output_addr = reinterpret_cast(out_tensors_.at(kOutputIndex)->MutableData()); - ConvFp32(ori_input_data, packed_input_, packed_weight_, reinterpret_cast(bias_data_), tmp_output_block_, - output_addr, task_id, conv_param_, gemm_func_); + ConvFp32(ori_input_data, packed_input_, packed_weight_, reinterpret_cast(bias_data_), col_major_input_, + output_addr, task_id, conv_param_); return RET_OK; } @@ -186,19 +162,6 @@ int ConvolutionCPUKernel::Run() { return RET_OK; } -bool CheckIfUseSlideWindow(ConvParameter *conv_param) { - int in_channel = conv_param->input_channel_; - int out_h = conv_param->output_h_; - int out_w = conv_param->output_w_; - int out_channel = conv_param->output_channel_; - int ic4 = UP_DIV(in_channel, C4NUM); - int oc4 = UP_DIV(out_channel, C4NUM); - if (out_h * out_w <= 32 || ic4 < 4 || oc4 < 4) { - return true; - } - return false; -} - kernel::LiteKernel *CpuConvFp32KernelCreator(const std::vector &inputs, const std::vector &outputs, OpParameter *op_parameter, const InnerContext *ctx, const kernel::KernelKey &desc, diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.h b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.h index 97188bd4b6..41e41d9599 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution.h @@ -43,23 +43,21 @@ class ConvolutionCPUKernel : public ConvolutionBaseCPUKernel { int RunImpl(int task_id); int InitWeightBias(); int InitTmpBuffer(); - void ConfigInputOutput(); private: void FreeTmpBuffer() { - if (tmp_output_block_ != nullptr) { - ctx_->allocator->Free(tmp_output_block_); - tmp_output_block_ = nullptr; - } if (packed_input_ != nullptr) { ctx_->allocator->Free(packed_input_); packed_input_ = nullptr; } + if (col_major_input_ != nullptr) { + ctx_->allocator->Free(col_major_input_); + col_major_input_ = nullptr; + } } float *packed_input_ = nullptr; float *packed_weight_ = nullptr; - float *tmp_output_block_ = nullptr; - GEMM_FUNC_FP32 gemm_func_ = nullptr; + float *col_major_input_ = nullptr; }; } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.cc deleted file mode 100644 index f63dca1e81..0000000000 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.cc +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/runtime/kernel/arm/fp32/convolution_3x3.h" -#include "nnacl/fp32/conv.h" -#include "src/runtime/kernel/arm/base/layout_transform.h" -#include "schema/model_generated.h" -#include "src/kernel_registry.h" -#include "include/errorcode.h" -#include "src/runtime/runtime_api.h" - -using mindspore::kernel::KERNEL_ARCH::kCPU; -using mindspore::lite::KernelRegistrar; -using mindspore::lite::RET_ERROR; -using mindspore::lite::RET_OK; -using mindspore::schema::PrimitiveType_Conv2D; - -namespace mindspore::kernel { -void ProcessFilter(float *origin_weight, float *dst_weight, ConvParameter *conv_param, int oc_block, int oc_block_num) { - auto input_channel = conv_param->input_channel_; - auto output_channel = conv_param->output_channel_; - auto kernel_plane = conv_param->kernel_w_ * conv_param->kernel_h_; - int iC4 = UP_DIV(input_channel, C4NUM); - - size_t tmp_size = oc_block_num * oc_block * iC4 * C4NUM * kernel_plane * sizeof(float); - auto tmp_addr = reinterpret_cast(malloc(tmp_size)); - if (tmp_addr == nullptr) { - MS_LOG(ERROR) << "malloc tmp_addr failed."; - return; - } - memset(tmp_addr, 0, tmp_size); - - PackNHWCToNC4HW4Fp32(origin_weight, tmp_addr, output_channel, kernel_plane, input_channel); - Conv3x3Fp32FilterTransform(tmp_addr, dst_weight, iC4, output_channel, kernel_plane, oc_block); - free(tmp_addr); -} - -int Convolution3x3CPUKernel::InitWeightBias() { - auto filter_tensor = in_tensors_.at(kWeightIndex); - auto input_channel = filter_tensor->Channel(); - auto output_channel = filter_tensor->Batch(); - conv_param_->input_channel_ = input_channel; - conv_param_->output_channel_ = output_channel; - int iC4 = UP_DIV(input_channel, C4NUM); - int oC4 = UP_DIV(output_channel, C4NUM); - int oc_block, oc_block_num; - oc_block = C8NUM; - oc_block_num = UP_DIV(output_channel, C8NUM); - const int k_plane = 16; - // init weight - size_t transformed_size = iC4 * C4NUM * oc_block_num * oc_block * k_plane * sizeof(float); - transformed_filter_addr_ = reinterpret_cast(malloc(transformed_size)); - if (transformed_filter_addr_ == nullptr) { - MS_LOG(ERROR) << "malloc transformed filter addr failed."; - return RET_ERROR; - } - memset(transformed_filter_addr_, 0, transformed_size); - auto weight_data = reinterpret_cast(in_tensors_.at(kWeightIndex)->MutableData()); - ProcessFilter(weight_data, transformed_filter_addr_, conv_param_, oc_block, oc_block_num); - - // init bias - size_t new_bias_size = oC4 * C4NUM * sizeof(float); - bias_data_ = reinterpret_cast(malloc(new_bias_size)); - if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "malloc bias data failed."; - return RET_ERROR; - } - memset(bias_data_, 0, new_bias_size); - if (in_tensors_.size() == kInputSize2) { - auto ori_bias_addr = reinterpret_cast(in_tensors_.at(kBiasIndex)->MutableData()); - memcpy(bias_data_, ori_bias_addr, output_channel * sizeof(float)); - } else { - MS_ASSERT(in_tensors_.size() == kInputSize1); - } - return RET_OK; -} - -int Convolution3x3CPUKernel::InitTmpBuffer() { - int ic4 = UP_DIV(conv_param_->input_channel_, C4NUM); - int oC4 = UP_DIV(conv_param_->output_channel_, C4NUM); - int oC8 = UP_DIV(conv_param_->output_channel_, C8NUM); - const int k_plane = 16; - MS_ASSERT(ctx_->allocator != nullptr); - -#ifdef ENABLE_ARM32 - const int tile_num = 4; -#else - const int tile_num = 12; -#endif - - size_t tile_buffer_size = thread_count_ * tile_num * C16NUM * ic4 * C4NUM * sizeof(float); - tile_buffer_ = reinterpret_cast(ctx_->allocator->Malloc(tile_buffer_size)); - if (tile_buffer_ == nullptr) { - MS_LOG(ERROR) << "malloc tile buffer failed."; - return RET_ERROR; - } - - size_t block_unit_buffer_size = thread_count_ * k_plane * C4NUM * sizeof(float); - block_unit_buffer_ = reinterpret_cast(ctx_->allocator->Malloc(block_unit_buffer_size)); - if (block_unit_buffer_ == nullptr) { - MS_LOG(ERROR) << "malloc block_unit_buffer_ failed."; - return RET_ERROR; - } - - size_t tmp_dst_buffer_size = thread_count_ * tile_num * k_plane * oC8 * C8NUM * sizeof(float); - tmp_dst_buffer_ = reinterpret_cast(ctx_->allocator->Malloc(tmp_dst_buffer_size)); - if (tmp_dst_buffer_ == nullptr) { - MS_LOG(ERROR) << "malloc tmp_dst_buffer_ failed."; - return RET_ERROR; - } - - size_t col_buffer_size = thread_count_ * tile_num * C4NUM * ic4 * sizeof(float); - col_buffer_ = reinterpret_cast(ctx_->allocator->Malloc(col_buffer_size)); - if (col_buffer_ == nullptr) { - MS_LOG(ERROR) << "malloc col_buffer_ failed."; - return RET_ERROR; - } - - size_t nc4hw4_out_size = - oC4 * C4NUM * conv_param_->output_batch_ * conv_param_->output_h_ * conv_param_->output_w_ * sizeof(float); - nc4hw4_out_ = reinterpret_cast(ctx_->allocator->Malloc(nc4hw4_out_size)); - if (nc4hw4_out_ == nullptr) { - MS_LOG(ERROR) << "malloc nc4hw4_out_ failed."; - return RET_ERROR; - } - - tmp_buffer_address_list_[0] = tile_buffer_; - tmp_buffer_address_list_[1] = block_unit_buffer_; - tmp_buffer_address_list_[2] = tmp_dst_buffer_; - tmp_buffer_address_list_[3] = nc4hw4_out_; - tmp_buffer_address_list_[4] = col_buffer_; - return RET_OK; -} - -int Convolution3x3CPUKernel::Init() { - auto ret = InitWeightBias(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Init weight bias failed.ret: " << ret; - return RET_ERROR; - } - if (!InferShapeDone()) { - return RET_OK; - } - return ReSize(); -} - -int Convolution3x3CPUKernel::ReSize() { - auto ret = ConvolutionBaseCPUKernel::CheckResizeValid(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Resize is invalid."; - return ret; - } - - ret = ConvolutionBaseCPUKernel::Init(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "ConvolutionBase init failed.ret: " << ret; - return RET_ERROR; - } - return RET_OK; -} - -int Convolution3x3CPUKernel::RunImpl(int task_id) { - auto input_tensor = in_tensors_.at(kInputIndex); - auto ori_input_data = reinterpret_cast(input_tensor->MutableData()); - Conv3x3Fp32(ori_input_data, transformed_filter_addr_, reinterpret_cast(bias_data_), tmp_buffer_address_list_, - task_id, conv_param_); - return RET_OK; -} - -int Convolution3x3Impl(void *cdata, int task_id) { - auto conv3x3 = reinterpret_cast(cdata); - auto error_code = conv3x3->RunImpl(task_id); - if (error_code != RET_OK) { - MS_LOG(ERROR) << "Convolution3x3 Run error task_id[" << task_id << "] error_code[" << error_code << "]"; - return RET_ERROR; - } - return RET_OK; -} - -int Convolution3x3CPUKernel::PostProcess() { - auto output_addr = reinterpret_cast(out_tensors_.at(kOutputIndex)->MutableData()); - auto act_type = conv_param_->act_type_; - switch (act_type) { - case ActType_No: - PackNC4HW4ToNHWCFp32(nc4hw4_out_, output_addr, conv_param_->output_batch_, - conv_param_->output_h_ * conv_param_->output_w_, conv_param_->output_channel_); - break; - case ActType_Relu: - PackNC4HW4ToNHWCReluFp32(nc4hw4_out_, output_addr, conv_param_->output_batch_, - conv_param_->output_h_ * conv_param_->output_w_, conv_param_->output_channel_); - break; - case ActType_Relu6: - PackNC4HW4ToNHWCRelu6Fp32(nc4hw4_out_, output_addr, conv_param_->output_batch_, - conv_param_->output_h_ * conv_param_->output_w_, conv_param_->output_channel_); - break; - default: - MS_LOG(ERROR) << "Unsupport activation type."; - return RET_ERROR; - } - return RET_OK; -} - -int Convolution3x3CPUKernel::Run() { - auto prepare_ret = Prepare(); - if (prepare_ret != RET_OK) { - MS_LOG(ERROR) << "Prepare fail!ret: " << prepare_ret; - return prepare_ret; - } - - auto ret = InitTmpBuffer(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Init tmp buffer failed.ret: " << ret; - return RET_ERROR; - } - - int error_code = ParallelLaunch(this->context_->thread_pool_, Convolution3x3Impl, this, thread_count_); - if (error_code != RET_OK) { - MS_LOG(ERROR) << "conv3x3 error error_code[" << error_code << "]"; - FreeTmpBuffer(); - return RET_ERROR; - } - - ret = PostProcess(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Post process failed."; - FreeTmpBuffer(); - return ret; - } - FreeTmpBuffer(); - return RET_OK; -} -} // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.h b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.h deleted file mode 100644 index 30bfad66bf..0000000000 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_3x3.h +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_3X3_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_3X3_H_ - -#include -#include "src/lite_kernel.h" -#include "src/runtime/kernel/arm/base/convolution_base.h" -#include "nnacl/winograd_transform.h" - -namespace mindspore::kernel { -class Convolution3x3CPUKernel : public ConvolutionBaseCPUKernel { - public: - Convolution3x3CPUKernel(OpParameter *parameter, const std::vector &inputs, - const std::vector &outputs, const lite::InnerContext *ctx, - const mindspore::lite::PrimitiveC *primitive) - : ConvolutionBaseCPUKernel(parameter, inputs, outputs, ctx, primitive) {} - ~Convolution3x3CPUKernel() override { - if (transformed_filter_addr_ != nullptr) { - free(transformed_filter_addr_); - } - } - int Init() override; - int ReSize() override; - int Run() override; - int RunImpl(int task_id); - int InitWeightBias(); - int InitTmpBuffer(); - int PostProcess(); - - private: - void FreeTmpBuffer() { - if (tile_buffer_ != nullptr) { - ctx_->allocator->Free(tile_buffer_); - tile_buffer_ = nullptr; - } - if (block_unit_buffer_ != nullptr) { - ctx_->allocator->Free(block_unit_buffer_); - block_unit_buffer_ = nullptr; - } - if (tmp_dst_buffer_ != nullptr) { - ctx_->allocator->Free(tmp_dst_buffer_); - tmp_dst_buffer_ = nullptr; - } - if (nc4hw4_out_ != nullptr) { - ctx_->allocator->Free(nc4hw4_out_); - nc4hw4_out_ = nullptr; - } - if (col_buffer_ != nullptr) { - ctx_->allocator->Free(col_buffer_); - col_buffer_ = nullptr; - } - } - - float *transformed_filter_addr_ = nullptr; - float *tile_buffer_ = nullptr; - float *block_unit_buffer_ = nullptr; - float *tmp_dst_buffer_ = nullptr; - float *col_buffer_ = nullptr; - float *nc4hw4_out_ = nullptr; - TmpBufferAddress tmp_buffer_address_list_[5]; -}; -void ProcessFilter(float *origin_weight, float *dst_weight, ConvParameter *conv_param, int oc_block, int oc_block_num); -} // namespace mindspore::kernel - -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_3X3_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.cc b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.cc deleted file mode 100644 index a2a037788e..0000000000 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.cc +++ /dev/null @@ -1,201 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/runtime/kernel/arm/fp32/convolution_slidewindow.h" -#include "nnacl/common_func.h" -#include "schema/model_generated.h" -#include "src/kernel_registry.h" -#include "include/errorcode.h" -#include "src/runtime/runtime_api.h" - -namespace mindspore::kernel { -using mindspore::kernel::KERNEL_ARCH::kCPU; -using mindspore::lite::KernelRegistrar; -using mindspore::lite::RET_ERROR; -using mindspore::lite::RET_INFER_INVALID; -using mindspore::lite::RET_OK; -using mindspore::schema::PrimitiveType_Conv2D; - -int ConvolutionSWCPUKernel::InitWeightBias() { - auto filter_tensor = in_tensors_.at(kWeightIndex); - auto input_channel = filter_tensor->Channel(); - auto output_channel = filter_tensor->Batch(); - int kernel_h = filter_tensor->Height(); - int kernel_w = filter_tensor->Width(); - conv_param_->input_channel_ = input_channel; - conv_param_->output_channel_ = output_channel; - int ic4 = UP_DIV(input_channel, C4NUM); - int kernel_plane = kernel_h * kernel_w; - int oc_block = C4NUM; - int oc_block_num = UP_DIV(output_channel, C4NUM); - int pack_weight_size = oc_block_num * oc_block * ic4 * C4NUM * kernel_plane; - - auto origin_weight = reinterpret_cast(in_tensors_.at(kWeightIndex)->MutableData()); - packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float))); - if (packed_weight_ == nullptr) { - MS_LOG(ERROR) << "malloc packed weight failed."; - return RET_ERROR; - } - memset(packed_weight_, 0, pack_weight_size * sizeof(float)); - for (int oc = 0; oc < output_channel; ++oc) { - int src_oc_offset = oc * kernel_h * kernel_w * input_channel; - int dst_oc_offset = oc * kernel_h * kernel_w * ic4 * C4NUM; - for (int i = 0; i < kernel_h * kernel_w; ++i) { - const float *src = origin_weight + src_oc_offset + i * input_channel; - float *dst = packed_weight_ + dst_oc_offset + i * ic4 * C4NUM; - memcpy(dst, src, input_channel * sizeof(float)); - } - } - - bias_data_ = reinterpret_cast(malloc(oc_block_num * oc_block * sizeof(float))); - if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "malloc bias failed."; - return RET_ERROR; - } - memset(bias_data_, 0, oc_block_num * oc_block * sizeof(float)); - if (in_tensors_.size() == kInputSize2) { - auto ori_bias = reinterpret_cast(in_tensors_.at(kBiasIndex)->MutableData()); - memcpy(bias_data_, ori_bias, output_channel * sizeof(float)); - } else { - MS_ASSERT(in_tensors_.size() == kInputSize1); - } - return RET_OK; -} - -int ConvolutionSWCPUKernel::InitTmpBuffer() { - int out_channel = conv_param_->output_channel_; - int oc4 = UP_DIV(out_channel, C4NUM); - MS_ASSERT(ctx_->allocator != nullptr); - int ic4 = UP_DIV(conv_param_->input_channel_, C4NUM); - size_t nhwc4_input_size = - ic4 * C4NUM * conv_param_->input_batch_ * conv_param_->input_h_ * conv_param_->input_w_ * sizeof(float); - nhwc4_input_ = ctx_->allocator->Malloc(nhwc4_input_size); - if (nhwc4_input_ == nullptr) { - MS_LOG(ERROR) << "malloc nhwc4 input failed."; - return RET_ERROR; - } - - tmp_output_block_ = reinterpret_cast(ctx_->allocator->Malloc( - conv_param_->output_batch_ * conv_param_->output_h_ * conv_param_->output_w_ * oc4 * C4NUM * sizeof(float))); - if (tmp_output_block_ == nullptr) { - MS_LOG(ERROR) << "malloc tmp output block failed."; - return RET_ERROR; - } - return RET_OK; -} - -void ConvolutionSWCPUKernel::ConfigInputOutput() { - // set output format - auto output_tensor = out_tensors_.at(kOutputIndex); - output_tensor->SetFormat(schema::Format::Format_NHWC); -} - -int ConvolutionSWCPUKernel::Init() { - auto ret = InitWeightBias(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Init weight bias failed."; - return RET_ERROR; - } - if (!InferShapeDone()) { - return RET_OK; - } - // config input output - ConfigInputOutput(); - return ReSize(); -} - -int ConvolutionSWCPUKernel::ReSize() { - auto ret = ConvolutionBaseCPUKernel::CheckResizeValid(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Resize is invalid."; - return ret; - } - - if (slidingWindow_param_ != nullptr) { - delete slidingWindow_param_; - slidingWindow_param_ = nullptr; - } - - ret = ConvolutionBaseCPUKernel::Init(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "ConvolutionBase init failed."; - return RET_ERROR; - } - - // init sliding window param - slidingWindow_param_ = new (std::nothrow) SlidingWindowParam; - if (slidingWindow_param_ == nullptr) { - MS_LOG(ERROR) << "new SlidingWindowParam fail!"; - return RET_ERROR; - } - InitSlidingParamConv(slidingWindow_param_, conv_param_, C4NUM); - - return RET_OK; -} - -int ConvolutionSWCPUKernel::RunImpl(int task_id) { - auto output_addr = reinterpret_cast(out_tensors_.at(kOutputIndex)->MutableData()); - ConvSWFp32(reinterpret_cast(nhwc4_input_), packed_weight_, reinterpret_cast(bias_data_), - tmp_output_block_, output_addr, task_id, conv_param_, slidingWindow_param_); - return RET_OK; -} - -int ConvolutionSWImpl(void *cdata, int task_id) { - auto conv = reinterpret_cast(cdata); - auto error_code = conv->RunImpl(task_id); - if (error_code != RET_OK) { - MS_LOG(ERROR) << "Convolution Sliding Window Run error task_id[" << task_id << "] error_code[" << error_code << "]"; - return RET_ERROR; - } - return RET_OK; -} - -int ConvolutionSWCPUKernel::Run() { - auto prepare_ret = Prepare(); - if (prepare_ret != RET_OK) { - MS_LOG(ERROR) << "Prepare fail!ret: " << prepare_ret; - return prepare_ret; - } - - // init tmp input, output - auto ret = InitTmpBuffer(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "Init tmp buffer failed."; - return RET_ERROR; - } - auto input_tensor = in_tensors_.at(kInputIndex); - auto ori_input_data = input_tensor->MutableData(); - PackNHWCToNHWC4Fp32(ori_input_data, nhwc4_input_, conv_param_->input_batch_, - conv_param_->input_h_ * conv_param_->input_w_, conv_param_->input_channel_); - - int error_code = ParallelLaunch(this->context_->thread_pool_, ConvolutionSWImpl, this, thread_count_); - if (error_code != RET_OK) { - MS_LOG(ERROR) << "conv error error_code[" << error_code << "]"; - FreeTmpBuffer(); - return RET_ERROR; - } - - auto out_tensor = out_tensors_.front(); - auto out_data = reinterpret_cast(out_tensor->MutableData()); - int oc4_res = conv_param_->output_channel_ % C4NUM; - if (oc4_res != 0) { - PackNHWC4ToNHWCFp32(tmp_output_block_, out_data, conv_param_->output_batch_, - conv_param_->output_h_ * conv_param_->output_w_, conv_param_->output_channel_); - } - FreeTmpBuffer(); - return RET_OK; -} -} // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.h b/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.h deleted file mode 100644 index fcf616ee3f..0000000000 --- a/mindspore/lite/src/runtime/kernel/arm/fp32/convolution_slidewindow.h +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_SLIDEWINDOW_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_SLIDEWINDOW_H_ - -#include -#include "src/lite_kernel.h" -#include "nnacl/op_base.h" -#include "src/runtime/kernel/arm/base/convolution_base.h" -#include "nnacl/fp32/conv.h" -#include "nnacl/fp32/conv_depthwise.h" - -namespace mindspore::kernel { -class ConvolutionSWCPUKernel : public ConvolutionBaseCPUKernel { - public: - ConvolutionSWCPUKernel(OpParameter *parameter, const std::vector &inputs, - const std::vector &outputs, const lite::InnerContext *ctx, - const mindspore::lite::PrimitiveC *primitive) - : ConvolutionBaseCPUKernel(parameter, inputs, outputs, ctx, primitive) {} - - ~ConvolutionSWCPUKernel() override { - if (packed_weight_ != nullptr) { - free(packed_weight_); - packed_weight_ = nullptr; - } - if (slidingWindow_param_ != nullptr) { - delete slidingWindow_param_; - slidingWindow_param_ = nullptr; - } - } - - int Init() override; - int ReSize() override; - int Run() override; - int RunImpl(int task_id); - int InitWeightBias(); - int InitTmpBuffer(); - void ConfigInputOutput(); - - private: - void FreeTmpBuffer() { - if (nhwc4_input_ != nullptr) { - ctx_->allocator->Free(nhwc4_input_); - nhwc4_input_ = nullptr; - } - if (tmp_output_block_ != nullptr) { - ctx_->allocator->Free(tmp_output_block_); - tmp_output_block_ = nullptr; - } - } - float *packed_weight_ = nullptr; - float *tmp_output_block_ = nullptr; - SlidingWindowParam *slidingWindow_param_ = nullptr; -}; -} // namespace mindspore::kernel -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP32_CONVOLUTION_SLIDEWINDOW_H_ diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/common/pack_tests.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/common/pack_tests.cc index e3329df5c3..b5ed5f71c4 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/common/pack_tests.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/common/pack_tests.cc @@ -107,35 +107,6 @@ TEST_F(TestPack, PackInputFp32) { MS_LOG(INFO) << "TestPackInputFp32 passed"; } -TEST_F(TestPack, PackWeightFp32) { - auto conv_param = new ConvParameter; - InitConvParamPack(conv_param); - - int k_h = conv_param->kernel_h_; - int k_w = conv_param->kernel_w_; - int in_channel = conv_param->input_channel_; - int out_channel = conv_param->output_channel_; - int ic4 = UP_DIV(in_channel, C4NUM); - int oc8 = UP_DIV(out_channel, C8NUM); - - size_t weight_size; - std::string weight_path = "./test_data/conv/convfp32_weight_32_3_3_3.bin"; - auto weight_data = reinterpret_cast(mindspore::lite::ReadFile(weight_path.c_str(), &weight_size)); - auto packed_weight = reinterpret_cast(malloc(k_h * k_w * ic4 * C4NUM * oc8 * C8NUM * sizeof(float))); - PackWeightFp32(weight_data, conv_param, packed_weight, C8NUM, oc8); - - printf("==================output data=================\n"); - for (int i = 0; i < 20; i++) { - std::cout << packed_weight[i] << " ,"; - } - std::cout << std::endl; - - free(packed_weight); - delete conv_param; - - MS_LOG(INFO) << "TestPackWeightFp32 passed"; -} - #ifdef ENABLE_FP16 TEST_F(TestPack, PackInputFp16) { size_t input_size;