zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bc738d4ec3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bc738d4ec3'
${ noResults }
mindspore2022/mindspore/lite/nnacl/int8/conv_depthwise_int8.c

806 lines
41 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/int8/conv_depthwise_int8.h"

  18. #include <string.h>

  19. #include "nnacl/quantization/fixed_point.h"

  20. #include "nnacl/int8/common_func_int8.h"

  21. 

  22. /*conv depthwise int8 begin*/

  23. #ifndef ENABLE_ARM

  24. void ConvDwInt8Row(int32_t *output_ptr, const int8_t *input_ptr, const int16_t *weight_ptr, int num_pixels,

  25.                    int output_channel, int input_step, int8_t input_zp) {

  26.   for (int i = 0; i < num_pixels; i++) {

  27.     for (int c = 0; c < output_channel; c++) {

  28.       const int16_t input = input_ptr[c] - input_zp;

  29.       *output_ptr++ += input * weight_ptr[c];

  30.     }

  31.     input_ptr += input_step;

  32.   }

  33. }

  34. #endif

  35. 

  36. void ConvDwInt8Post(int8_t *dst, int32_t *buffer, int output_w, int channel, int32_t output_zp, int32_t *out_multiplier,

  37.                     int32_t *left_shift, int32_t *right_shift, int32_t acc_min, int32_t acc_max, bool per_channel) {

  38.   if (per_channel) {

  39.     // support perchannel

  40.     for (int w = 0; w < output_w; w++) {

  41.       int channel4 = 0;

  42. #ifdef ENABLE_ARM

  43.       channel4 = channel / 4 * 4;

  44.       ConvDwInt8PostAlign4PerChannel(dst, buffer, channel4, output_zp, out_multiplier, left_shift, right_shift, acc_min,

  45.                                      acc_max);

  46. #endif

  47.       for (int c = channel4; c < channel; c++) {

  48.         buffer[c] = RoundingDivideByPOT(

  49.           SaturatingRoundingDoublingHighMul(buffer[c] * (1 << (unsigned int)left_shift[c]), out_multiplier[c]),

  50.           -right_shift[c]);

  51.         buffer[c] += output_zp;

  52.         buffer[c] = MSMAX(buffer[c], acc_min);

  53.         buffer[c] = MSMIN(buffer[c], acc_max);

  54.         dst[c] = (buffer[c]);

  55.       }

  56.       buffer += channel;

  57.       dst += channel;

  58.     }

  59.   } else {

  60.     int num_pixels = output_w * channel;

  61.     int align_num = 0;

  62. #ifdef ENABLE_ARM

  63.     align_num = num_pixels / 4 * 4;

  64.     ConvDwInt8PostAlign4(dst, buffer, align_num, output_zp, out_multiplier[0], left_shift[0], right_shift[0], acc_min,

  65.                          acc_max);

  66. #endif

  67.     for (int i = align_num; i < num_pixels; i++) {

  68.       buffer[i] = RoundingDivideByPOT(

  69.         SaturatingRoundingDoublingHighMul(buffer[i] * (1 << (unsigned int)left_shift[0]), out_multiplier[0]),

  70.         -right_shift[0]);

  71.       buffer[i] += output_zp;

  72.       buffer[i] = MSMAX(buffer[i], acc_min);

  73.       buffer[i] = MSMIN(buffer[i], acc_max);

  74.       dst[i] = (buffer[i]);

  75.     }

  76.   }

  77. }

  78. 

  79. void ConvDwInt8(int8_t *output_data, int32_t *row_buffer, const int8_t *input_data, const int16_t *weight_data,

  80.                 const int32_t *bias_data, const ConvParameter *conv_param, int task_id) {

  81.   int step_h = UP_DIV(conv_param->output_h_, conv_param->thread_num_);

  82.   int start_h = step_h * task_id;

  83.   int end_h = MSMIN(start_h + step_h, conv_param->output_h_);

  84. 

  85.   bool filter_per_channel = conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL;

  86.   int *out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_;

  87.   int *left_shift = conv_param->conv_quant_arg_.left_shift_;

  88.   int *right_shift = conv_param->conv_quant_arg_.right_shift_;

  89. 

  90.   int intput_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  91.   int output_zp = conv_param->conv_quant_arg_.output_quant_args_[0].zp_;

  92.   int acc_min = conv_param->conv_quant_arg_.out_act_min_[0];

  93.   int acc_max = conv_param->conv_quant_arg_.out_act_max_[0];

  94. 

  95.   for (int b = 0; b < conv_param->output_batch_; b++) {

  96.     const int8_t *src = input_data + b * conv_param->input_h_ * conv_param->input_w_ * conv_param->input_channel_;

  97.     int8_t *dst = output_data + b * conv_param->output_h_ * conv_param->output_w_ * conv_param->output_channel_;

  98.     for (int oh = start_h; oh < end_h; oh++) {

  99.       int8_t *dst_data = dst + oh * conv_param->output_w_ * conv_param->output_channel_;

  100. 

  101.       int ih_origin = oh * conv_param->stride_h_ - conv_param->pad_u_;

  102.       int start_kh = MSMAX(0, UP_DIV(-ih_origin, conv_param->dilation_h_));

  103.       int end_kh = MSMIN(conv_param->kernel_h_, UP_DIV(conv_param->input_h_ - ih_origin, conv_param->dilation_h_));

  104. 

  105.       // init acc

  106.       for (int ow = 0; ow < conv_param->output_w_; ow++) {

  107.         memcpy(row_buffer + ow * conv_param->output_channel_, bias_data, conv_param->output_channel_ * sizeof(int32_t));

  108.       }

  109.       for (int kh = start_kh; kh < end_kh; kh++) {

  110.         int ih = ih_origin + conv_param->dilation_w_ * kh;

  111. 

  112.         const int8_t *src_kh = src + ih * conv_param->input_w_ * conv_param->input_channel_;

  113.         const int16_t *weight_kh = weight_data + kh * conv_param->kernel_w_ * conv_param->output_channel_;

  114. 

  115.         int in_sw_step = conv_param->stride_w_ * conv_param->input_channel_;

  116.         for (int kw = 0; kw < conv_param->kernel_w_; kw++) {

  117.           int out_w_start = MSMAX(

  118.             0, (conv_param->pad_l_ - conv_param->dilation_w_ * kw + conv_param->stride_w_ - 1) / conv_param->stride_w_);

  119.           int out_w_end = MSMIN(conv_param->output_w_, (conv_param->input_w_ + conv_param->pad_l_ -

  120.                                                         conv_param->dilation_w_ * kw + conv_param->stride_w_ - 1) /

  121.                                                          conv_param->stride_w_);

  122. 

  123.           int32_t *acc_w = row_buffer + out_w_start * conv_param->output_channel_;

  124.           int iw_origin = (out_w_start * conv_param->stride_w_) - conv_param->pad_l_ + conv_param->dilation_w_ * kw;

  125. 

  126.           const int8_t *src_kw = src_kh + iw_origin * conv_param->input_channel_;

  127.           int num_pixels = out_w_end - out_w_start;

  128. 

  129.           ConvDwInt8Row(acc_w, src_kw, weight_kh, num_pixels, conv_param->output_channel_, in_sw_step, intput_zp);

  130.           weight_kh += conv_param->output_channel_;

  131.         }

  132.       }

  133.       // post func, acc int32 -> dst int8

  134.       ConvDwInt8Post(dst_data, row_buffer, conv_param->output_w_, conv_param->output_channel_, output_zp,

  135.                      out_multiplier, left_shift, right_shift, acc_min, acc_max, filter_per_channel);

  136.     }

  137.   }

  138. }

  139. /*conv depthwise int8 end*/

  140. 

  141. /*conv depthwise 3x3 int8 begin*/

  142. bool CheckIfUse3X3(const ConvParameter *conv_param, int channel) {

  143.   bool use_3x3 = conv_param->kernel_h_ == 3 && conv_param->kernel_w_ == 3 && conv_param->stride_h_ == 1 &&

  144.                  conv_param->stride_w_ == 1 && (conv_param->pad_u_ == 0 || conv_param->pad_u_ == 1) &&

  145.                  (conv_param->pad_l_ == 0 || conv_param->pad_l_ == 1) && conv_param->pad_u_ == conv_param->pad_l_ &&

  146.                  conv_param->dilation_h_ == 1 && conv_param->dilation_w_ == 1 && (channel % 8 == 0);

  147.   return use_3x3;

  148. }

  149. 

  150. void InitInputBuffer(int8_t *buffer, const int8_t *input, const ConvParameter *conv_param, int block_input_h,

  151.                      int block_input_w) {

  152.   for (int h = 0; h < block_input_h; h++) {

  153.     const int8_t *src = input;

  154.     for (int w = 0; w < block_input_w; w++) {

  155.       memcpy(buffer, src, 64);

  156.       src += conv_param->input_channel_;

  157.       buffer += 64;

  158.     }

  159.     input += conv_param->input_w_ * conv_param->input_channel_;

  160.   }

  161. }

  162. 

  163. void ConvDw3x3Int8Window(int8_t *output, const int8_t *buffer, const int16_t *weight, const int32_t *bias, int col_size,

  164.                          int row_size, int channel, int output_h, int output_w, int8_t in_zp, int32_t out_zp,

  165.                          int out_multiplier, int left_shift, int right_shift, int32_t acc_min, int32_t acc_max,

  166.                          int stride) {

  167.   for (int w = 0; w < output_w; w++) {

  168.     int tmp_buffer[C8NUM];

  169.     for (int i = 0; i < C8NUM; i++) {

  170.       tmp_buffer[i] = 0;

  171.     }

  172.     int8_t *output_tmp = output;

  173.     const int8_t *src_kh = buffer;

  174.     const int16_t *weight_kh = weight;

  175.     for (int kh = 0; kh < 3; kh++) {

  176.       const int8_t *src_kw = src_kh;

  177.       const int16_t *weight_kw = weight_kh;

  178.       for (int kw = 0; kw < 3; kw++) {

  179.         for (int c = 0; c < 8; c++) {

  180.           tmp_buffer[c] += (src_kw[c] - in_zp) * weight_kw[c];

  181.         }

  182.         src_kw += col_size;

  183.         weight_kw += channel;

  184.       }

  185.       src_kh += row_size;

  186.       weight_kh += 3 * channel;

  187.     }

  188.     for (int c = 0; c < C8NUM; c++) {

  189.       tmp_buffer[c] += bias[c];

  190.       tmp_buffer[c] = RoundingDivideByPOT(

  191.         SaturatingRoundingDoublingHighMul(tmp_buffer[c] * (1 << (unsigned int)left_shift), out_multiplier),

  192.         -right_shift);

  193.       tmp_buffer[c] += out_zp;

  194.       tmp_buffer[c] = MSMAX(tmp_buffer[c], acc_min);

  195.       tmp_buffer[c] = MSMIN(tmp_buffer[c], acc_max);

  196.       *output_tmp++ = (tmp_buffer[c]);

  197.     }

  198.     output += channel;

  199.     buffer += col_size * stride;

  200.   }

  201. }

  202. 

  203. void ConvDw3x3Int8Block(int8_t *output, const int8_t *buffer, const int16_t *weight, const int32_t *bias, int start_c,

  204.                         int end_c, int col_size, int row_size, int channel, int output_h, int output_w, int8_t in_zp,

  205.                         int32_t out_zp, int out_multiplier, int left_shift, int right_shift, int32_t acc_min,

  206.                         int32_t acc_max, int stride) {

  207.   for (; start_c <= end_c - 8; start_c += 8) {

  208. #ifdef ENABLE_ARM64

  209.     ConvDw3x3Int8Neon64(output, buffer, weight, bias, col_size, row_size, channel, output_h, output_w, in_zp, out_zp,

  210.                         out_multiplier, left_shift, right_shift, acc_min, acc_max);

  211. #else

  212.     ConvDw3x3Int8Window(output, buffer, weight, bias, col_size, row_size, channel, output_h, output_w, in_zp, out_zp,

  213.                         out_multiplier, left_shift, right_shift, acc_min, acc_max, stride);

  214. #endif

  215.     output += 8;

  216.     buffer += 8;

  217.     weight += 8;

  218.     bias += 8;

  219.   }

  220. }

  221. 

  222. void ConvDw3x3Int8Row(int8_t *output, int8_t *buffer, const int8_t *input, const int16_t *weight, const int32_t *bias,

  223.                       const ConvParameter *conv_param, int start_w, int end_w, int block_output_h, int block_output_w,

  224.                       int block_input_h, int block_input_w) {

  225.   int out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_[0];

  226.   int left_shift = conv_param->conv_quant_arg_.left_shift_[0];

  227.   int right_shift = conv_param->conv_quant_arg_.right_shift_[0];

  228.   int in_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  229.   int out_zp = conv_param->conv_quant_arg_.output_quant_args_[0].zp_;

  230.   int acc_min = conv_param->conv_quant_arg_.out_act_min_[0];

  231.   int acc_max = conv_param->conv_quant_arg_.out_act_max_[0];

  232. 

  233.   int ih_offset = 64 * block_input_w;

  234.   int w = start_w;

  235.   if (conv_param->output_channel_ > 64 || (conv_param->output_channel_ < 64 && conv_param->input_w_ > 150)) {

  236.     for (; w <= end_w - block_output_w; w += block_output_w) {

  237.       int8_t *output_ptr = output;

  238.       const int8_t *input_ptr = input;

  239.       const int16_t *weight_ptr = weight;

  240.       const int32_t *bias_ptr = bias;

  241.       int c = 0;

  242.       for (; c <= conv_param->output_channel_ - 64; c += 64) {

  243.         InitInputBuffer(buffer, input_ptr, conv_param, block_input_h, block_input_w);

  244.         ConvDw3x3Int8Block(output_ptr, buffer, weight_ptr, bias_ptr, 0, 64, 64, ih_offset, conv_param->input_channel_,

  245.                            block_output_h, block_output_w, in_zp, out_zp, out_multiplier, left_shift, right_shift,

  246.                            acc_min, acc_max, conv_param->stride_h_);

  247.         output_ptr += 64;

  248.         input_ptr += 64;

  249.         weight_ptr += 64;

  250.         bias_ptr += 64;

  251.       }

  252.       // left channel

  253.       ConvDw3x3Int8Block(output_ptr, input_ptr, weight_ptr, bias_ptr, c, conv_param->input_channel_,

  254.                          conv_param->input_channel_, conv_param->input_w_ * conv_param->input_channel_,

  255.                          conv_param->input_channel_, block_output_h, block_output_w, in_zp, out_zp, out_multiplier,

  256.                          left_shift, right_shift, acc_min, acc_max, conv_param->stride_h_);

  257.       output += block_output_w * conv_param->input_channel_;

  258.       input += conv_param->stride_w_ * block_output_w * conv_param->input_channel_;

  259.     }

  260.   }

  261.   // left width

  262.   int left_width = end_w - w;

  263.   if (left_width > 0) {

  264.     ConvDw3x3Int8Block(output, input, weight, bias, 0, conv_param->input_channel_, conv_param->input_channel_,

  265.                        conv_param->input_w_ * conv_param->input_channel_, conv_param->input_channel_, block_output_h,

  266.                        left_width, in_zp, out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max,

  267.                        conv_param->stride_h_);

  268.   }

  269. }

  270. 

  271. void ConvDw3x3Int8(int8_t *output_data, int8_t *buffer, const int8_t *input_data, const int16_t *weight_data,

  272.                    const int32_t *bias_data, const ConvParameter *conv_param, const SlidingWindowParam *sliding,

  273.                    int task_id) {

  274.   int output_h = sliding->bottom_ - sliding->top_;

  275.   int step_oh = UP_DIV(output_h, conv_param->thread_num_);

  276.   int start_oh = step_oh * task_id + sliding->top_;

  277.   int end_oh = MSMIN(start_oh + step_oh, sliding->bottom_);

  278.   int start_ow = sliding->left_;

  279.   int end_ow = sliding->right_;

  280. 

  281.   int block_output_h = 1;

  282.   int block_output_w = conv_param->stride_w_ == 1 ? 30 : 14;

  283.   int block_input_h = 3;

  284.   int block_input_w = conv_param->stride_w_ * (block_output_w - 1) + 3;

  285. 

  286.   for (int b = 0; b < conv_param->output_batch_; b++) {

  287.     int start_ih = start_oh * conv_param->stride_h_ - conv_param->pad_u_;

  288.     int start_iw = start_ow * conv_param->stride_w_ - conv_param->pad_l_;

  289.     const int8_t *src = input_data + b * conv_param->input_h_ * conv_param->input_w_ * conv_param->input_channel_ +

  290.                         start_ih * conv_param->input_w_ * conv_param->input_channel_ +

  291.                         start_iw * conv_param->input_channel_;

  292.     int8_t *dst = output_data + b * conv_param->output_h_ * conv_param->output_w_ * conv_param->output_channel_ +

  293.                   start_oh * conv_param->output_w_ * conv_param->output_channel_ +

  294.                   start_ow * conv_param->output_channel_;

  295. 

  296.     for (int oh = start_oh; oh < end_oh; oh++) {

  297.       ConvDw3x3Int8Row(dst, buffer, src, weight_data, bias_data, conv_param, start_ow, end_ow, block_output_h,

  298.                        block_output_w, block_input_h, block_input_w);

  299.       src += conv_param->stride_h_ * conv_param->input_w_ * conv_param->input_channel_;

  300.       dst += conv_param->output_w_ * conv_param->output_channel_;

  301.     }

  302.   }

  303. }

  304. 

  305. void ConvDw3x3Int8BorderPixel(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int height,

  306.                               int width, int in_kh_step, int in_kw_step, int channel, int8_t in_zp, int32_t out_zp,

  307.                               int out_multiplier, int left_shift, int right_shift, int32_t acc_min, int32_t acc_max) {

  308.   for (int c = 0; c < channel; c += 8) {

  309.     int tmp_buffer[8];

  310.     for (int i = 0; i < 8; i++) {

  311.       tmp_buffer[i] = 0;

  312.     }

  313.     const int8_t *src_kh = src;

  314.     const int16_t *weight_kh = weight;

  315.     for (int kh = 0; kh < height; kh++) {

  316.       const int8_t *src_kw = src_kh;

  317.       const int16_t *weight_kw = weight_kh;

  318.       for (int kw = 0; kw < width; kw++) {

  319.         for (int i = 0; i < 8; i++) {

  320.           tmp_buffer[i] += (src_kw[c + i] - in_zp) * weight_kw[c + i];

  321.         }

  322.         src_kw += in_kw_step;

  323.         weight_kw += channel;

  324.       }  // kernel_w loop

  325.       src_kh += in_kh_step;

  326.       weight_kh += 3 * channel;

  327.     }  // kernel_h loop

  328. 

  329.     for (int i = 0; i < 8; i++) {

  330.       tmp_buffer[i] += bias[c + i];

  331.       tmp_buffer[i] = RoundingDivideByPOT(

  332.         SaturatingRoundingDoublingHighMul(tmp_buffer[i] * (1 << (unsigned int)left_shift), out_multiplier),

  333.         -right_shift);

  334.       tmp_buffer[i] += out_zp;

  335.       tmp_buffer[i] = MSMAX(tmp_buffer[i], acc_min);

  336.       tmp_buffer[i] = MSMIN(tmp_buffer[i], acc_max);

  337.       dst[c + i] = (tmp_buffer[i]);

  338.     }

  339.   }

  340. }

  341. 

  342. #ifndef ENABLE_ARM64

  343. void ConvDw3x3Int8Corner(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int in_kh_step,

  344.                          int in_kw_step, int channel, int8_t in_zp, int32_t out_zp, int out_multiplier, int left_shift,

  345.                          int right_shift, int32_t acc_min, int32_t acc_max) {

  346.   ConvDw3x3Int8BorderPixel(dst, src, weight, bias, 2, 2, in_kh_step, in_kw_step, channel, in_zp, out_zp, out_multiplier,

  347.                            left_shift, right_shift, acc_min, acc_max);

  348. }

  349. 

  350. void ConvDw3x3Int8Vertical(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int in_kh_step,

  351.                            int in_kw_step, int channel, int8_t in_zp, int32_t out_zp, int out_multiplier,

  352.                            int left_shift, int right_shift, int32_t acc_min, int32_t acc_max) {

  353.   ConvDw3x3Int8BorderPixel(dst, src, weight, bias, 2, 3, in_kh_step, in_kw_step, channel, in_zp, out_zp, out_multiplier,

  354.                            left_shift, right_shift, acc_min, acc_max);

  355. }

  356. void ConvDw3x3Int8Horizontal(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int in_kh_step,

  357.                              int in_kw_step, int channel, int8_t in_zp, int32_t out_zp, int out_multiplier,

  358.                              int left_shift, int right_shift, int32_t acc_min, int32_t acc_max) {

  359.   ConvDw3x3Int8BorderPixel(dst, src, weight, bias, 3, 2, in_kh_step, in_kw_step, channel, in_zp, out_zp, out_multiplier,

  360.                            left_shift, right_shift, acc_min, acc_max);

  361. }

  362. #endif

  363. 

  364. void ConvDw3x3Int8Border(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int top,

  365.                          int bottom, int left, int right, const ConvParameter *conv_param,

  366.                          const SlidingWindowParam *sliding, int8_t in_zp, int32_t out_zp, int out_multiplier,

  367.                          int left_shift, int right_shift, int32_t acc_min, int32_t acc_max) {

  368.   int8_t *dst_h = dst + top * sliding->out_h_step_;

  369.   for (int oh = top; oh < bottom; oh++) {

  370.     int ih = oh * conv_param->stride_h_ - conv_param->pad_u_;

  371.     int start_kh = MSMAX(0, UP_DIV(-ih, conv_param->dilation_h_));

  372.     int end_kh = MSMIN(conv_param->kernel_h_, UP_DIV(conv_param->input_h_ - ih, conv_param->dilation_h_));

  373.     const int8_t *src_h = src + ih * sliding->in_h_step_;

  374. 

  375.     int8_t *dst_kernel = dst_h + left * sliding->block_channel_;

  376.     for (int ow = left; ow < right; ow++) {

  377.       int iw = ow * conv_param->stride_w_ - conv_param->pad_l_;

  378.       int start_kw = MSMAX(0, UP_DIV(-iw, conv_param->dilation_w_));

  379.       int end_kw = MSMIN(conv_param->kernel_w_, UP_DIV(conv_param->input_w_ - iw, conv_param->dilation_w_));

  380.       const int8_t *src_w = src_h + iw * sliding->block_channel_;

  381. 

  382.       const int8_t *src_kernel = src_w + start_kh * sliding->in_kh_step_ + start_kw * sliding->in_kw_step_;

  383.       const int16_t *weight_kernel =

  384.         weight + (start_kh * conv_param->kernel_w_ + start_kw) * conv_param->input_channel_;

  385. 

  386.       ConvDw3x3Int8BorderPixel(dst_kernel, src_kernel, weight_kernel, bias, end_kh - start_kh, end_kw - start_kw,

  387.                                sliding->in_kh_step_, sliding->in_kw_step_, conv_param->input_channel_, in_zp, out_zp,

  388.                                out_multiplier, left_shift, right_shift, acc_min, acc_max);

  389. 

  390.       dst_kernel += sliding->block_channel_;

  391.     }  // width loop

  392.     dst_h += sliding->out_h_step_;

  393.   }  // height loop

  394. }

  395. 

  396. void ConvDw3x3Int8Pad(int8_t *output_data, const int8_t *input_data, const int16_t *weight_data,

  397.                       const int32_t *bias_data, const ConvParameter *conv_param, const SlidingWindowParam *sliding) {

  398.   int out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_[0];

  399.   int left_shift = conv_param->conv_quant_arg_.left_shift_[0];

  400.   int right_shift = conv_param->conv_quant_arg_.right_shift_[0];

  401.   int in_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  402.   int out_zp = conv_param->conv_quant_arg_.output_quant_args_[0].zp_;

  403.   int acc_min = conv_param->conv_quant_arg_.out_act_min_[0];

  404.   int acc_max = conv_param->conv_quant_arg_.out_act_max_[0];

  405.   int input_row_size = conv_param->input_w_ * conv_param->input_channel_;

  406.   int weight_row_size = conv_param->kernel_w_ * conv_param->input_channel_;

  407.   int output_row_size = conv_param->output_w_ * conv_param->output_channel_;

  408.   int in_kh_step = sliding->in_kh_step_;

  409.   int in_kw_step = sliding->in_kw_step_;

  410. 

  411.   // top

  412.   const int8_t *input = input_data;

  413.   const int16_t *weight = weight_data + weight_row_size + conv_param->input_channel_;

  414.   int8_t *output = output_data;

  415.   ConvDw3x3Int8Corner(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  416.                       out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  417.   input += (conv_param->stride_w_ - 1) * conv_param->input_channel_;

  418.   weight = weight_data + weight_row_size;

  419.   output += conv_param->output_channel_;

  420.   for (int out_w = sliding->left_; out_w < sliding->right_; out_w++) {

  421.     ConvDw3x3Int8Vertical(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  422.                           out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  423.     input += conv_param->stride_w_ * conv_param->input_channel_;

  424.     output += conv_param->output_channel_;

  425.   }

  426.   ConvDw3x3Int8Corner(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  427.                       out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  428. 

  429.   // left

  430.   input = input_data + (conv_param->stride_h_ - 1) * input_row_size;

  431.   weight = weight_data + conv_param->input_channel_;

  432.   output = output_data + output_row_size;

  433.   for (int out_h = sliding->top_; out_h < sliding->bottom_; out_h++) {

  434.     ConvDw3x3Int8Horizontal(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  435.                             out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  436.     input += conv_param->stride_h_ * input_row_size;

  437.     output += output_row_size;

  438.   }

  439. 

  440.   // right

  441.   input =

  442.     input_data + (conv_param->input_w_ - 2) * conv_param->input_channel_ + (conv_param->stride_h_ - 1) * input_row_size;

  443.   weight = weight_data;

  444.   output = output_data + output_row_size + (conv_param->output_w_ - 1) * conv_param->output_channel_;

  445.   for (int out_h = sliding->top_; out_h < sliding->bottom_; out_h++) {

  446.     ConvDw3x3Int8Horizontal(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  447.                             out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  448.     input += conv_param->stride_h_ * input_row_size;

  449.     output += output_row_size;

  450.   }

  451. 

  452.   // bottom

  453.   input = input_data + (conv_param->input_h_ - 2) * input_row_size;

  454.   weight = weight_data + conv_param->input_channel_;

  455.   output = output_data + (conv_param->output_h_ - 1) * output_row_size;

  456.   ConvDw3x3Int8Corner(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  457.                       out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  458.   input += conv_param->stride_w_ == 1 ? 0 : conv_param->input_channel_;

  459.   weight = weight_data;

  460.   output += conv_param->output_channel_;

  461.   for (int out_w = sliding->left_; out_w < sliding->right_; out_w++) {

  462.     ConvDw3x3Int8Vertical(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  463.                           out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  464.     input += conv_param->stride_w_ * conv_param->input_channel_;

  465.     output += conv_param->output_channel_;

  466.   }

  467.   ConvDw3x3Int8Corner(output, input, weight, bias_data, in_kh_step, in_kw_step, conv_param->input_channel_, in_zp,

  468.                       out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  469. }

  470. /*conv depthwise 3x3 int8 end*/

  471. 

  472. /*conv depthwise sliding window perchannel int8 begin*/

  473. void DepthwiseBorderPixelInt8(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int height,

  474.                               int width, int in_kh_step, int in_kw_step, int kernel_w, int8_t *input_zp,

  475.                               int32_t *out_zp, int *out_multiplier, int *left_shift, int *right_shift, int32_t *acc_min,

  476.                               int32_t *acc_max) {

  477.   int tmp_buffer[C8NUM];

  478.   for (int i = 0; i < C8NUM; i++) {

  479.     tmp_buffer[i] = 0;

  480.   }

  481.   const int8_t *src_kh = src;

  482.   const int16_t *weight_kh = weight;

  483.   for (int kh = 0; kh < height; kh++) {

  484.     const int8_t *src_kw = src_kh;

  485.     const int16_t *weight_kw = weight_kh;

  486.     for (int kw = 0; kw < width; kw++) {

  487.       for (int c = 0; c < C8NUM; c++) {

  488.         tmp_buffer[c] += (src_kw[c] - input_zp[c]) * weight_kw[c];

  489.       }

  490.       src_kw += in_kw_step;

  491.       weight_kw += C8NUM;

  492.     }  // kernel_w loop

  493.     src_kh += in_kh_step;

  494.     weight_kh += kernel_w * C8NUM;

  495.   }  // kernel_h loop

  496. 

  497.   for (int c = 0; c < C8NUM; c++) {

  498.     tmp_buffer[c] += bias[c];

  499.     tmp_buffer[c] = RoundingDivideByPOT(

  500.       SaturatingRoundingDoublingHighMul(tmp_buffer[c] * (1 << (unsigned int)left_shift[c]), out_multiplier[c]),

  501.       -right_shift[c]);

  502.     tmp_buffer[c] += out_zp[c];

  503.     tmp_buffer[c] = MSMAX(tmp_buffer[c], acc_min[c]);

  504.     tmp_buffer[c] = MSMIN(tmp_buffer[c], acc_max[c]);

  505.     dst[c] = (tmp_buffer[c]);

  506.   }

  507. }

  508. 

  509. void DepthwiseBorderInt8(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int top,

  510.                          int bottom, int left, int right, const ConvParameter *conv_param,

  511.                          const SlidingWindowParam *sliding, int8_t *in_zp, int32_t *out_zp, int *out_multiplier,

  512.                          int *left_shift, int *right_shift, int32_t *acc_min, int32_t *acc_max) {

  513.   int8_t *dst_h = dst + top * sliding->out_h_step_;

  514.   for (int oh = top; oh < bottom; oh++) {

  515.     int ih = oh * conv_param->stride_h_ - conv_param->pad_u_;

  516.     int start_kh = MSMAX(0, UP_DIV(-ih, conv_param->dilation_h_));

  517.     int end_kh = MSMIN(conv_param->kernel_h_, UP_DIV(conv_param->input_h_ - ih, conv_param->dilation_h_));

  518.     const int8_t *src_h = src + ih * sliding->in_h_step_;

  519. 

  520.     int8_t *dst_kernel = dst_h + left * sliding->block_channel_;

  521.     for (int ow = left; ow < right; ow++) {

  522.       int iw = ow * conv_param->stride_w_ - conv_param->pad_l_;

  523.       int start_kw = MSMAX(0, UP_DIV(-iw, conv_param->dilation_w_));

  524.       int end_kw = MSMIN(conv_param->kernel_w_, UP_DIV(conv_param->input_w_ - iw, conv_param->dilation_w_));

  525.       const int8_t *src_w = src_h + iw * sliding->block_channel_;

  526. 

  527.       const int8_t *src_kernel = src_w + start_kh * sliding->in_kh_step_ + start_kw * sliding->in_kw_step_;

  528.       const int16_t *weight_kernel = weight + (start_kh * conv_param->kernel_w_ + start_kw) * C8NUM;

  529. 

  530.       DepthwiseBorderPixelInt8(dst_kernel, src_kernel, weight_kernel, bias, end_kh - start_kh, end_kw - start_kw,

  531.                                sliding->in_kh_step_, sliding->in_kw_step_, conv_param->kernel_w_, in_zp, out_zp,

  532.                                out_multiplier, left_shift, right_shift, acc_min, acc_max);

  533. 

  534.       dst_kernel += sliding->block_channel_;

  535.     }  // width loop

  536.     dst_h += sliding->out_h_step_;

  537.   }  // height loop

  538. }

  539. 

  540. #ifndef ENABLE_ARM64

  541. void DepthwiseCenterInt8(int8_t *dst, const int8_t *src, const int16_t *weight, const int32_t *bias, int height,

  542.                          int width, int kernel_h, int kernel_w, int out_h_step, int block_channel, int in_sh_step,

  543.                          int in_sw_step, int in_kh_step, int in_kw_step, int8_t *in_zp, int32_t *out_zp,

  544.                          int32_t *out_multiplier, int32_t *left_shift, int32_t *right_shift, int32_t *acc_min,

  545.                          int32_t *acc_max) {

  546.   int tmp_buffer[C8NUM];

  547.   int8_t *dst_h = dst;

  548.   const int8_t *src_h = src;

  549.   for (int oh = 0; oh < height; oh++) {

  550.     int8_t *dst_w = dst_h;

  551.     const int8_t *src_w = src_h;

  552.     for (int ow = 0; ow < width; ow++) {

  553.       const int8_t *src_kh = src_w;

  554.       const int16_t *weight_kh = weight;

  555. 

  556.       for (int i = 0; i < C8NUM; i++) {

  557.         tmp_buffer[i] = 0;

  558.       }

  559.       for (int kh = 0; kh < kernel_h; kh++) {

  560.         const int8_t *src_kw = src_kh;

  561.         const int16_t *weight_kw = weight_kh;

  562.         for (int kw = 0; kw < kernel_w; kw++) {

  563.           for (int c = 0; c < C8NUM; c++) {

  564.             tmp_buffer[c] += (src_kw[c] - in_zp[c]) * weight_kw[c];

  565.           }

  566.           src_kw += in_kw_step;

  567.           weight_kw += C8NUM;

  568.         }  // kernel_w loop

  569.         src_kh += in_kh_step;

  570.         weight_kh += kernel_w * C8NUM;

  571.       }  // kernel_h loop

  572.       // add bias relu

  573.       for (int c = 0; c < C8NUM; c++) {

  574.         tmp_buffer[c] += bias[c];

  575.         tmp_buffer[c] = RoundingDivideByPOT(

  576.           SaturatingRoundingDoublingHighMul(tmp_buffer[c] * (1 << (unsigned int)left_shift[c]), out_multiplier[c]),

  577.           -right_shift[c]);

  578.         tmp_buffer[c] += out_zp[c];

  579.         tmp_buffer[c] = MSMAX(tmp_buffer[c], acc_min[c]);

  580.         tmp_buffer[c] = MSMIN(tmp_buffer[c], acc_max[c]);

  581.         dst_w[c] = (tmp_buffer[c]);

  582.       }

  583.       dst_w += block_channel;

  584.       src_w += in_sw_step;

  585.     }  // dst_width loop

  586.     dst_h += out_h_step;

  587.     src_h += in_sh_step;

  588.   }  // dst_height loop

  589. }

  590. #endif

  591. 

  592. void ConvDwSWInt8(int8_t *output_data, const int8_t *input_data, const int16_t *weight_data, const int32_t *bias_data,

  593.                   int8_t *input_zp, int32_t *output_zp, const ConvParameter *conv_param,

  594.                   const SlidingWindowParam *sliding, int task_id) {

  595.   const int8_t *src = input_data;

  596.   int8_t *dst = output_data;

  597.   for (int b = 0; b < conv_param->output_batch_; b++) {

  598.     for (int oc = task_id; oc < sliding->c_block_; oc += conv_param->thread_num_) {

  599.       const int8_t *src_data = src + oc * C8NUM;

  600.       int8_t *dst_data = dst + oc * C8NUM;

  601.       const int16_t *weight = weight_data + oc * sliding->kernel_step_;

  602.       const int32_t *bias = bias_data + oc * C8NUM;

  603. 

  604.       int *out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_ + oc * C8NUM;

  605.       int *left_shift = conv_param->conv_quant_arg_.left_shift_ + oc * C8NUM;

  606.       int *right_shift = conv_param->conv_quant_arg_.right_shift_ + oc * C8NUM;

  607.       int *acc_min = conv_param->conv_quant_arg_.out_act_min_ + oc * C8NUM;

  608.       int *acc_max = conv_param->conv_quant_arg_.out_act_max_ + oc * C8NUM;

  609.       int8_t *in_zp = input_zp + oc * C8NUM;

  610.       int32_t *out_zp = output_zp + oc * C8NUM;

  611. 

  612.       DepthwiseBorderInt8(dst_data, src_data, weight, bias, 0, sliding->top_, 0, conv_param->output_w_, conv_param,

  613.                           sliding, in_zp, out_zp, out_multiplier, left_shift, right_shift, acc_min, acc_max);

  614.       DepthwiseBorderInt8(dst_data, src_data, weight, bias, sliding->bottom_, conv_param->output_h_, 0,

  615.                           conv_param->output_w_, conv_param, sliding, in_zp, out_zp, out_multiplier, left_shift,

  616.                           right_shift, acc_min, acc_max);

  617.       DepthwiseBorderInt8(dst_data, src_data, weight, bias, sliding->top_, sliding->bottom_, 0, sliding->left_,

  618.                           conv_param, sliding, in_zp, out_zp, out_multiplier, left_shift, right_shift, acc_min,

  619.                           acc_max);

  620.       DepthwiseBorderInt8(dst_data, src_data, weight, bias, sliding->top_, sliding->bottom_, sliding->right_,

  621.                           conv_param->output_w_, conv_param, sliding, in_zp, out_zp, out_multiplier, left_shift,

  622.                           right_shift, acc_min, acc_max);

  623. 

  624.       if (sliding->right_ > sliding->left_ && sliding->bottom_ > sliding->top_) {

  625.         int in_h_start = sliding->top_ * conv_param->stride_h_ - conv_param->pad_u_;

  626.         int in_w_start = sliding->left_ * conv_param->stride_w_ - conv_param->pad_l_;

  627.         const int8_t *in_t = src_data + in_h_start * sliding->in_h_step_ + in_w_start * sliding->block_channel_;

  628.         int8_t *out_t = dst_data + sliding->top_ * sliding->out_h_step_ + sliding->left_ * sliding->block_channel_;

  629. #ifdef ENABLE_ARM

  630.         ConvDwInt8Center(out_t, in_t, weight, bias, sliding->bottom_ - sliding->top_, sliding->right_ - sliding->left_,

  631.                          conv_param->kernel_h_, conv_param->kernel_w_, sliding->out_h_step_ * sizeof(int8_t),

  632.                          sliding->block_channel_ * sizeof(int8_t), sliding->in_sh_step_ * sizeof(int8_t),

  633.                          sliding->in_sw_step_ * sizeof(int8_t), sliding->in_kh_step_ * sizeof(int8_t),

  634.                          sliding->in_kw_step_ * sizeof(int8_t), in_zp, out_zp, out_multiplier, left_shift, right_shift,

  635.                          acc_min, acc_max);

  636. #else

  637.         DepthwiseCenterInt8(out_t, in_t, weight, bias, sliding->bottom_ - sliding->top_,

  638.                             sliding->right_ - sliding->left_, conv_param->kernel_h_, conv_param->kernel_w_,

  639.                             sliding->out_h_step_, sliding->block_channel_, sliding->in_sh_step_, sliding->in_sw_step_,

  640.                             sliding->in_kh_step_, sliding->in_kw_step_, in_zp, out_zp, out_multiplier, left_shift,

  641.                             right_shift, acc_min, acc_max);

  642. #endif

  643.       }

  644.     }  // output C8 loop

  645.     src += sliding->in_step_;

  646.     dst += sliding->out_step_;

  647.   }  // batch loop

  648.   // output nhwc8

  649. }

  650. /*conv depthwise sliding window perchannel int8 end*/

  651. 

  652. /*deconv depthwise int8 begin*/

  653. void DeconvDepthwiseBorderPixelInt8(int32_t *dst, const int16_t *src, const int16_t *weight, int height, int width,

  654.                                     int in_kh_step, int in_kw_step, int kernel_w) {

  655.   int32_t *dst_kh = dst;

  656.   const int16_t *weight_kh = weight;

  657.   for (int kh = 0; kh < height; kh++) {

  658.     int32_t *dst_kw = dst_kh;

  659.     const int16_t *weight_kw = weight_kh;

  660.     for (int kw = 0; kw < width; kw++) {

  661.       for (int c = 0; c < C4NUM; c++) {

  662.         dst_kw[c] += src[c] * weight_kw[c];

  663.       }

  664.       dst_kw += in_kw_step;

  665.       weight_kw += C4NUM;

  666.     }  // kernel_w loop

  667.     dst_kh += in_kh_step;

  668.     weight_kh += kernel_w * C4NUM;

  669.   }  // kernel_h loop

  670. }

  671. 

  672. void DeconvDepthwiseBorderInt8(int32_t *dst, const int16_t *src, const int16_t *weight, int top, int bottom, int left,

  673.                                int right, const ConvParameter *conv_param, const SlidingWindowParam *sliding) {

  674.   const int16_t *src_h = src + top * sliding->out_h_step_;

  675.   for (int ih = top; ih < bottom; ih++) {

  676.     int oh = ih * conv_param->stride_h_ - conv_param->pad_u_;

  677.     int start_kh = MSMAX(0, UP_DIV(-oh, conv_param->dilation_h_));

  678.     int end_kh = MSMIN(conv_param->kernel_h_, UP_DIV(conv_param->output_h_ - oh, conv_param->dilation_h_));

  679.     int32_t *dst_h = dst + oh * sliding->in_h_step_;

  680. 

  681.     const int16_t *src_kernel = src_h + left * sliding->block_channel_;

  682.     for (int iw = left; iw < right; iw++) {

  683.       int ow = iw * conv_param->stride_w_ - conv_param->pad_l_;

  684.       int start_kw = MSMAX(0, UP_DIV(-ow, conv_param->dilation_w_));

  685.       int end_kw = MSMIN(conv_param->kernel_w_, UP_DIV(conv_param->output_w_ - ow, conv_param->dilation_w_));

  686.       int32_t *dst_w = dst_h + ow * C4NUM;

  687. 

  688.       const int16_t *weight_kernel = weight + (start_kh * conv_param->kernel_w_ + start_kw) * C4NUM;

  689.       int32_t *dst_kernel = dst_w + start_kh * sliding->in_kh_step_ + start_kw * sliding->in_kw_step_;

  690. 

  691.       DeconvDepthwiseBorderPixelInt8(dst_kernel, src_kernel, weight_kernel, end_kh - start_kh, end_kw - start_kw,

  692.                                      sliding->in_kh_step_, sliding->in_kw_step_, conv_param->kernel_w_);

  693.       src_kernel += sliding->block_channel_;

  694.     }  // width loop

  695.     src_h += sliding->out_h_step_;

  696.   }  // height loop

  697. }

  698. 

  699. #ifndef ENABLE_ARM64

  700. void DeconvDepthwiseCenterInt8(int32_t *dst, const int16_t *src, const int16_t *weight, int height, int width,

  701.                                int kernel_h, int kernel_w, int out_h_step, int block_channel, int in_sh_step,

  702.                                int in_sw_step, int in_kh_step, int in_kw_step) {

  703.   int32_t *dst_h = dst;

  704.   const int16_t *src_h = src;

  705.   for (int oh = 0; oh < height; oh++) {

  706.     int32_t *dst_w = dst_h;

  707.     const int16_t *src_w = src_h;

  708.     for (int ow = 0; ow < width; ow++) {

  709.       int32_t *dst_kh = dst_w;

  710.       const int16_t *weight_kh = weight;

  711.       for (int kh = 0; kh < kernel_h; kh++) {

  712.         int32_t *dst_kw = dst_kh;

  713.         const int16_t *weight_kw = weight_kh;

  714.         for (int kw = 0; kw < kernel_w; kw++) {

  715.           for (int c = 0; c < C4NUM; c++) {

  716.             dst_kw[c] += src_w[c] * weight_kw[c];

  717.           }

  718.           dst_kw += in_kw_step;

  719.           weight_kw += C4NUM;

  720.         }  // kernel_w loop

  721.         dst_kh += in_kh_step;

  722.         weight_kh += kernel_w * C4NUM;

  723.       }  // kernel_h loop

  724.       dst_w += in_sw_step;

  725.       src_w += block_channel;

  726.     }  // dst_width loop

  727.     dst_h += in_sh_step;

  728.     src_h += out_h_step;

  729.   }  // dst_height loop

  730. }

  731. #endif

  732. 

  733. void DeconvDepthwisePostFuncInt8(int8_t *dst, int32_t *output_buffer, const int32_t *bias, int block_channel,

  734.                                  const ConvParameter *conv_param, int out_multiplier, int left_shift, int right_shift,

  735.                                  int32_t out_zp, int32_t acc_min, int32_t acc_max) {

  736.   int8_t *dst_k = dst;

  737.   int32_t *buffer_k = output_buffer;

  738.   for (int k = 0; k < conv_param->output_h_ * conv_param->output_w_; k++) {

  739.     for (int c = 0; c < C4NUM; c++) {

  740.       buffer_k[c] += bias[c];

  741.       buffer_k[c] = RoundingDivideByPOT(

  742.         SaturatingRoundingDoublingHighMul(buffer_k[c] * (1 << (unsigned int)left_shift), out_multiplier), -right_shift);

  743.       buffer_k[c] += out_zp;

  744.       buffer_k[c] = MSMAX(buffer_k[c], acc_min);

  745.       buffer_k[c] = MSMIN(buffer_k[c], acc_max);

  746.       dst_k[c] = (buffer_k[c]);

  747.     }

  748.     dst_k += block_channel;

  749.     buffer_k += C4NUM;

  750.   }

  751. }

  752. 

  753. void DeconvDwInt8(int8_t *output_data, int32_t *output_buffer, const int16_t *input_data, const int16_t *weight_data,

  754.                   const int32_t *bias_data, const ConvParameter *conv_param, const SlidingWindowParam *sliding,

  755.                   int task_id) {

  756.   const int16_t *src = input_data;

  757.   int8_t *dst = output_data;

  758.   int buffer_size = conv_param->output_h_ * conv_param->output_w_ * C4NUM;

  759.   for (int b = 0; b < conv_param->output_batch_; b++) {

  760.     for (int oc = task_id; oc < sliding->c_block_; oc += conv_param->thread_num_) {

  761.       memset(output_buffer, 0, buffer_size * sizeof(int32_t));

  762.       const int16_t *src_data = src + oc * C4NUM;

  763.       const int16_t *weight = weight_data + oc * sliding->kernel_step_;

  764.       const int32_t *bias = bias_data + oc * C4NUM;

  765.       int8_t *dst_data = dst + oc * C4NUM;

  766.       DeconvDepthwiseBorderInt8(output_buffer, src_data, weight, 0, sliding->top_, 0, conv_param->input_w_, conv_param,

  767.                                 sliding);

  768.       DeconvDepthwiseBorderInt8(output_buffer, src_data, weight, sliding->bottom_, conv_param->input_h_, 0,

  769.                                 conv_param->input_w_, conv_param, sliding);

  770.       DeconvDepthwiseBorderInt8(output_buffer, src_data, weight, sliding->top_, sliding->bottom_, 0, sliding->left_,

  771.                                 conv_param, sliding);

  772.       DeconvDepthwiseBorderInt8(output_buffer, src_data, weight, sliding->top_, sliding->bottom_, sliding->right_,

  773.                                 conv_param->input_w_, conv_param, sliding);

  774. 

  775.       if (sliding->right_ > sliding->left_ && sliding->bottom_ > sliding->top_) {

  776.         int oh_h_start = sliding->top_ * conv_param->stride_h_ - conv_param->pad_u_;

  777.         int oh_w_start = sliding->left_ * conv_param->stride_w_ - conv_param->pad_l_;

  778.         int32_t *out_t = output_buffer + oh_h_start * sliding->in_h_step_ + oh_w_start * sliding->block_channel_;

  779.         const int16_t *in_t =

  780.           src_data + sliding->top_ * sliding->out_h_step_ + sliding->left_ * sliding->block_channel_;

  781. #ifdef ENABLE_ARM64

  782.         DeconvDwInt8Center(out_t, in_t, weight, sliding->bottom_ - sliding->top_, sliding->right_ - sliding->left_,

  783.                            conv_param->kernel_h_, conv_param->kernel_w_, sliding->out_h_step_ * sizeof(int16_t),

  784.                            sliding->block_channel_ * sizeof(int16_t), sliding->in_sh_step_ * sizeof(int32_t),

  785.                            sliding->in_sw_step_ * sizeof(int32_t), sliding->in_kh_step_ * sizeof(int32_t),

  786.                            sliding->in_kw_step_ * sizeof(int32_t));

  787. #else

  788.         DeconvDepthwiseCenterInt8(out_t, in_t, weight, sliding->bottom_ - sliding->top_,

  789.                                   sliding->right_ - sliding->left_, conv_param->kernel_h_, conv_param->kernel_w_,

  790.                                   sliding->out_h_step_, sliding->block_channel_, sliding->in_sh_step_,

  791.                                   sliding->in_sw_step_, sliding->in_kh_step_, sliding->in_kw_step_);

  792. #endif

  793.       }

  794.       DeconvDepthwisePostFuncInt8(

  795.         dst_data, output_buffer, bias, sliding->block_channel_, conv_param,

  796.         conv_param->conv_quant_arg_.quant_multiplier_[0], conv_param->conv_quant_arg_.left_shift_[0],

  797.         conv_param->conv_quant_arg_.right_shift_[0], conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  798.         conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0]);

  799.     }  // output C4 loop

  800.     src += sliding->in_step_;

  801.     dst += sliding->out_step_;

  802.   }  // batch loop

  803.   // output nhwc4

  804. }

  805. /*deconv depthwise int8 end*/