Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c0f3baf842
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c0f3baf842'
${ noResults }
mindspore/mindspore/lite/nnacl/int8/conv_int8.c

893 lines
34 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/int8/conv_int8.h"

  18. #include <string.h>

  19. #include "nnacl/winograd_transform.h"

  20. #include "nnacl/int8/common_func.h"

  21. 

  22. void Conv3x3Int8Gemm(int32_t *dst, const int16_t *src, const int16_t *weight, int oc, int ic8, size_t real_cal_num) {

  23.   int oc4 = UP_DIV(oc, C4NUM);

  24. #ifdef ENABLE_ARM

  25.   IndirectGemmInt16to32_8x4(dst, src, weight, 16, ic8, oc4, oc4 * 4 * 16 * sizeof(int32_t));

  26. #else

  27.   const int input_unit_square = 16;

  28.   for (int c = 0; c < oc4; c++) {

  29.     int filter_oc_offset = c * input_unit_square * ic8 * C8NUM * C4NUM;

  30.     int dst_oc_offset = c * input_unit_square * C4NUM;

  31.     for (int n = 0; n < real_cal_num; n++) {

  32.       int src_tile_offset = n * C8NUM;

  33.       int dst_tile_offset = dst_oc_offset + n * oc4 * C4NUM * input_unit_square;

  34.       for (int i = 0; i < 4; i++) {

  35.         int filter_h_offset = filter_oc_offset + i * 4 * ic8 * C8NUM * C4NUM;

  36.         int src_h_offset = src_tile_offset + i * C8NUM * ic8 * C8NUM * C4NUM;

  37.         int dst_h_offset = dst_tile_offset + i * 4 * 4;

  38.         for (int m = 0; m < 4; m++) {

  39.           int filter_w_offset = filter_h_offset + m * 4 * C8NUM * ic8;

  40.           int src_w_offset = src_h_offset + m * 8 * ic8 * C8NUM;

  41.           int dst_w_offset = dst_h_offset + m * C4NUM;

  42. 

  43.           int32_t acc[4] = {0};

  44.           for (int z = 0; z < 4; z++) {

  45.             int filter_offset = filter_w_offset + z;

  46.             for (int j = 0; j < ic8; j++) {

  47.               int filter_c8_offset = filter_offset + j * 4 * 8;

  48.               int src_c8_offset = src_w_offset + j * 8 * 8;

  49. 

  50.               for (int k = 0; k < 8; k++) {

  51.                 const int16_t *w_ptr = weight + filter_c8_offset + k * 4;

  52.                 const int16_t *input_ptr = src + src_c8_offset + k;

  53.                 acc[z] += w_ptr[0] * input_ptr[0];

  54.               }

  55.             }

  56.             (dst + dst_w_offset + z)[0] = acc[z];

  57.           }

  58.         }

  59.       }

  60.     }

  61.   }

  62. #endif

  63. }

  64. 

  65. void ConvInt8(int8_t *input_data, int8_t *packed_input, int8_t *matmul_input, int8_t *packed_weight,

  66.               const int32_t *bias_data, int8_t *output_data, int32_t *filter_zp, int32_t *input_sum, int task_id,

  67.               ConvParameter *conv_param, MATMUL_OPT_R_FUNC matmul_func, bool is_optimize) {

  68.   int kernel_h = conv_param->kernel_h_;

  69.   int kernel_w = conv_param->kernel_w_;

  70.   int in_batch = conv_param->input_batch_;

  71.   int in_channel = conv_param->input_channel_;

  72.   int in_h = conv_param->input_h_;

  73.   int in_w = conv_param->input_w_;

  74.   int out_h = conv_param->output_h_;

  75.   int out_w = conv_param->output_w_;

  76.   int out_channel = conv_param->output_channel_;

  77.   int tile_n = conv_param->tile_num_;

  78.   int thread_count = conv_param->thread_num_;

  79.   int output_count = out_h * out_w;

  80.   int output_tile_count = UP_DIV(output_count, tile_n);

  81.   int kernel_plane = kernel_h * kernel_w;

  82.   int unit_size;

  83.   int input_sum_offset;

  84.   int up_round_oc;

  85. #ifdef ENABLE_ARM32

  86.   up_round_oc = UP_ROUND(out_channel, C2NUM);

  87.   unit_size = UP_ROUND(kernel_plane * in_channel, C16NUM);

  88. #else

  89.   if (is_optimize) {

  90.     up_round_oc = UP_ROUND(out_channel, C8NUM);

  91.     unit_size = UP_ROUND(kernel_plane * in_channel, C4NUM);

  92.   } else {

  93.     up_round_oc = UP_ROUND(out_channel, C4NUM);

  94.     unit_size = UP_ROUND(kernel_plane * in_channel, C16NUM);

  95.   }

  96. #endif

  97.   bool per_channel;

  98.   if (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL) {

  99.     input_sum_offset = tile_n * up_round_oc;

  100.     per_channel = true;

  101.   } else {

  102.     input_sum_offset = tile_n;

  103.     per_channel = false;

  104.   }

  105. 

  106.   for (int b = 0; b < in_batch; b++) {

  107.     int in_batch_offset = b * in_channel * in_h * in_w;

  108.     int out_batch_offset = b * out_channel * out_h * out_w;

  109.     for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {

  110.       int start_index = thread_id * tile_n;

  111.       int real_cal_num = (output_count - start_index) < tile_n ? (output_count - start_index) : tile_n;

  112.       int32_t *tmp_input_sum = input_sum + task_id * input_sum_offset;

  113.       int8_t *gemm_input = packed_input + task_id * unit_size * tile_n;

  114.       int8_t *matmul = matmul_input + task_id * kernel_plane * in_channel * tile_n;

  115.       memset(matmul, conv_param->conv_quant_arg_.input_quant_args_[0].zp_, kernel_plane * in_channel * tile_n);

  116.       Im2ColPackUnitInt8Opt(input_data + in_batch_offset, gemm_input, matmul, real_cal_num, start_index, filter_zp,

  117.                             tmp_input_sum, conv_param, per_channel, is_optimize);

  118. 

  119.       int out_offset = thread_id * tile_n * out_channel + out_batch_offset;

  120.       int8_t *gemm_output = output_data + out_offset;

  121. #ifdef ENABLE_ARM32

  122.       MatmulInt8Neon32(

  123.         gemm_input, packed_weight, gemm_output, real_cal_num, out_channel, unit_size, tmp_input_sum, bias_data,

  124.         conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  125.         conv_param->conv_quant_arg_.output_quant_args_[0].zp_, conv_param->conv_quant_arg_.quant_multiplier_,

  126.         conv_param->conv_quant_arg_.left_shift_, conv_param->conv_quant_arg_.right_shift_, out_channel, per_channel);

  127. #elif ENABLE_ARM64

  128.       if (is_optimize) {

  129.         matmul_func(gemm_input, packed_weight, gemm_output, real_cal_num, out_channel, unit_size, out_channel,

  130.                     tmp_input_sum, bias_data, conv_param->conv_quant_arg_.left_shift_,

  131.                     conv_param->conv_quant_arg_.right_shift_, conv_param->conv_quant_arg_.quant_multiplier_,

  132.                     conv_param->conv_quant_arg_.output_quant_args_[0].zp_, conv_param->conv_quant_arg_.out_act_min_[0],

  133.                     conv_param->conv_quant_arg_.out_act_max_[0], per_channel);

  134.       } else {

  135.         MatmulInt8Neon64(gemm_input, packed_weight, gemm_output, UP_ROUND(real_cal_num, C4NUM),

  136.                          UP_ROUND(out_channel, C4NUM), unit_size, tmp_input_sum, bias_data,

  137.                          conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  138.                          conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  139.                          conv_param->conv_quant_arg_.quant_multiplier_, conv_param->conv_quant_arg_.left_shift_,

  140.                          conv_param->conv_quant_arg_.right_shift_, real_cal_num, out_channel, out_channel, per_channel);

  141.       }

  142. #else

  143.       MatMulInt8_8x8_r(

  144.         gemm_input, packed_weight, gemm_output, real_cal_num, out_channel, unit_size, out_channel, tmp_input_sum,

  145.         bias_data, conv_param->conv_quant_arg_.left_shift_, conv_param->conv_quant_arg_.right_shift_,

  146.         conv_param->conv_quant_arg_.quant_multiplier_, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  147.         conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0], per_channel);

  148. #endif

  149.     }

  150.   }

  151. }

  152. 

  153. void Conv1x1PreOptPeroc(const int8_t *src_input, int8_t *packed_input, int32_t *input_sum, size_t input_channel,

  154.                         size_t output_channel, size_t plane_size, int32_t *filter_zp, size_t inputsum_stride) {

  155.   int ic4 = UP_ROUND(input_channel, C4NUM);

  156.   int oc8 = UP_ROUND(output_channel, C8NUM);

  157.   int hw8 = UP_ROUND(plane_size, C8NUM);

  158.   size_t hw_8div = plane_size / C8NUM * C8NUM;

  159.   size_t oc_8div = output_channel / C8NUM * C8NUM;

  160.   size_t oc_8res = output_channel - oc_8div;

  161.   size_t ic_4div = input_channel / C4NUM * C4NUM;

  162. 

  163.   const int8_t *src_r = src_input;

  164.   int8_t *pack_r = packed_input;

  165.   int32_t *input_sum_r = input_sum;

  166. 

  167.   for (int hwi = 0; hwi < hw_8div; hwi += C8NUM) {

  168.     const int8_t *src_ic = src_r;

  169.     int8_t *pack_ic = pack_r;

  170.     int32_t *input_sum_oc = input_sum_r;

  171. #ifdef ENABLE_ARM64

  172.     size_t src_stride = input_channel;

  173.     size_t ic_4res = input_channel - ic_4div;

  174.     size_t input_sum_stride = inputsum_stride * 4 - C8NUM * C8NUM * 4;

  175.     asm volatile(

  176.       "dup v16.4s, wzr \n"

  177.       "dup v17.4s, wzr \n"

  178. 

  179.       "mov x10, %[src_ic] \n"

  180.       "mov x11, %[pack_ic] \n"

  181. 

  182.       "mov x0, #0 \n"

  183.       "1: \n"

  184.       "cmp x0, %[ic_4div] \n"

  185.       "add x0, x0, #4\n"

  186.       "mov x12, x10 \n"

  187.       "add x10, x10, #4\n"

  188.       "blt 2f \n"

  189.       "cmp %[ic_4res], #0\n"

  190.       "beq 6f \n"

  191.       "cmp %[ic_4res], #1\n"

  192.       "beq 3f \n"

  193.       "cmp %[ic_4res], #2\n"

  194.       "beq 4f \n"

  195.       "cmp %[ic_4res], #3\n"

  196.       "beq 5f \n"

  197. 

  198.       "2: \n"

  199.       "ld1 {v0.s}[0], [x12], %[src_stride]\n"

  200.       "ld1 {v0.s}[1], [x12], %[src_stride]\n"

  201.       "ld1 {v0.s}[2], [x12], %[src_stride]\n"

  202.       "ld1 {v0.s}[3], [x12], %[src_stride]\n"

  203.       "ld1 {v1.s}[0], [x12], %[src_stride]\n"

  204.       "ld1 {v1.s}[1], [x12], %[src_stride]\n"

  205.       "ld1 {v1.s}[2], [x12], %[src_stride]\n"

  206.       "ld1 {v1.s}[3], [x12], %[src_stride]\n"

  207. 

  208.       "st1 {v0.16b}, [x11], #16\n"

  209.       "st1 {v1.16b}, [x11], #16\n"

  210. 

  211.       "saddlp v4.8h, v0.16b \n"

  212.       "saddlp v5.8h, v1.16b \n"

  213.       "saddlp v0.4s, v4.8h \n"

  214.       "saddlp v1.4s, v5.8h \n"

  215.       "add v16.4s, v16.4s, v0.4s \n"

  216.       "add v17.4s, v17.4s, v1.4s \n"

  217.       "b 1b \n"

  218. 

  219.       "3: \n" /* col res 1 */

  220.       "dup v0.4s, wzr \n"

  221.       "dup v1.4s, wzr \n"

  222. 

  223.       "ld1 {v0.b}[0],  [x12], %[src_stride]\n"

  224.       "ld1 {v0.b}[4],  [x12], %[src_stride]\n"

  225.       "ld1 {v0.b}[8],  [x12], %[src_stride]\n"

  226.       "ld1 {v0.b}[12], [x12], %[src_stride]\n"

  227.       "ld1 {v1.b}[0],  [x12], %[src_stride]\n"

  228.       "ld1 {v1.b}[4],  [x12], %[src_stride]\n"

  229.       "ld1 {v1.b}[8],  [x12], %[src_stride]\n"

  230.       "ld1 {v1.b}[12], [x12], %[src_stride]\n"

  231. 

  232.       "st1 {v0.16b}, [x11], #16\n"

  233.       "st1 {v1.16b}, [x11], #16\n"

  234. 

  235.       "saddlp v4.8h, v0.16b \n"

  236.       "saddlp v5.8h, v1.16b \n"

  237.       "saddlp v0.4s, v4.8h \n"

  238.       "saddlp v1.4s, v5.8h \n"

  239.       "add v16.4s, v16.4s, v0.4s \n"

  240.       "add v17.4s, v17.4s, v1.4s \n"

  241.       "b 6f \n"

  242. 

  243.       "4: \n" /* col res 2 */

  244.       "dup v0.4s, wzr \n"

  245.       "dup v1.4s, wzr \n"

  246. 

  247.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  248.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  249.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  250.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  251.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  252.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  253.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  254.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  255. 

  256.       "st1 {v0.16b}, [x11], #16\n"

  257.       "st1 {v1.16b}, [x11], #16\n"

  258. 

  259.       "saddlp v4.8h, v0.16b \n"

  260.       "saddlp v5.8h, v1.16b \n"

  261.       "saddlp v0.4s, v4.8h \n"

  262.       "saddlp v1.4s, v5.8h \n"

  263.       "add v16.4s, v16.4s, v0.4s \n"

  264.       "add v17.4s, v17.4s, v1.4s \n"

  265.       "b 6f \n"

  266. 

  267.       "5: \n" /* col res 3 */

  268.       "dup v0.4s, wzr \n"

  269.       "dup v1.4s, wzr \n"

  270.       "add x13, x12, #2 \n"

  271. 

  272.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  273.       "ld1 {v0.b}[2], [x13], %[src_stride]\n"

  274.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  275.       "ld1 {v0.b}[6], [x13], %[src_stride]\n"

  276.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  277.       "ld1 {v0.b}[10], [x13], %[src_stride]\n"

  278.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  279.       "ld1 {v0.b}[14], [x13], %[src_stride]\n"

  280.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  281.       "ld1 {v1.b}[2], [x13], %[src_stride]\n"

  282.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  283.       "ld1 {v1.b}[6], [x13], %[src_stride]\n"

  284.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  285.       "ld1 {v1.b}[10], [x13], %[src_stride]\n"

  286.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  287.       "ld1 {v1.b}[14], [x13], %[src_stride]\n"

  288. 

  289.       "st1 {v0.16b}, [x11], #16\n"

  290.       "st1 {v1.16b}, [x11], #16\n"

  291. 

  292.       "saddlp v4.8h, v0.16b \n"

  293.       "saddlp v5.8h, v1.16b \n"

  294.       "saddlp v0.4s, v4.8h \n"

  295.       "saddlp v1.4s, v5.8h \n"

  296.       "add v16.4s, v16.4s, v0.4s \n"

  297.       "add v17.4s, v17.4s, v1.4s \n"

  298.       "b 6f \n"

  299. 

  300.       "6: \n"

  301.       "dup v0.4s, v16.s[0]  \n"

  302.       "dup v1.4s, v16.s[1]  \n"

  303.       "dup v2.4s, v16.s[2]  \n"

  304.       "dup v3.4s, v16.s[3]  \n"

  305.       "dup v4.4s, v17.s[0]  \n"

  306.       "dup v5.4s, v17.s[1]  \n"

  307.       "dup v6.4s, v17.s[2]  \n"

  308.       "dup v7.4s, v17.s[3]  \n"

  309.       "mov x4, #0 \n"

  310.       "mov x10, %[filter_zp] \n"

  311.       "mov x11, %[input_sum_oc] \n"

  312. 

  313.       "7: \n"

  314.       "cmp x4, %[oc_8div] \n"

  315.       "beq 8f \n"

  316.       "add x4, x4, #8\n"

  317.       "ld1 {v16.4s}, [x10], #16\n"

  318.       "ld1 {v17.4s}, [x10], #16\n"

  319. 

  320.       "mul v18.4s, v16.4s, v0.4s \n"

  321.       "mul v19.4s, v17.4s, v0.4s \n"

  322.       "st1 {v18.4s}, [x11], #16 \n"

  323.       "st1 {v19.4s}, [x11], #16 \n"

  324. 

  325.       "mul v20.4s, v16.4s, v1.4s \n"

  326.       "mul v21.4s, v17.4s, v1.4s \n"

  327.       "st1 {v20.4s}, [x11], #16 \n"

  328.       "st1 {v21.4s}, [x11], #16 \n"

  329. 

  330.       "mul v22.4s, v16.4s, v2.4s \n"

  331.       "mul v23.4s, v17.4s, v2.4s \n"

  332.       "st1 {v22.4s}, [x11], #16 \n"

  333.       "st1 {v23.4s}, [x11], #16 \n"

  334. 

  335.       "mul v24.4s, v16.4s, v3.4s \n"

  336.       "mul v25.4s, v17.4s, v3.4s \n"

  337.       "st1 {v24.4s}, [x11], #16 \n"

  338.       "st1 {v25.4s}, [x11], #16 \n"

  339. 

  340.       "mul v18.4s, v16.4s, v4.4s \n"

  341.       "mul v19.4s, v17.4s, v4.4s \n"

  342.       "st1 {v18.4s}, [x11], #16 \n"

  343.       "st1 {v19.4s}, [x11], #16 \n"

  344. 

  345.       "mul v20.4s, v16.4s, v5.4s \n"

  346.       "mul v21.4s, v17.4s, v5.4s \n"

  347.       "st1 {v20.4s}, [x11], #16 \n"

  348.       "st1 {v21.4s}, [x11], #16 \n"

  349. 

  350.       "mul v22.4s, v16.4s, v6.4s \n"

  351.       "mul v23.4s, v17.4s, v6.4s \n"

  352.       "st1 {v22.4s}, [x11], #16 \n"

  353.       "st1 {v23.4s}, [x11], #16 \n"

  354. 

  355.       "mul v24.4s, v16.4s, v7.4s \n"

  356.       "mul v25.4s, v17.4s, v7.4s \n"

  357.       "st1 {v24.4s}, [x11], #16 \n"

  358.       "st1 {v25.4s}, [x11], #16 \n"

  359. 

  360.       "add x11, x11, %[input_sum_stride] \n"

  361.       "b 7b \n"

  362. 

  363.       "8: \n"

  364.       "cmp %[oc_8res], #0\n"

  365.       "beq 17f \n"

  366. 

  367.       "dup v16.4s, wzr \n"

  368.       "dup v17.4s, wzr \n"

  369.       "cmp %[oc_8res], #1\n"

  370.       "beq 9f \n"

  371.       "cmp %[oc_8res], #2\n"

  372.       "beq 10f \n"

  373.       "cmp %[oc_8res], #3\n"

  374.       "beq 11f \n"

  375.       "cmp %[oc_8res], #4\n"

  376.       "beq 12f \n"

  377.       "cmp %[oc_8res], #5\n"

  378.       "beq 13f \n"

  379.       "cmp %[oc_8res], #6\n"

  380.       "beq 14f \n"

  381.       "cmp %[oc_8res], #7\n"

  382.       "beq 15f \n"

  383. 

  384.       "9: \n"

  385.       "ld1 {v16.s}[0], [x10] \n"

  386.       "b 16f \n"

  387. 

  388.       "10: \n"

  389.       "ld1 {v16.h}[0], [x10] \n"

  390.       "b 16f \n"

  391. 

  392.       "11: \n"

  393.       "ld1 {v16.h}[0], [x10] \n"

  394.       "add x10, x10, #8 \n"

  395.       "ld1 {v16.s}[2], [x10] \n"

  396.       "b 16f \n"

  397. 

  398.       "12: \n"

  399.       "ld1 {v16.4s}, [x10] \n"

  400.       "b 16f \n"

  401. 

  402.       "13: \n"

  403.       "ld1 {v16.4s}, [x10], #16\n"

  404.       "ld1 {v17.s}[0], [x10] \n"

  405.       "b 16f \n"

  406. 

  407.       "14: \n"

  408.       "ld1 {v16.4s}, [x10], #16\n"

  409.       "ld1 {v17.h}[0], [x10] \n"

  410.       "b 16f \n"

  411. 

  412.       "15: \n"

  413.       "ld1 {v16.4s}, [x10], #16\n"

  414.       "ld1 {v17.h}[0], [x10] \n"

  415.       "add x10, x10, #8 \n"

  416.       "ld1 {v17.s}[2], [x10] \n"

  417.       "b 16f \n"

  418. 

  419.       "16: \n"

  420.       "mul v18.4s, v16.4s, v0.4s \n"

  421.       "mul v19.4s, v17.4s, v0.4s \n"

  422.       "mul v20.4s, v16.4s, v1.4s \n"

  423.       "mul v21.4s, v17.4s, v1.4s \n"

  424.       "mul v22.4s, v16.4s, v2.4s \n"

  425.       "mul v23.4s, v17.4s, v2.4s \n"

  426.       "mul v24.4s, v16.4s, v3.4s \n"

  427.       "mul v25.4s, v17.4s, v3.4s \n"

  428.       "st1 {v18.4s}, [x11], #16 \n"

  429.       "st1 {v19.4s}, [x11], #16 \n"

  430.       "st1 {v20.4s}, [x11], #16 \n"

  431.       "st1 {v21.4s}, [x11], #16 \n"

  432.       "st1 {v22.4s}, [x11], #16 \n"

  433.       "st1 {v23.4s}, [x11], #16 \n"

  434.       "st1 {v24.4s}, [x11], #16 \n"

  435.       "st1 {v25.4s}, [x11], #16 \n"

  436. 

  437.       "mul v18.4s, v16.4s, v4.4s \n"

  438.       "mul v19.4s, v17.4s, v4.4s \n"

  439.       "mul v20.4s, v16.4s, v5.4s \n"

  440.       "mul v21.4s, v17.4s, v5.4s \n"

  441.       "mul v22.4s, v16.4s, v6.4s \n"

  442.       "mul v23.4s, v17.4s, v6.4s \n"

  443.       "mul v24.4s, v16.4s, v7.4s \n"

  444.       "mul v25.4s, v17.4s, v7.4s \n"

  445.       "st1 {v18.4s}, [x11], #16 \n"

  446.       "st1 {v19.4s}, [x11], #16 \n"

  447.       "st1 {v20.4s}, [x11], #16 \n"

  448.       "st1 {v21.4s}, [x11], #16 \n"

  449.       "st1 {v22.4s}, [x11], #16 \n"

  450.       "st1 {v23.4s}, [x11], #16 \n"

  451.       "st1 {v24.4s}, [x11], #16 \n"

  452.       "st1 {v25.4s}, [x11], #16 \n"

  453. 

  454.       "17: \n"

  455. 

  456.       :

  457.       : [ src_ic ] "r"(src_ic), [ pack_ic ] "r"(pack_ic), [ filter_zp ] "r"(filter_zp),

  458.         [ input_sum_oc ] "r"(input_sum_oc), [ input_sum_stride ] "r"(input_sum_stride), [ src_stride ] "r"(src_stride),

  459.         [ ic_4div ] "r"(ic_4div), [ ic_4res ] "r"(ic_4res), [ oc_8div ] "r"(oc_8div), [ oc_8res ] "r"(oc_8res)

  460.       : "x0", "x1", "x4", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",

  461.         "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25");

  462. #else

  463.     int32_t tmp_sum_value[8] = {0};

  464.     for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  465.       for (int i = 0; i < C8NUM; i++) {

  466.         tmp_sum_value[i] += src_ic[0 + i * input_channel];

  467.         tmp_sum_value[i] += src_ic[1 + i * input_channel];

  468.         tmp_sum_value[i] += src_ic[2 + i * input_channel];

  469.         tmp_sum_value[i] += src_ic[3 + i * input_channel];

  470.         pack_ic[0 + i * C4NUM] = src_ic[0 + i * input_channel];

  471.         pack_ic[1 + i * C4NUM] = src_ic[1 + i * input_channel];

  472.         pack_ic[2 + i * C4NUM] = src_ic[2 + i * input_channel];

  473.         pack_ic[3 + i * C4NUM] = src_ic[3 + i * input_channel];

  474.       }

  475.       src_ic += C4NUM;

  476.       pack_ic += C4NUM * C8NUM;

  477.     }

  478.     for (int ici = ic_4div; ici < input_channel; ici += 1) {

  479.       for (int i = 0; i < C8NUM; i++) {

  480.         tmp_sum_value[i] += src_ic[i * input_channel];

  481.         pack_ic[i * C4NUM] = src_ic[i * input_channel];

  482.       }

  483.       src_ic += 1;

  484.       pack_ic += 1;

  485.     }

  486. 

  487.     for (int ici = input_channel; ici < ic4; ici += 1) {

  488.       for (int i = 0; i < C8NUM; i++) {

  489.         pack_ic[i * C4NUM] = 0;

  490.       }

  491.       pack_ic += 1;

  492.     }

  493. 

  494.     for (int oci = 0; oci < oc_8div; oci += C8NUM) {

  495.       for (int ri = 0; ri < C8NUM; ri++) {

  496.         input_sum_oc[ri * C8NUM + 0] = tmp_sum_value[ri] * filter_zp[oci + 0];

  497.         input_sum_oc[ri * C8NUM + 1] = tmp_sum_value[ri] * filter_zp[oci + 1];

  498.         input_sum_oc[ri * C8NUM + 2] = tmp_sum_value[ri] * filter_zp[oci + 2];

  499.         input_sum_oc[ri * C8NUM + 3] = tmp_sum_value[ri] * filter_zp[oci + 3];

  500.         input_sum_oc[ri * C8NUM + 4] = tmp_sum_value[ri] * filter_zp[oci + 4];

  501.         input_sum_oc[ri * C8NUM + 5] = tmp_sum_value[ri] * filter_zp[oci + 5];

  502.         input_sum_oc[ri * C8NUM + 6] = tmp_sum_value[ri] * filter_zp[oci + 6];

  503.         input_sum_oc[ri * C8NUM + 7] = tmp_sum_value[ri] * filter_zp[oci + 7];

  504.       }

  505.       input_sum_oc += inputsum_stride;

  506.     }

  507.     if (oc_8div != output_channel) {

  508.       for (int oci = 0; oci < oc_8res; oci += 1) {

  509.         for (int ri = 0; ri < C8NUM; ri++) {

  510.           input_sum_oc[ri * C8NUM + oci] = tmp_sum_value[ri] * filter_zp[oc_8div + oci];

  511.         }

  512.       }

  513.       for (int oci = oc_8res; oci < C8NUM; oci += 1) {

  514.         for (int ri = 0; ri < C8NUM; ri++) {

  515.           input_sum_oc[ri * C8NUM + oci] = 0;

  516.         }

  517.       }

  518.     } /* oc8 res done */

  519. #endif

  520.     src_r += input_channel * C8NUM;

  521.     pack_r += ic4 * C8NUM;

  522.     input_sum_r += C8NUM * C8NUM;

  523.   }

  524. 

  525.   if (hw_8div != plane_size) {

  526.     memset(pack_r, 0, C8NUM * ic4);

  527.     for (int hwi = hw_8div; hwi < plane_size; hwi += 1) {

  528.       int32_t *input_sum_oc = input_sum_r;

  529.       int32_t tmp_sum_value = 0;

  530.       const int8_t *src_ic = src_r;

  531.       int8_t *pack_ic = pack_r;

  532.       for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  533.         tmp_sum_value += src_ic[0];

  534.         tmp_sum_value += src_ic[1];

  535.         tmp_sum_value += src_ic[2];

  536.         tmp_sum_value += src_ic[3];

  537.         pack_ic[0] = src_ic[0];

  538.         pack_ic[1] = src_ic[1];

  539.         pack_ic[2] = src_ic[2];

  540.         pack_ic[3] = src_ic[3];

  541.         src_ic += C4NUM;

  542.         pack_ic += C4NUM * C8NUM;

  543.       }

  544.       for (int ici = ic_4div; ici < input_channel; ici += 1) {

  545.         tmp_sum_value += src_ic[0];

  546.         pack_ic[0] = src_ic[0];

  547.         src_ic += 1;

  548.         pack_ic += 1;

  549.       }

  550. 

  551.       for (int oci = 0; oci < oc_8div; oci += C8NUM) {

  552.         for (int curoi = 0; curoi < C8NUM; curoi++) {

  553.           input_sum_oc[curoi] = tmp_sum_value * filter_zp[oci + curoi];

  554.         }

  555.         input_sum_oc += inputsum_stride;

  556.       }

  557.       if (oc_8div != output_channel) {

  558.         for (int oci = 0; oci < oc_8res; oci += 1) {

  559.           input_sum_oc[oci] = tmp_sum_value * filter_zp[oc_8div + oci];

  560.         }

  561.         for (int oci = oc_8res; oci < C8NUM; oci += 1) {

  562.           input_sum_oc[oci] = 0;

  563.         }

  564.       } /* oc8 res done */

  565. 

  566.       src_r += input_channel;

  567.       pack_r += C4NUM;

  568.       input_sum_r += C8NUM;

  569.     }

  570. 

  571.     for (int hwi = plane_size; hwi < hw8; hwi++) {

  572.       for (int oc = 0; oc < oc8; oc++) {

  573.         int oc8div = oc / C8NUM, oc8res = oc % C8NUM;

  574.         input_sum[oc8div * inputsum_stride + hwi * C8NUM + oc8res] = 0;

  575.       }

  576.     }

  577.   }

  578.   return;

  579. }

  580. 

  581. void Conv1x1PreOptPert(const int8_t *src_input, int8_t *packed_input, int32_t *input_sum, size_t input_channel,

  582.                        size_t plane_size, ConvParameter *conv_param) {

  583.   int ic4 = UP_ROUND(input_channel, C4NUM);

  584.   size_t hw_8div = plane_size / C8NUM * C8NUM;

  585.   size_t ic_4div = input_channel / C4NUM * C4NUM;

  586.   int32_t filter_zp = conv_param->conv_quant_arg_.filter_quant_args_[0].zp_;

  587. 

  588.   const int8_t *src_r = src_input;

  589.   int8_t *pack_r = packed_input;

  590.   /* per layer */

  591.   for (int hwi = 0; hwi < hw_8div; hwi += C8NUM) {

  592.     const int8_t *src_ic = src_r;

  593.     int8_t *pack_ic = pack_r;

  594.     int32_t *input_sum_r = input_sum + hwi;

  595. #ifdef ENABLE_ARM64

  596.     size_t src_stride = input_channel;

  597.     size_t ic_4res = input_channel - ic_4div;

  598.     asm volatile(

  599.       "dup v16.4s, wzr \n"

  600.       "dup v17.4s, wzr \n"

  601.       "mov x14, %[input_sum_r] \n"

  602.       "dup v20.4s, %w[filter_zp]  \n"

  603. 

  604.       "mov x10, %[src_ic] \n"

  605.       "mov x11, %[pack_ic] \n"

  606. 

  607.       "mov x0, #0 \n"

  608.       "1: \n"

  609.       "cmp x0, %[ic_4div] \n"

  610.       "add x0, x0, #4\n"

  611.       "mov x12, x10 \n"

  612.       "add x10, x10, #4\n"

  613.       "blt 2f \n"

  614.       "cmp %[ic_4res], #0\n"

  615.       "beq 6f \n"

  616.       "cmp %[ic_4res], #1\n"

  617.       "beq 3f \n"

  618.       "cmp %[ic_4res], #2\n"

  619.       "beq 4f \n"

  620.       "cmp %[ic_4res], #3\n"

  621.       "beq 5f \n"

  622. 

  623.       "2: \n"

  624.       "ld1 {v0.s}[0], [x12], %[src_stride]\n"

  625.       "ld1 {v0.s}[1], [x12], %[src_stride]\n"

  626.       "ld1 {v0.s}[2], [x12], %[src_stride]\n"

  627.       "ld1 {v0.s}[3], [x12], %[src_stride]\n"

  628.       "ld1 {v1.s}[0], [x12], %[src_stride]\n"

  629.       "ld1 {v1.s}[1], [x12], %[src_stride]\n"

  630.       "ld1 {v1.s}[2], [x12], %[src_stride]\n"

  631.       "ld1 {v1.s}[3], [x12], %[src_stride]\n"

  632. 

  633.       "st1 {v0.16b}, [x11], #16\n"

  634.       "st1 {v1.16b}, [x11], #16\n"

  635. 

  636.       "saddlp v4.8h, v0.16b \n"

  637.       "saddlp v5.8h, v1.16b \n"

  638. 

  639.       "saddlp v0.4s, v4.8h \n"

  640.       "saddlp v1.4s, v5.8h \n"

  641. 

  642.       "add v16.4s, v16.4s, v0.4s \n"

  643.       "add v17.4s, v17.4s, v1.4s \n"

  644.       "b 1b \n"

  645. 

  646.       "3: \n" /* col res 1 */

  647.       "dup v0.4s, wzr \n"

  648.       "dup v1.4s, wzr \n"

  649. 

  650.       "ld1 {v0.b}[0],  [x12], %[src_stride]\n"

  651.       "ld1 {v0.b}[4],  [x12], %[src_stride]\n"

  652.       "ld1 {v0.b}[8],  [x12], %[src_stride]\n"

  653.       "ld1 {v0.b}[12], [x12], %[src_stride]\n"

  654.       "ld1 {v1.b}[0],  [x12], %[src_stride]\n"

  655.       "ld1 {v1.b}[4],  [x12], %[src_stride]\n"

  656.       "ld1 {v1.b}[8],  [x12], %[src_stride]\n"

  657.       "ld1 {v1.b}[12], [x12], %[src_stride]\n"

  658. 

  659.       "st1 {v0.16b}, [x11], #16\n"

  660.       "st1 {v1.16b}, [x11], #16\n"

  661.       "saddlp v4.8h, v0.16b \n"

  662.       "saddlp v5.8h, v1.16b \n"

  663.       "saddlp v0.4s, v4.8h \n"

  664.       "saddlp v1.4s, v5.8h \n"

  665.       "add v16.4s, v16.4s, v0.4s \n"

  666.       "add v17.4s, v17.4s, v1.4s \n"

  667.       "b 6f \n"

  668. 

  669.       "4: \n" /* col res 2 */

  670.       "dup v0.4s, wzr \n"

  671.       "dup v1.4s, wzr \n"

  672. 

  673.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  674.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  675.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  676.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  677.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  678.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  679.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  680.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  681. 

  682.       "st1 {v0.16b}, [x11], #16\n"

  683.       "st1 {v1.16b}, [x11], #16\n"

  684.       "saddlp v4.8h, v0.16b \n"

  685.       "saddlp v5.8h, v1.16b \n"

  686.       "saddlp v0.4s, v4.8h \n"

  687.       "saddlp v1.4s, v5.8h \n"

  688.       "add v16.4s, v16.4s, v0.4s \n"

  689.       "add v17.4s, v17.4s, v1.4s \n"

  690.       "b 6f \n"

  691. 

  692.       "5: \n" /* col res 3 */

  693.       "dup v0.4s, wzr \n"

  694.       "dup v1.4s, wzr \n"

  695.       "add x13, x12, #2 \n"

  696. 

  697.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  698.       "ld1 {v0.b}[2], [x13], %[src_stride]\n"

  699.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  700.       "ld1 {v0.b}[6], [x13], %[src_stride]\n"

  701.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  702.       "ld1 {v0.b}[10], [x13], %[src_stride]\n"

  703.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  704.       "ld1 {v0.b}[14], [x13], %[src_stride]\n"

  705.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  706.       "ld1 {v1.b}[2], [x13], %[src_stride]\n"

  707.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  708.       "ld1 {v1.b}[6], [x13], %[src_stride]\n"

  709.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  710.       "ld1 {v1.b}[10], [x13], %[src_stride]\n"

  711.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  712.       "ld1 {v1.b}[14], [x13], %[src_stride]\n"

  713. 

  714.       "st1 {v0.16b}, [x11], #16\n"

  715.       "st1 {v1.16b}, [x11], #16\n"

  716.       "saddlp v4.8h, v0.16b \n"

  717.       "saddlp v5.8h, v1.16b \n"

  718.       "saddlp v0.4s, v4.8h \n"

  719.       "saddlp v1.4s, v5.8h \n"

  720.       "add v16.4s, v16.4s, v0.4s \n"

  721.       "add v17.4s, v17.4s, v1.4s \n"

  722.       "b 6f \n"

  723. 

  724.       "6: \n"

  725.       "mul v16.4s, v16.4s, v20.4s \n"

  726.       "mul v17.4s, v17.4s, v20.4s \n"

  727. 

  728.       "st1 {v16.4s}, [x14], #16 \n"

  729.       "st1 {v17.4s}, [x14], #16 \n"

  730. 

  731.       :

  732.       : [ src_ic ] "r"(src_ic), [ pack_ic ] "r"(pack_ic), [ input_sum_r ] "r"(input_sum_r),

  733.         [ src_stride ] "r"(src_stride), [ ic_4div ] "r"(ic_4div), [ ic_4res ] "r"(ic_4res), [ filter_zp ] "r"(filter_zp)

  734.       : "x0", "x1", "x10", "x11", "x12", "x13", "x14", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17",

  735.         "v20");

  736. #else

  737.     int32_t tmp_sum_value[8] = {0};

  738.     for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  739.       for (int i = 0; i < C8NUM; i++) {

  740.         tmp_sum_value[i] += src_ic[0 + i * input_channel];

  741.         tmp_sum_value[i] += src_ic[1 + i * input_channel];

  742.         tmp_sum_value[i] += src_ic[2 + i * input_channel];

  743.         tmp_sum_value[i] += src_ic[3 + i * input_channel];

  744.         pack_ic[0 + i * C4NUM] = src_ic[0 + i * input_channel];

  745.         pack_ic[1 + i * C4NUM] = src_ic[1 + i * input_channel];

  746.         pack_ic[2 + i * C4NUM] = src_ic[2 + i * input_channel];

  747.         pack_ic[3 + i * C4NUM] = src_ic[3 + i * input_channel];

  748.       }

  749.       src_ic += C4NUM;

  750.       pack_ic += C4NUM * C8NUM;

  751.     }

  752.     for (int ici = ic_4div; ici < input_channel; ici += 1) {

  753.       for (int i = 0; i < C8NUM; i++) {

  754.         tmp_sum_value[i] += src_ic[i * input_channel];

  755.         pack_ic[i * C4NUM] = src_ic[i * input_channel];

  756.       }

  757.       src_ic += 1;

  758.       pack_ic += 1;

  759.     }

  760. 

  761.     for (int ici = input_channel; ici < ic4; ici += 1) {

  762.       for (int i = 0; i < C8NUM; i++) {

  763.         pack_ic[i * C4NUM] = 0;

  764.       }

  765.       pack_ic += 1;

  766.     }

  767. 

  768.     for (int i = 0; i < C8NUM; i++) {

  769.       input_sum_r[i] = tmp_sum_value[i] * filter_zp;

  770.     }

  771. #endif

  772.     src_r += input_channel * C8NUM;

  773.     pack_r += ic4 * C8NUM;

  774.   }

  775. 

  776.   if (hw_8div != plane_size) {

  777.     memset(pack_r, 0, C8NUM * ic4);

  778.     for (int hwi = hw_8div; hwi < plane_size; hwi += 1) {

  779.       int32_t tmp_sum_value = 0;

  780.       const int8_t *src_ic = src_r;

  781.       int8_t *pack_ic = pack_r;

  782.       for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  783.         tmp_sum_value += src_ic[0];

  784.         tmp_sum_value += src_ic[1];

  785.         tmp_sum_value += src_ic[2];

  786.         tmp_sum_value += src_ic[3];

  787.         pack_ic[0] = src_ic[0];

  788.         pack_ic[1] = src_ic[1];

  789.         pack_ic[2] = src_ic[2];

  790.         pack_ic[3] = src_ic[3];

  791.         src_ic += C4NUM;

  792.         pack_ic += C4NUM * C8NUM;

  793.       }

  794.       for (int ici = ic_4div; ici < input_channel; ici += 1) {

  795.         tmp_sum_value += src_ic[0];

  796.         pack_ic[0] = src_ic[0];

  797.         src_ic += 1;

  798.         pack_ic += 1;

  799.       }

  800.       input_sum[hwi] = tmp_sum_value * filter_zp;

  801.       src_r += input_channel;

  802.       pack_r += C4NUM;

  803.     }

  804.     for (int hwi = plane_size; hwi < UP_ROUND(plane_size, C8NUM); hwi++) {

  805.       input_sum[hwi] = 0;

  806.     }

  807.   }

  808.   return;

  809. }

  810. 

  811. void Conv1x1Int8Opt(const int8_t *packed_input, const int8_t *packed_weight, int8_t *dst, const int32_t *input_sum,

  812.                     const int32_t *bias, int row, int col, int deep4, int32_t *left_shift, int32_t *right_shift,

  813.                     int32_t *multiplier, ConvParameter *conv_param, MATMUL_OPT_R_FUNC matmul_func) {

  814.   int is_per_oc = (int)conv_param->conv_quant_arg_.filter_arg_num_ != 1;

  815.   matmul_func(packed_input, packed_weight, dst, row, col, deep4, conv_param->output_channel_, input_sum, bias,

  816.               left_shift, right_shift, multiplier, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  817.               conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0], is_per_oc);

  818.   return;

  819. }

  820. 

  821. void Conv1x1Int8Arm32(const int8_t *packed_input, const int8_t *packed_weight, int8_t *dst, const int32_t *input_sum,

  822.                       const int32_t *bias, int row, int col, int deep16, int32_t *left_shift, int32_t *right_shift,

  823.                       int32_t *multiplier, ConvParameter *conv_param) {

  824.   int is_per_channel = conv_param->conv_quant_arg_.filter_arg_num_ != 1 ? true : false;

  825. #ifdef ENABLE_ARM32

  826.   MatmulInt8Neon32(packed_input, packed_weight, dst, row, col, deep16, input_sum, bias,

  827.                    conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  828.                    conv_param->conv_quant_arg_.output_quant_args_[0].zp_, multiplier, left_shift, right_shift,

  829.                    conv_param->output_channel_, is_per_channel);

  830. #else

  831.   MatMulInt8_4x2_r(packed_input, packed_weight, dst, row, col, deep16, conv_param->output_channel_, input_sum, bias,

  832.                    left_shift, right_shift, multiplier, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  833.                    conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  834.                    is_per_channel);

  835. #endif

  836.   return;

  837. }

  838. 

  839. void Conv1x1Int8(const int8_t *packed_input, const int8_t *packed_weight, int8_t *dst, const int32_t *input_sum,

  840.                  const int32_t *bias, int row, int col, int deep16, int32_t *left_shift, int32_t *right_shift,

  841.                  int32_t *multiplier, ConvParameter *conv_param) {

  842.   int is_per_oc = (int)conv_param->conv_quant_arg_.filter_arg_num_ != 1;

  843. #ifdef ENABLE_ARM64

  844.   MatmulInt8Neon64(packed_input, packed_weight, dst, UP_ROUND(row, C4NUM), UP_ROUND(col, C4NUM), deep16, input_sum,

  845.                    bias, conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  846.                    conv_param->conv_quant_arg_.output_quant_args_[0].zp_, multiplier, left_shift, right_shift, row, col,

  847.                    conv_param->output_channel_, is_per_oc);

  848. #else

  849.   MatMulInt8_16x4_r(packed_input, packed_weight, dst, row, col, deep16, conv_param->output_channel_, input_sum, bias,

  850.                     left_shift, right_shift, multiplier, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  851.                     conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  852.                     is_per_oc);

  853. #endif

  854.   return;

  855. }

  856. 

  857. // int8 convolution 3x3

  858. void Conv3x3Int8(int16_t *input_data, int16_t *transed_weight, const int32_t *bias_data, int8_t *output_data,

  859.                  int16_t *tile_buffer, int16_t *block_unit_buffer, int32_t *tmp_dst_buffer, int8_t *tmp_out,

  860.                  int task_id, ConvParameter *conv_param) {

  861.   int thread_count = conv_param->thread_num_;

  862.   int ic8 = UP_DIV(conv_param->input_channel_, C8NUM);

  863.   int output_channel = conv_param->output_channel_;

  864.   int out_w_block = UP_DIV(conv_param->output_w_, OUPUT_UNIT);

  865.   int out_h_block = UP_DIV(conv_param->output_h_, OUPUT_UNIT);

  866.   int output_count = out_w_block * out_h_block;

  867.   int output_tile_count = UP_DIV(output_count, TILE_NUM);

  868.   int oc4 = UP_DIV(output_channel, C4NUM);

  869.   int tile_buffer_offset = TILE_NUM * 16 * ic8 * C8NUM;

  870.   const int block_unit_buffer_offset = 16 * C8NUM;

  871.   int tmp_dst_buffer_offset = TILE_NUM * 16 * oc4 * C4NUM;

  872. 

  873.   int input_batch = conv_param->input_batch_;

  874.   for (int batch = 0; batch < input_batch; batch++) {

  875.     int in_batch_offset = batch * ic8 * C8NUM * conv_param->input_h_ * conv_param->input_w_;

  876.     int tmp_out_batch_offset = batch * oc4 * C4NUM * conv_param->output_w_ * conv_param->output_h_;

  877.     for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {

  878.       int start_index = thread_id * TILE_NUM;

  879.       int real_cal_num = (output_count - start_index) < TILE_NUM ? (output_count - start_index) : TILE_NUM;

  880. 

  881.       Conv3x3Int8InputTransform(input_data + in_batch_offset, tile_buffer + task_id * tile_buffer_offset,

  882.                                 block_unit_buffer + task_id * block_unit_buffer_offset, start_index, real_cal_num,

  883.                                 out_w_block, conv_param);

  884. 

  885.       Conv3x3Int8Gemm(tmp_dst_buffer + task_id * tmp_dst_buffer_offset, tile_buffer + task_id * tile_buffer_offset,

  886.                       transed_weight, output_channel, ic8, real_cal_num);

  887. 

  888.       Conv3x3Int8OutputTransform(tmp_dst_buffer + task_id * tmp_dst_buffer_offset, tmp_out + tmp_out_batch_offset,

  889.                                  bias_data, start_index, real_cal_num, out_w_block, conv_param);

  890.     }

  891.   }

  892. }