Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 0db75b70d7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '0db75b70d7'
${ noResults }
mindspore/mindspore/lite/nnacl/int8/conv_int8.c

799 lines
36 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/int8/conv_int8.h"

  18. #include <string.h>

  19. #include "nnacl/winograd_transform.h"

  20. #include "nnacl/int8/common_func.h"

  21. 

  22. void IndirectGemmInt8(int8_t *dst, int32_t *tmp_dst, const int8_t *src, const int8_t *weight, const int32_t *bias,

  23.                       int ic4, size_t kernel_plane, size_t output_channel, const int32_t *input_sum,

  24.                       ConvParameter *conv_param) {

  25.   int32_t *shift_before = conv_param->conv_quant_arg_.left_shift_;

  26.   int32_t *shift_after = conv_param->conv_quant_arg_.right_shift_;

  27.   int32_t *out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_;

  28.   int32_t out_zp = conv_param->conv_quant_arg_.output_quant_args_[0].zp_;

  29.   int32_t act_min = conv_param->conv_quant_arg_.out_act_min_[0];

  30.   int32_t act_max = conv_param->conv_quant_arg_.out_act_max_[0];

  31. 

  32. #ifdef ENABLE_ARM64

  33.   size_t asymmetric = conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC;

  34.   size_t per_channel = conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL;

  35.   IndirectGemmInt8_4x4(dst, src, weight, bias, UP_DIV(kernel_plane, C4NUM), ic4, output_channel,

  36.                        output_channel * sizeof(int8_t), input_sum, act_min, act_max, out_zp, out_multiplier,

  37.                        shift_before, shift_after, asymmetric, per_channel);

  38. #else

  39.   int oc4 = UP_DIV(output_channel, C4NUM);

  40.   int tile_num = conv_param->tile_num_;

  41.   int plane_c4 = UP_DIV(kernel_plane, C4NUM);

  42.   for (int oc = 0; oc < output_channel; oc++) {

  43.     int oc4_block = oc / C4NUM;

  44.     int oc4_res = oc % C4NUM;

  45.     int weight_oc4_offset = oc4_block * C4NUM * plane_c4 * C4NUM * ic4 * C4NUM + oc4_res * C4NUM * C4NUM;

  46.     int dst_oc_offset = oc;

  47.     for (int n = 0; n < tile_num; n++) {

  48.       int src_tile_offset = n * C4NUM * C4NUM;

  49.       int dst_tile_offset = dst_oc_offset + n * output_channel;

  50. 

  51.       for (int b = 0; b < kernel_plane; b++) {

  52.         int plane_c4_block = b / C4NUM;

  53.         int plane_c4_res = b % C4NUM;

  54.         int src_plane_offset = src_tile_offset + plane_c4_block * tile_num * C4NUM * ic4 * C4NUM + plane_c4_res * C4NUM;

  55.         int weight_plane_offset =

  56.           weight_oc4_offset + plane_c4_block * C4NUM * C4NUM * ic4 * C4NUM + plane_c4_res * C4NUM;

  57.         for (int i = 0; i < ic4; i++) {

  58.           int src_ic4_offset = src_plane_offset + i * tile_num * C4NUM * C4NUM;

  59.           int weight_ic4_offset = weight_plane_offset + i * C4NUM * C4NUM * C4NUM;

  60.           for (int j = 0; j < C4NUM; j++) {

  61.             int weight_ic_offset = weight_ic4_offset + j;

  62.             tmp_dst[dst_tile_offset] += weight[weight_ic_offset] * src[src_ic4_offset + j];

  63.           }  // in c4num loop

  64.         }    // ic4 loop

  65.       }      // kernel_plane loop

  66.       if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  67.           (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  68.         int result = tmp_dst[dst_tile_offset] + bias[oc];

  69.         result = RoundingDivideByPOT(

  70.           SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[oc]), out_multiplier[oc]),

  71.           -shift_after[oc]);

  72.         result += out_zp;

  73.         result = result > act_min ? result : act_min;

  74.         result = result < act_max ? result : act_max;

  75.         dst[dst_tile_offset] = (int8_t)result;

  76.       } else if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  77.                  !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  78.         int result = tmp_dst[dst_tile_offset] + bias[oc];

  79.         result = RoundingDivideByPOT(

  80.           SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[0]), out_multiplier[0]),

  81.           -shift_after[0]);

  82.         result += out_zp;

  83.         result = result > act_min ? result : act_min;

  84.         result = result < act_max ? result : act_max;

  85.         dst[dst_tile_offset] = (int8_t)result;

  86.       } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  87.                  !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  88.         tmp_dst[dst_tile_offset] -= input_sum[n];

  89.         int result = tmp_dst[dst_tile_offset] + bias[oc];

  90.         result = RoundingDivideByPOT(

  91.           SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[0]), out_multiplier[0]),

  92.           -shift_after[0]);

  93.         result += out_zp;

  94.         result = result > act_min ? result : act_min;

  95.         result = result < act_max ? result : act_max;

  96.         dst[dst_tile_offset] = (int8_t)result;

  97.       } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  98.                  (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  99.         tmp_dst[dst_tile_offset] -= input_sum[n * oc4 * C4NUM + oc];

  100.         int result = tmp_dst[dst_tile_offset] + bias[oc];

  101.         result = RoundingDivideByPOT(

  102.           SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[oc]), out_multiplier[oc]),

  103.           -shift_after[oc]);

  104.         result += out_zp;

  105.         result = result > act_min ? result : act_min;

  106.         result = result < act_max ? result : act_max;

  107.         dst[dst_tile_offset] = (int8_t)result;

  108.       }

  109.     }  // tile_num loop

  110.   }    // output_channel loop

  111. #endif

  112. }

  113. 

  114. void IndirectGemmInt8Opt(int8_t *dst, int32_t *tmp_dst, const int8_t *src, const int8_t *weight, const int32_t *bias,

  115.                          int ic4, size_t kernel_plane, size_t output_channel, const int32_t *input_sum,

  116.                          ConvParameter *conv_param, GEMM_FUNC gemm_func) {

  117.   int32_t *shift_before = conv_param->conv_quant_arg_.left_shift_;

  118.   int32_t *shift_after = conv_param->conv_quant_arg_.right_shift_;

  119.   int32_t *out_multiplier = conv_param->conv_quant_arg_.quant_multiplier_;

  120.   int32_t out_zp = conv_param->conv_quant_arg_.output_quant_args_[0].zp_;

  121.   int32_t act_min = conv_param->conv_quant_arg_.out_act_min_[0];

  122.   int32_t act_max = conv_param->conv_quant_arg_.out_act_max_[0];

  123.   int oc4 = UP_DIV(output_channel, C4NUM);

  124.   if (gemm_func != NULL) {

  125. #ifdef __aarch64__

  126.     size_t asymmetric = conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC;

  127.     size_t per_channel = conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL;

  128.     gemm_func(dst, src, weight, bias, kernel_plane, ic4, output_channel, output_channel * sizeof(int8_t), input_sum,

  129.               act_min, act_max, out_zp, out_multiplier, shift_before, shift_after, asymmetric, per_channel);

  130. #endif

  131.   } else {

  132.     int tile_num = conv_param->tile_num_;

  133.     for (int oc = 0; oc < output_channel; oc++) {

  134.       int oc4_block = oc / C4NUM;

  135.       int oc4_res = oc % C4NUM;

  136.       int weight_oc4_offset = oc4_block * C4NUM * kernel_plane * ic4 * C4NUM + oc4_res * C4NUM;

  137.       int dst_oc_offset = oc;

  138.       for (int n = 0; n < tile_num; n++) {

  139.         int src_tile_offset = n * C4NUM;

  140.         int dst_tile_offset = dst_oc_offset + n * output_channel;

  141. 

  142.         for (int b = 0; b < kernel_plane; b++) {

  143.           int src_plane_offset = src_tile_offset + b * tile_num * ic4 * C4NUM;

  144.           int weight_plane_offset = weight_oc4_offset + b * C4NUM * ic4 * C4NUM;

  145.           for (int i = 0; i < ic4; i++) {

  146.             int src_ic4_offset = src_plane_offset + i * tile_num * C4NUM;

  147.             int weight_ic4_offset = weight_plane_offset + i * C4NUM * C4NUM;

  148.             for (int j = 0; j < C4NUM; j++) {

  149.               int weight_ic_offset = weight_ic4_offset + j;

  150.               tmp_dst[dst_tile_offset] += weight[weight_ic_offset] * src[src_ic4_offset + j];

  151.             }  // in c4num loop

  152.           }    // ic4 loop

  153.         }      // kernel_plane loop

  154.         if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  155.             (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  156.           int result = tmp_dst[dst_tile_offset] + bias[oc];

  157.           result = RoundingDivideByPOT(

  158.             SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[oc]), out_multiplier[oc]),

  159.             -shift_after[oc]);

  160.           result += out_zp;

  161.           result = result > act_min ? result : act_min;

  162.           result = result < act_max ? result : act_max;

  163.           dst[dst_tile_offset] = (int8_t)result;

  164.         } else if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  165.                    !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  166.           int result = tmp_dst[dst_tile_offset] + bias[oc];

  167.           result = RoundingDivideByPOT(

  168.             SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[0]), out_multiplier[0]),

  169.             -shift_after[0]);

  170.           result += out_zp;

  171.           result = result > act_min ? result : act_min;

  172.           result = result < act_max ? result : act_max;

  173.           dst[dst_tile_offset] = (int8_t)result;

  174.         } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  175.                    !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  176.           tmp_dst[dst_tile_offset] -= input_sum[n];

  177.           int result = tmp_dst[dst_tile_offset] + bias[oc];

  178.           result = RoundingDivideByPOT(

  179.             SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[0]), out_multiplier[0]),

  180.             -shift_after[0]);

  181.           result += out_zp;

  182.           result = result > act_min ? result : act_min;

  183.           result = result < act_max ? result : act_max;

  184.           dst[dst_tile_offset] = (int8_t)result;

  185.         } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  186.                    (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  187.           tmp_dst[dst_tile_offset] -= input_sum[n * oc4 * C4NUM + oc];

  188.           int result = tmp_dst[dst_tile_offset] + bias[oc];

  189.           result = RoundingDivideByPOT(

  190.             SaturatingRoundingDoublingHighMul(result * (1 << (unsigned int)shift_before[oc]), out_multiplier[oc]),

  191.             -shift_after[oc]);

  192.           result += out_zp;

  193.           result = result > act_min ? result : act_min;

  194.           result = result < act_max ? result : act_max;

  195.           dst[dst_tile_offset] = (int8_t)result;

  196.         }

  197.       }  // tile_num loop

  198.     }    // output_channel loop

  199.   }

  200. }

  201. 

  202. void Conv3x3Int8Gemm(int32_t *dst, const int16_t *src, const int16_t *weight, int oc, int ic8, size_t real_cal_num) {

  203.   int oc4 = UP_DIV(oc, C4NUM);

  204. #ifdef ENABLE_ARM64

  205.   IndirectGemmInt16to32_8x4(dst, src, weight, 16, ic8, oc4, oc4 * 4 * 16 * sizeof(int32_t));

  206. #else

  207.   const int input_unit_square = 16;

  208.   for (int c = 0; c < oc4; c++) {

  209.     int filter_oc_offset = c * input_unit_square * ic8 * C8NUM * C4NUM;

  210.     int dst_oc_offset = c * input_unit_square * C4NUM;

  211.     for (int n = 0; n < real_cal_num; n++) {

  212.       int src_tile_offset = n * C8NUM;

  213.       int dst_tile_offset = dst_oc_offset + n * oc4 * C4NUM * input_unit_square;

  214.       for (int i = 0; i < 4; i++) {

  215.         int filter_h_offset = filter_oc_offset + i * 4 * ic8 * C8NUM * C4NUM;

  216.         int src_h_offset = src_tile_offset + i * C8NUM * ic8 * C8NUM * C4NUM;

  217.         int dst_h_offset = dst_tile_offset + i * 4 * 4;

  218.         for (int m = 0; m < 4; m++) {

  219.           int filter_w_offset = filter_h_offset + m * 4 * C8NUM * ic8;

  220.           int src_w_offset = src_h_offset + m * 8 * ic8 * C8NUM;

  221.           int dst_w_offset = dst_h_offset + m * C4NUM;

  222. 

  223.           int32_t acc[4] = {0};

  224.           for (int z = 0; z < 4; z++) {

  225.             int filter_offset = filter_w_offset + z;

  226.             for (int j = 0; j < ic8; j++) {

  227.               int filter_c8_offset = filter_offset + j * 4 * 8;

  228.               int src_c8_offset = src_w_offset + j * 8 * 8;

  229. 

  230.               for (int k = 0; k < 8; k++) {

  231.                 const int16_t *w_ptr = weight + filter_c8_offset + k * 4;

  232.                 const int16_t *input_ptr = src + src_c8_offset + k;

  233.                 acc[z] += w_ptr[0] * input_ptr[0];

  234.               }

  235.             }

  236.             (dst + dst_w_offset + z)[0] = acc[z];

  237.           }

  238.         }

  239.       }

  240.     }

  241.   }

  242. #endif

  243. }

  244. 

  245. // int8 conv common

  246. void ConvInt8(int8_t *input_data, int8_t *packed_input, int8_t *packed_weight, const int32_t *bias_data,

  247.               int32_t *tmp_dst, int8_t *tmp_out, int8_t *output_data, int32_t *input_sum, int task_id,

  248.               ConvParameter *conv_param) {

  249.   int kernel_h = conv_param->kernel_h_;

  250.   int kernel_w = conv_param->kernel_w_;

  251.   int in_batch = conv_param->input_batch_;

  252.   int in_channel = conv_param->input_channel_;

  253.   int in_h = conv_param->input_h_;

  254.   int in_w = conv_param->input_w_;

  255.   int out_h = conv_param->output_h_;

  256.   int out_w = conv_param->output_w_;

  257.   int out_channel = conv_param->output_channel_;

  258.   int oc4 = UP_DIV(out_channel, C4NUM);

  259.   int32_t input_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  260. 

  261.   int tile_n = conv_param->tile_num_;

  262.   int thread_count = conv_param->thread_num_;

  263.   int output_count = out_h * out_w;

  264.   int output_tile_count = UP_DIV(output_count, tile_n);

  265.   int ic4 = UP_DIV(in_channel, C4NUM);

  266.   int kernel_plane = kernel_h * kernel_w;

  267.   int plane_block = UP_DIV(kernel_plane, C4NUM);

  268.   int unit_size = plane_block * C4NUM * ic4 * C4NUM;

  269.   int packed_input_size = output_tile_count * tile_n * unit_size;

  270.   int input_sum_offset;

  271.   if (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL) {

  272.     input_sum_offset = tile_n * oc4 * C4NUM;

  273.   } else {

  274.     input_sum_offset = tile_n;

  275.   }

  276. 

  277.   for (int b = 0; b < in_batch; b++) {

  278.     int in_batch_offset = b * ic4 * C4NUM * in_h * in_w;

  279.     int out_batch_offset = b * out_channel * out_h * out_w;

  280.     int gemm_in_batch_offset = b * packed_input_size;

  281.     for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {

  282.       int start_index = thread_id * tile_n;

  283.       int real_cal_num = (output_count - start_index) < tile_n ? (output_count - start_index) : tile_n;

  284.       int32_t *tmp_input_sum = input_sum + task_id * input_sum_offset;

  285.       int8_t *gemm_input = packed_input + thread_id * unit_size * tile_n + gemm_in_batch_offset;

  286.       // clear tmp buffer before compute

  287.       memset(gemm_input, (int8_t)input_zp, unit_size * tile_n);

  288.       int out_offset = thread_id * tile_n * out_channel + out_batch_offset;

  289. 

  290.       size_t tmp_dst_size = tile_n * conv_param->output_channel_ * sizeof(int32_t);

  291.       int tmp_dst_offset = task_id * tile_n * conv_param->output_channel_;

  292.       memset(tmp_dst + tmp_dst_offset, 0, tmp_dst_size);

  293. 

  294.       Im2ColPackUnitInt8(input_data + in_batch_offset, gemm_input, real_cal_num, start_index, tmp_input_sum,

  295.                          conv_param);

  296.       if (real_cal_num == tile_n) {

  297.         int8_t *gemm_output = output_data + out_offset;

  298.         IndirectGemmInt8(gemm_output, tmp_dst + tmp_dst_offset, gemm_input, packed_weight, bias_data, ic4, kernel_plane,

  299.                          out_channel, tmp_input_sum, conv_param);

  300.       } else {

  301.         // res part

  302.         int8_t *tmp_out_ptr = tmp_out + task_id * tile_n * out_channel;

  303.         IndirectGemmInt8(tmp_out_ptr, tmp_dst + tmp_dst_offset, gemm_input, packed_weight, bias_data, ic4, kernel_plane,

  304.                          out_channel, tmp_input_sum, conv_param);

  305.         memcpy(output_data + out_offset, tmp_out_ptr, real_cal_num * out_channel);

  306.       }

  307.     }

  308.   }

  309. }

  310. 

  311. void ConvInt8Opt(int8_t *input_data, int8_t *packed_input, int8_t *packed_weight, const int32_t *bias_data,

  312.                  int32_t *tmp_dst, int8_t *tmp_out, int8_t *output_data, int32_t *input_sum, int task_id,

  313.                  ConvParameter *conv_param, GEMM_FUNC gemm_func) {

  314.   int kernel_h = conv_param->kernel_h_;

  315.   int kernel_w = conv_param->kernel_w_;

  316.   int in_batch = conv_param->input_batch_;

  317.   int in_channel = conv_param->input_channel_;

  318.   int in_h = conv_param->input_h_;

  319.   int in_w = conv_param->input_w_;

  320.   int out_h = conv_param->output_h_;

  321.   int out_w = conv_param->output_w_;

  322.   int out_channel = conv_param->output_channel_;

  323.   int oc4 = UP_DIV(out_channel, C4NUM);

  324.   int32_t input_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  325.   int tile_n = conv_param->tile_num_;

  326.   int thread_count = conv_param->thread_num_;

  327.   int output_count = out_h * out_w;

  328.   int output_tile_count = UP_DIV(output_count, tile_n);

  329.   int ic4 = UP_DIV(in_channel, C4NUM);

  330.   int kernel_plane = kernel_h * kernel_w;

  331.   int unit_size = kernel_plane * ic4 * C4NUM;

  332.   int packed_input_size = output_tile_count * tile_n * unit_size;

  333.   int input_sum_offset;

  334.   if (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL) {

  335.     input_sum_offset = tile_n * oc4 * C4NUM;

  336.   } else {

  337.     input_sum_offset = tile_n;

  338.   }

  339. 

  340.   for (int b = 0; b < in_batch; b++) {

  341.     int in_batch_offset = b * ic4 * C4NUM * in_h * in_w;

  342.     int out_batch_offset = b * out_channel * out_h * out_w;

  343.     int gemm_in_batch_offset = b * packed_input_size;

  344.     for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {

  345.       int start_index = thread_id * tile_n;

  346.       int real_cal_num = (output_count - start_index) < tile_n ? (output_count - start_index) : tile_n;

  347.       int32_t *tmp_input_sum = input_sum + task_id * input_sum_offset;

  348.       int8_t *gemm_input = packed_input + thread_id * unit_size * tile_n + gemm_in_batch_offset;

  349.       // clear tmp buffer before compute

  350.       memset(gemm_input, (int8_t)input_zp, unit_size * tile_n);

  351.       int out_offset = thread_id * tile_n * out_channel + out_batch_offset;

  352. 

  353.       size_t tmp_dst_size = tile_n * conv_param->output_channel_ * sizeof(int32_t);

  354.       int tmp_dst_offset = task_id * tile_n * conv_param->output_channel_;

  355.       memset(tmp_dst + tmp_dst_offset, 0, tmp_dst_size);

  356. 

  357.       Im2ColPackUnitInt8Opt(input_data + in_batch_offset, gemm_input, real_cal_num, start_index, tmp_input_sum,

  358.                             conv_param);

  359.       if (real_cal_num == tile_n) {

  360.         int8_t *gemm_output = output_data + out_offset;

  361.         IndirectGemmInt8Opt(gemm_output, tmp_dst + tmp_dst_offset, gemm_input, packed_weight, bias_data, ic4,

  362.                             kernel_plane, out_channel, tmp_input_sum, conv_param, gemm_func);

  363.       } else {

  364.         // res part

  365.         int8_t *tmp_out_ptr = tmp_out + task_id * tile_n * out_channel;

  366.         IndirectGemmInt8Opt(tmp_out_ptr, tmp_dst + tmp_dst_offset, gemm_input, packed_weight, bias_data, ic4,

  367.                             kernel_plane, out_channel, tmp_input_sum, conv_param, gemm_func);

  368.         memcpy(output_data + out_offset, tmp_out_ptr, real_cal_num * out_channel);

  369.       }

  370.     }

  371.   }

  372. }

  373. 

  374. void Conv1x1PreOptPeroc(const int8_t *src_input, int8_t *packed_input, int32_t *input_sum, size_t input_channel,

  375.                         size_t output_channel, size_t plane_size, int32_t *filter_zp, size_t inputsum_stride) {

  376.   int ic4 = UP_ROUND(input_channel, C4NUM);

  377.   int oc8 = UP_ROUND(output_channel, C8NUM);

  378.   int hw8 = UP_ROUND(plane_size, C8NUM);

  379.   size_t hw_8div = plane_size / C8NUM * C8NUM;

  380.   size_t oc_8div = output_channel / C8NUM * C8NUM;

  381.   size_t oc_8res = output_channel - oc_8div;

  382.   size_t ic_4div = input_channel / C4NUM * C4NUM;

  383. 

  384.   const int8_t *src_r = src_input;

  385.   int8_t *pack_r = packed_input;

  386.   int32_t *input_sum_r = input_sum;

  387. 

  388.   /* per layer */

  389.   for (int hwi = 0; hwi < hw_8div; hwi += C8NUM) {

  390.     const int8_t *src_ic = src_r;

  391.     int8_t *pack_ic = pack_r;

  392.     int32_t *input_sum_oc = input_sum_r;

  393. 

  394.     int32_t tmp_sum_value[8] = {0};

  395.     for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  396.       for (int i = 0; i < C8NUM; i++) {

  397.         tmp_sum_value[i] += src_ic[0 + i * input_channel];

  398.         tmp_sum_value[i] += src_ic[1 + i * input_channel];

  399.         tmp_sum_value[i] += src_ic[2 + i * input_channel];

  400.         tmp_sum_value[i] += src_ic[3 + i * input_channel];

  401.         pack_ic[0 + i * C4NUM] = src_ic[0 + i * input_channel];

  402.         pack_ic[1 + i * C4NUM] = src_ic[1 + i * input_channel];

  403.         pack_ic[2 + i * C4NUM] = src_ic[2 + i * input_channel];

  404.         pack_ic[3 + i * C4NUM] = src_ic[3 + i * input_channel];

  405.       }

  406.       src_ic += C4NUM;

  407.       pack_ic += C4NUM * C8NUM;

  408.     }

  409.     for (int ici = ic_4div; ici < input_channel; ici += 1) {

  410.       for (int i = 0; i < C8NUM; i++) {

  411.         tmp_sum_value[i] += src_ic[i * input_channel];

  412.         pack_ic[i * C4NUM] = src_ic[i * input_channel];

  413.       }

  414.       src_ic += 1;

  415.       pack_ic += 1;

  416.     }

  417. 

  418.     for (int oci = 0; oci < oc_8div; oci += C8NUM) {

  419.       for (int ri = 0; ri < C8NUM; ri++) {

  420.         input_sum_oc[ri * C8NUM + 0] = tmp_sum_value[ri] * filter_zp[oci + 0];

  421.         input_sum_oc[ri * C8NUM + 1] = tmp_sum_value[ri] * filter_zp[oci + 1];

  422.         input_sum_oc[ri * C8NUM + 2] = tmp_sum_value[ri] * filter_zp[oci + 2];

  423.         input_sum_oc[ri * C8NUM + 3] = tmp_sum_value[ri] * filter_zp[oci + 3];

  424.         input_sum_oc[ri * C8NUM + 4] = tmp_sum_value[ri] * filter_zp[oci + 4];

  425.         input_sum_oc[ri * C8NUM + 5] = tmp_sum_value[ri] * filter_zp[oci + 5];

  426.         input_sum_oc[ri * C8NUM + 6] = tmp_sum_value[ri] * filter_zp[oci + 6];

  427.         input_sum_oc[ri * C8NUM + 7] = tmp_sum_value[ri] * filter_zp[oci + 7];

  428.       }

  429.       input_sum_oc += inputsum_stride;

  430.     }

  431.     if (oc_8div != output_channel) {

  432.       for (int oci = 0; oci < oc_8res; oci += 1) {

  433.         for (int ri = 0; ri < C8NUM; ri++) {

  434.           input_sum_oc[ri * C8NUM + oci] = tmp_sum_value[ri] * filter_zp[oc_8div + oci];

  435.         }

  436.       }

  437.       for (int oci = oc_8res; oci < C8NUM; oci += 1) {

  438.         for (int ri = 0; ri < C8NUM; ri++) {

  439.           input_sum_oc[ri * C8NUM + oci] = 0;

  440.         }

  441.       }

  442.     } /* oc8 res done */

  443. 

  444.     src_r += input_channel * C8NUM;

  445.     pack_r += ic4 * C8NUM;

  446.     input_sum_r += C8NUM * C8NUM;

  447.   }

  448. 

  449.   if (hw_8div != plane_size) {

  450.     memset(pack_r, 0, C8NUM * ic4);

  451.     for (int hwi = hw_8div; hwi < plane_size; hwi += 1) {

  452.       int32_t *input_sum_oc = input_sum_r;

  453.       int32_t tmp_sum_value = 0;

  454.       const int8_t *src_ic = src_r;

  455.       int8_t *pack_ic = pack_r;

  456.       for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  457.         tmp_sum_value += src_ic[0];

  458.         tmp_sum_value += src_ic[1];

  459.         tmp_sum_value += src_ic[2];

  460.         tmp_sum_value += src_ic[3];

  461.         pack_ic[0] = src_ic[0];

  462.         pack_ic[1] = src_ic[1];

  463.         pack_ic[2] = src_ic[2];

  464.         pack_ic[3] = src_ic[3];

  465.         src_ic += C4NUM;

  466.         pack_ic += C4NUM * C8NUM;

  467.       }

  468.       for (int ici = ic_4div; ici < input_channel; ici += 1) {

  469.         tmp_sum_value += src_ic[0];

  470.         pack_ic[0] = src_ic[0];

  471.         src_ic += 1;

  472.         pack_ic += 1;

  473.       }

  474. 

  475.       for (int oci = 0; oci < oc_8div; oci += C8NUM) {

  476.         for (int curoi = 0; curoi < C8NUM; curoi++) {

  477.           input_sum_oc[curoi] = tmp_sum_value * filter_zp[oci + curoi];

  478.         }

  479.         input_sum_oc += inputsum_stride;

  480.       }

  481.       if (oc_8div != output_channel) {

  482.         for (int oci = 0; oci < oc_8res; oci += 1) {

  483.           input_sum_oc[oci] = tmp_sum_value * filter_zp[oc_8div + oci];

  484.         }

  485.         for (int oci = oc_8res; oci < C8NUM; oci += 1) {

  486.           input_sum_oc[oci] = 0;

  487.         }

  488.       } /* oc8 res done */

  489. 

  490.       src_r += input_channel;

  491.       pack_r += C4NUM;

  492.       input_sum_r += C8NUM;

  493.     }

  494. 

  495.     for (int hwi = plane_size; hwi < hw8; hwi++) {

  496.       for (int oc = 0; oc < oc8; oc++) {

  497.         int oc8div = oc / C8NUM, oc8res = oc % C8NUM;

  498.         input_sum[oc8div * inputsum_stride + hwi * C8NUM + oc8res] = 0;

  499.       }

  500.     }

  501.   }

  502.   return;

  503. }

  504. 

  505. void Conv1x1PreOptPert(const int8_t *src_input, int8_t *packed_input, int32_t *input_sum, size_t input_channel,

  506.                        size_t plane_size, ConvParameter *conv_param) {

  507.   int ic4 = UP_ROUND(input_channel, C4NUM);

  508.   size_t hw_8div = plane_size / C8NUM * C8NUM;

  509.   size_t ic_4div = input_channel / C4NUM * C4NUM;

  510.   int32_t filter_zp = conv_param->conv_quant_arg_.filter_quant_args_[0].zp_;

  511. 

  512.   const int8_t *src_r = src_input;

  513.   int8_t *pack_r = packed_input;

  514.   /* per layer */

  515.   for (int hwi = 0; hwi < hw_8div; hwi += C8NUM) {

  516.     const int8_t *src_ic = src_r;

  517.     int8_t *pack_ic = pack_r;

  518.     int32_t *input_sum_r = input_sum + hwi;

  519. #ifdef ENABLE_ARM64

  520.     size_t src_stride = input_channel;

  521.     size_t ic_4res = input_channel - ic_4div;

  522.     asm volatile(

  523.       "dup v10.4s, wzr \n"

  524.       "dup v11.4s, wzr \n"

  525.       "mov x20, %[input_sum_r] \n"

  526.       "dup v20.4s, %w[filter_zp]  \n"

  527. 

  528.       "mov x10, %[src_ic] \n"

  529.       "mov x11, %[pack_ic] \n"

  530. 

  531.       "mov x0, #0 \n"

  532.       "1: \n"

  533.       "cmp x0, %[ic_4div] \n"

  534.       "add x0, x0, #4\n"

  535.       "mov x12, x10 \n"

  536.       "add x10, x10, #4\n"

  537.       "blt 2f \n"

  538.       "cmp %[ic_4res], #0\n"

  539.       "beq 6f \n"

  540.       "cmp %[ic_4res], #1\n"

  541.       "beq 3f \n"

  542.       "cmp %[ic_4res], #2\n"

  543.       "beq 4f \n"

  544.       "cmp %[ic_4res], #3\n"

  545.       "beq 5f \n"

  546. 

  547.       "2: \n"

  548.       "ld1 {v0.s}[0], [x12], %[src_stride]\n"

  549.       "ld1 {v0.s}[1], [x12], %[src_stride]\n"

  550.       "ld1 {v0.s}[2], [x12], %[src_stride]\n"

  551.       "ld1 {v0.s}[3], [x12], %[src_stride]\n"

  552.       "ld1 {v1.s}[0], [x12], %[src_stride]\n"

  553.       "ld1 {v1.s}[1], [x12], %[src_stride]\n"

  554.       "ld1 {v1.s}[2], [x12], %[src_stride]\n"

  555.       "ld1 {v1.s}[3], [x12], %[src_stride]\n"

  556. 

  557.       "st1 {v0.16b}, [x11], #16\n"

  558.       "st1 {v1.16b}, [x11], #16\n"

  559. 

  560.       "saddlp v4.8h, v0.16b \n"

  561.       "saddlp v5.8h, v1.16b \n"

  562. 

  563.       "saddlp v0.4s, v4.8h \n"

  564.       "saddlp v1.4s, v5.8h \n"

  565. 

  566.       "add v10.4s, v10.4s, v0.4s \n"

  567.       "add v11.4s, v11.4s, v1.4s \n"

  568.       "b 1b \n"

  569. 

  570.       "3: \n" /* col res 1 */

  571.       "dup v0.4s, wzr \n"

  572.       "dup v1.4s, wzr \n"

  573. 

  574.       "ld1 {v0.b}[0],  [x12], %[src_stride]\n"

  575.       "ld1 {v0.b}[4],  [x12], %[src_stride]\n"

  576.       "ld1 {v0.b}[8],  [x12], %[src_stride]\n"

  577.       "ld1 {v0.b}[12], [x12], %[src_stride]\n"

  578.       "ld1 {v1.b}[0],  [x12], %[src_stride]\n"

  579.       "ld1 {v1.b}[4],  [x12], %[src_stride]\n"

  580.       "ld1 {v1.b}[8],  [x12], %[src_stride]\n"

  581.       "ld1 {v1.b}[12], [x12], %[src_stride]\n"

  582. 

  583.       "st1 {v0.16b}, [x11], #16\n"

  584.       "st1 {v1.16b}, [x11], #16\n"

  585.       "saddlp v4.8h, v0.16b \n"

  586.       "saddlp v5.8h, v1.16b \n"

  587.       "saddlp v0.4s, v4.8h \n"

  588.       "saddlp v1.4s, v5.8h \n"

  589.       "add v10.4s, v10.4s, v0.4s \n"

  590.       "add v11.4s, v11.4s, v1.4s \n"

  591.       "b 6f \n"

  592. 

  593.       "4: \n" /* col res 2 */

  594.       "dup v0.4s, wzr \n"

  595.       "dup v1.4s, wzr \n"

  596. 

  597.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  598.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  599.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  600.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  601.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  602.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  603.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  604.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  605. 

  606.       "st1 {v0.16b}, [x11], #16\n"

  607.       "st1 {v1.16b}, [x11], #16\n"

  608.       "saddlp v4.8h, v0.16b \n"

  609.       "saddlp v5.8h, v1.16b \n"

  610.       "saddlp v0.4s, v4.8h \n"

  611.       "saddlp v1.4s, v5.8h \n"

  612.       "add v10.4s, v10.4s, v0.4s \n"

  613.       "add v11.4s, v11.4s, v1.4s \n"

  614.       "b 6f \n"

  615. 

  616.       "5: \n" /* col res 3 */

  617.       "dup v0.4s, wzr \n"

  618.       "dup v1.4s, wzr \n"

  619.       "add x13, x12, #2 \n"

  620. 

  621.       "ld1 {v0.h}[0], [x12], %[src_stride]\n"

  622.       "ld1 {v0.b}[2], [x13], %[src_stride]\n"

  623.       "ld1 {v0.h}[2], [x12], %[src_stride]\n"

  624.       "ld1 {v0.b}[6], [x13], %[src_stride]\n"

  625.       "ld1 {v0.h}[4], [x12], %[src_stride]\n"

  626.       "ld1 {v0.b}[10], [x13], %[src_stride]\n"

  627.       "ld1 {v0.h}[6], [x12], %[src_stride]\n"

  628.       "ld1 {v0.b}[14], [x13], %[src_stride]\n"

  629.       "ld1 {v1.h}[0], [x12], %[src_stride]\n"

  630.       "ld1 {v1.b}[2], [x13], %[src_stride]\n"

  631.       "ld1 {v1.h}[2], [x12], %[src_stride]\n"

  632.       "ld1 {v1.b}[6], [x13], %[src_stride]\n"

  633.       "ld1 {v1.h}[4], [x12], %[src_stride]\n"

  634.       "ld1 {v1.b}[10], [x13], %[src_stride]\n"

  635.       "ld1 {v1.h}[6], [x12], %[src_stride]\n"

  636.       "ld1 {v1.b}[14], [x13], %[src_stride]\n"

  637. 

  638.       "st1 {v0.16b}, [x11], #16\n"

  639.       "st1 {v1.16b}, [x11], #16\n"

  640.       "saddlp v4.8h, v0.16b \n"

  641.       "saddlp v5.8h, v1.16b \n"

  642.       "saddlp v0.4s, v4.8h \n"

  643.       "saddlp v1.4s, v5.8h \n"

  644.       "add v10.4s, v10.4s, v0.4s \n"

  645.       "add v11.4s, v11.4s, v1.4s \n"

  646.       "b 6f \n"

  647. 

  648.       "6: \n"

  649.       "mul v10.4s, v10.4s, v20.4s \n"

  650.       "mul v11.4s, v11.4s, v20.4s \n"

  651. 

  652.       "st1 {v10.4s}, [x20], #16 \n"

  653.       "st1 {v11.4s}, [x20], #16 \n"

  654. 

  655.       :

  656.       : [ src_ic ] "r"(src_ic), [ pack_ic ] "r"(pack_ic), [ input_sum_r ] "r"(input_sum_r),

  657.         [ src_stride ] "r"(src_stride), [ ic_4div ] "r"(ic_4div), [ ic_4res ] "r"(ic_4res), [ filter_zp ] "r"(filter_zp)

  658.       : "x0", "x1", "x10", "x11", "x12", "x13", "x20", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v10", "v11",

  659.         "v20");

  660. #else

  661.     int32_t tmp_sum_value[8] = {0};

  662.     for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  663.       for (int i = 0; i < C8NUM; i++) {

  664.         tmp_sum_value[i] += src_ic[0 + i * input_channel];

  665.         tmp_sum_value[i] += src_ic[1 + i * input_channel];

  666.         tmp_sum_value[i] += src_ic[2 + i * input_channel];

  667.         tmp_sum_value[i] += src_ic[3 + i * input_channel];

  668.         pack_ic[0 + i * C4NUM] = src_ic[0 + i * input_channel];

  669.         pack_ic[1 + i * C4NUM] = src_ic[1 + i * input_channel];

  670.         pack_ic[2 + i * C4NUM] = src_ic[2 + i * input_channel];

  671.         pack_ic[3 + i * C4NUM] = src_ic[3 + i * input_channel];

  672.       }

  673.       src_ic += C4NUM;

  674.       pack_ic += C4NUM * C8NUM;

  675.     }

  676.     for (int ici = ic_4div; ici < input_channel; ici += 1) {

  677.       for (int i = 0; i < C8NUM; i++) {

  678.         tmp_sum_value[i] += src_ic[i * input_channel];

  679.         pack_ic[i * C4NUM] = src_ic[i * input_channel];

  680.       }

  681.       src_ic += 1;

  682.       pack_ic += 1;

  683.     }

  684. 

  685.     for (int i = 0; i < C8NUM; i++) {

  686.       input_sum_r[i] = tmp_sum_value[i] * filter_zp;

  687.     }

  688. #endif

  689.     src_r += input_channel * C8NUM;

  690.     pack_r += ic4 * C8NUM;

  691.   }

  692. 

  693.   if (hw_8div != plane_size) {

  694.     memset(pack_r, 0, C8NUM * ic4);

  695.     for (int hwi = hw_8div; hwi < plane_size; hwi += 1) {

  696.       int32_t tmp_sum_value = 0;

  697.       const int8_t *src_ic = src_r;

  698.       int8_t *pack_ic = pack_r;

  699.       for (int ici = 0; ici < ic_4div; ici += C4NUM) {

  700.         tmp_sum_value += src_ic[0];

  701.         tmp_sum_value += src_ic[1];

  702.         tmp_sum_value += src_ic[2];

  703.         tmp_sum_value += src_ic[3];

  704.         pack_ic[0] = src_ic[0];

  705.         pack_ic[1] = src_ic[1];

  706.         pack_ic[2] = src_ic[2];

  707.         pack_ic[3] = src_ic[3];

  708.         src_ic += C4NUM;

  709.         pack_ic += C4NUM * C8NUM;

  710.       }

  711.       for (int ici = ic_4div; ici < input_channel; ici += 1) {

  712.         tmp_sum_value += src_ic[0];

  713.         pack_ic[0] = src_ic[0];

  714.         src_ic += 1;

  715.         pack_ic += 1;

  716.       }

  717.       input_sum[hwi] = tmp_sum_value * filter_zp;

  718.       src_r += input_channel;

  719.       pack_r += C4NUM;

  720.     }

  721.     for (int hwi = plane_size; hwi < UP_ROUND(plane_size, C8NUM); hwi++) {

  722.       input_sum[hwi] = 0;

  723.     }

  724.   }

  725.   return;

  726. }

  727. 

  728. void Conv1x1Int8Opt(const int8_t *packed_input, const int8_t *packed_weight, int8_t *dst, const int32_t *input_sum,

  729.                     const int32_t *bias, int row, int col, int deep4, int32_t *left_shift, int32_t *right_shift,

  730.                     int32_t *multiplier, ConvParameter *conv_param, MATMUL_OPT_R_FUNC matmul_func) {

  731.   matmul_func(packed_input, packed_weight, dst, row, col, deep4, conv_param->output_channel_, input_sum, bias,

  732.               left_shift, right_shift, multiplier, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  733.               conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  734.               conv_param->conv_quant_arg_.filter_arg_num_ != 1);

  735.   return;

  736. }

  737. 

  738. void Conv1x1Int8(const int8_t *packed_input, const int8_t *packed_weight, int8_t *dst, const int32_t *input_sum,

  739.                  const int32_t *bias, int row, int col, int deep16, int32_t *left_shift, int32_t *right_shift,

  740.                  int32_t *multiplier, ConvParameter *conv_param) {

  741.   if (conv_param->conv_quant_arg_.filter_arg_num_ > 1) {

  742.     return MatMulInt8_16x4_r(packed_input, packed_weight, dst, row, col, deep16, conv_param->output_channel_, input_sum,

  743.                              bias, left_shift, right_shift, multiplier,

  744.                              conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  745.                              conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  746.                              conv_param->conv_quant_arg_.filter_arg_num_ != 1);

  747.   }

  748. #ifdef ENABLE_ARM64

  749.   MatmulInt8Neon64(packed_input, packed_weight, dst, UP_ROUND(row, C4NUM), UP_ROUND(col, C4NUM), deep16, input_sum,

  750.                    bias, conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  751.                    conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  752.                    conv_param->conv_quant_arg_.quant_multiplier_[0], conv_param->conv_quant_arg_.left_shift_[0],

  753.                    conv_param->conv_quant_arg_.right_shift_[0], row, col, conv_param->output_channel_);

  754. #else

  755.   MatMulInt8_16x4_r(packed_input, packed_weight, dst, row, col, deep16, conv_param->output_channel_, input_sum, bias,

  756.                     left_shift, right_shift, multiplier, conv_param->conv_quant_arg_.output_quant_args_[0].zp_,

  757.                     conv_param->conv_quant_arg_.out_act_min_[0], conv_param->conv_quant_arg_.out_act_max_[0],

  758.                     conv_param->conv_quant_arg_.filter_arg_num_ != 1);

  759. #endif

  760.   return;

  761. }

  762. 

  763. // int8 convolution 3x3

  764. void Conv3x3Int8(int16_t *input_data, int16_t *transed_weight, const int32_t *bias_data, int8_t *output_data,

  765.                  int16_t *tile_buffer, int16_t *block_unit_buffer, int32_t *tmp_dst_buffer, int8_t *tmp_out,

  766.                  int task_id, ConvParameter *conv_param) {

  767.   int thread_count = conv_param->thread_num_;

  768.   int ic8 = UP_DIV(conv_param->input_channel_, C8NUM);

  769.   int output_channel = conv_param->output_channel_;

  770.   int out_w_block = UP_DIV(conv_param->output_w_, OUPUT_UNIT);

  771.   int out_h_block = UP_DIV(conv_param->output_h_, OUPUT_UNIT);

  772.   int output_count = out_w_block * out_h_block;

  773.   int output_tile_count = UP_DIV(output_count, TILE_NUM);

  774.   int oc4 = UP_DIV(output_channel, C4NUM);

  775.   int tile_buffer_offset = TILE_NUM * 16 * ic8 * C8NUM;

  776.   const int block_unit_buffer_offset = 16 * C8NUM;

  777.   int tmp_dst_buffer_offset = TILE_NUM * 16 * oc4 * C4NUM;

  778. 

  779.   int input_batch = conv_param->input_batch_;

  780.   for (int batch = 0; batch < input_batch; batch++) {

  781.     int in_batch_offset = batch * ic8 * C8NUM * conv_param->input_h_ * conv_param->input_w_;

  782.     int tmp_out_batch_offset = batch * oc4 * C4NUM * conv_param->output_w_ * conv_param->output_h_;

  783.     for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {

  784.       int start_index = thread_id * TILE_NUM;

  785.       int real_cal_num = (output_count - start_index) < TILE_NUM ? (output_count - start_index) : TILE_NUM;

  786. 

  787.       Conv3x3Int8InputTransform(input_data + in_batch_offset, tile_buffer + task_id * tile_buffer_offset,

  788.                                 block_unit_buffer + task_id * block_unit_buffer_offset, start_index, real_cal_num,

  789.                                 out_w_block, conv_param);

  790. 

  791.       Conv3x3Int8Gemm(tmp_dst_buffer + task_id * tmp_dst_buffer_offset, tile_buffer + task_id * tile_buffer_offset,

  792.                       transed_weight, output_channel, ic8, real_cal_num);

  793. 

  794.       Conv3x3Int8OutputTransform(tmp_dst_buffer + task_id * tmp_dst_buffer_offset, tmp_out + tmp_out_batch_offset,

  795.                                  bias_data, start_index, real_cal_num, out_w_block, conv_param);

  796.     }

  797.   }

  798. }