Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: efe805655e
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'efe805655e'
${ noResults }
mindspore/mindspore/lite/nnacl/pack.c

1199 lines
49 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/pack.h"

  18. #include <string.h>

  19. #include <stdlib.h>

  20. 

  21. void PackWeightFp32(float *weight_data, ConvParameter *conv_param, float *packed_weight, int oc_block,

  22.                     int oc_block_num) {

  23.   // original weight format : ohwi

  24.   if (oc_block_num == 0) {

  25.     return;

  26.   }

  27.   int kernel_h = conv_param->kernel_h_;

  28.   int kernel_w = conv_param->kernel_w_;

  29.   int in_channel = conv_param->input_channel_;

  30.   int out_channel = conv_param->output_channel_;

  31.   int ic4 = UP_DIV(in_channel, C4NUM);

  32.   int kernel_plane = kernel_h * kernel_w;

  33.   int pack_weight_size = oc_block * oc_block_num * ic4 * C4NUM * kernel_plane;

  34. 

  35.   int unit_size = oc_block * C4NUM;

  36.   const int block_size = pack_weight_size / oc_block_num;

  37. 

  38.   for (int m = 0; m < kernel_plane; m++) {

  39.     int kernel_plane_stride = m * in_channel;

  40.     int packed_kernel_plane_stride = m * unit_size * ic4;

  41.     for (int i = 0; i < ic4; i++) {

  42.       int channel_block_stride = kernel_plane_stride + i * C4NUM;

  43.       int packed_channel_block_size = packed_kernel_plane_stride + i * unit_size;

  44.       int ic_remainder = in_channel - i * C4NUM;

  45.       int real_ic_num = ic_remainder < C4NUM ? ic_remainder : C4NUM;

  46.       for (int h = 0; h < real_ic_num; h++) {

  47.         int block_stride = channel_block_stride + h;

  48.         int packed_block_stride = packed_channel_block_size + h * oc_block;

  49.         for (int j = 0; j < oc_block_num; j++) {

  50.           int kernel_block_stride = block_stride + j * oc_block * kernel_plane * in_channel;

  51.           int packed_kernel_block_size = packed_block_stride + j * block_size;

  52.           int oc_remainder = out_channel - j * oc_block;

  53.           int real_oc_num = oc_remainder < oc_block ? oc_remainder : oc_block;

  54.           for (int k = 0; k < real_oc_num; k++) {

  55.             float *origin_data_ptr = weight_data + kernel_block_stride + k * kernel_plane * in_channel;

  56.             float *packed_data_ptr = packed_weight + packed_kernel_block_size + k;

  57.             *packed_data_ptr = *origin_data_ptr;

  58.           }

  59.         }  // kernel block loop

  60.       }    // inchannel block loop

  61.     }      // channel block loop

  62.   }        // kernel plane loop

  63. }

  64. 

  65. void PackWeightKHWToHWKFp32(const void *src, void *dst, int plane, int channel) {

  66.   return PackNCHWToNHWCFp32(src, dst, 1, plane, channel);

  67. }

  68. 

  69. void PackWeightInt8(int8_t *weight_data, ConvParameter *conv_param, int8_t *packed_weight, int32_t *weight_sum) {

  70.   // original weight format : ohwi

  71.   int kernel_h = conv_param->kernel_h_;

  72.   int kernel_w = conv_param->kernel_w_;

  73.   int in_channel = conv_param->input_channel_;

  74.   int out_channel = conv_param->output_channel_;

  75.   int oc4 = UP_DIV(out_channel, C4NUM);

  76.   int ic4 = UP_DIV(in_channel, C4NUM);

  77.   int kernel_plane = kernel_h * kernel_w;

  78.   int plane_c4 = UP_DIV(kernel_plane, C4NUM);

  79.   int pack_weight_size = oc4 * C4NUM * ic4 * C4NUM * plane_c4 * C4NUM;

  80.   int block_size = pack_weight_size / oc4;

  81. 

  82.   for (int m = 0; m < kernel_plane; m++) {

  83.     int kernel_plane_stride = m * in_channel;

  84.     int plane_block = m / C4NUM;

  85.     int plane_res = m % C4NUM;

  86.     int packed_kernel_plane_stride = plane_block * C4NUM * C4NUM * ic4 * C4NUM + plane_res * C4NUM;

  87.     for (int i = 0; i < ic4; i++) {

  88.       int channel_block_stride = kernel_plane_stride + i * C4NUM;

  89.       int packed_channel_block_size = packed_kernel_plane_stride + i * C4NUM * C4NUM * C4NUM;

  90.       int ic_remainder = in_channel - i * C4NUM;

  91.       int real_ic_num = ic_remainder < C4NUM ? ic_remainder : C4NUM;

  92.       for (int h = 0; h < real_ic_num; h++) {

  93.         int block_stride = channel_block_stride + h;

  94.         int packed_block_stride = packed_channel_block_size + h;

  95.         for (int j = 0; j < oc4; j++) {

  96.           int kernel_block_stride = block_stride + j * C4NUM * kernel_plane * in_channel;

  97.           int packed_kernel_block_size = packed_block_stride + j * block_size;

  98.           int oc_remainder = out_channel - j * C4NUM;

  99.           int real_oc_num = oc_remainder < C4NUM ? oc_remainder : C4NUM;

  100.           for (int k = 0; k < real_oc_num; k++) {

  101.             int8_t *origin_data_ptr = weight_data + kernel_block_stride + k * kernel_plane * in_channel;

  102.             int8_t *packed_data_ptr = packed_weight + packed_kernel_block_size + k * C4NUM * C4NUM;

  103.             *packed_data_ptr = origin_data_ptr[0];

  104.             // value of weight must between [-127, 127]

  105.             if (packed_data_ptr[0] == -128) {

  106.               packed_data_ptr[0] = -127;

  107.             }

  108.             weight_sum[j * C4NUM + k] += (int32_t)packed_data_ptr[0];

  109.           }

  110.         }  // kernel block loop

  111.       }    // inchannel block loop

  112.     }      // channel block loop

  113.   }        // kernel plane loop

  114. }

  115. 

  116. void PackWeightInt8Opt(int8_t *weight_data, ConvParameter *conv_param, int8_t *packed_weight, int32_t *weight_sum) {

  117.   // original weight format : ohwi

  118.   int kernel_h = conv_param->kernel_h_;

  119.   int kernel_w = conv_param->kernel_w_;

  120.   int in_channel = conv_param->input_channel_;

  121.   int out_channel = conv_param->output_channel_;

  122.   int oc4 = UP_DIV(out_channel, C4NUM);

  123.   int ic4 = UP_DIV(in_channel, C4NUM);

  124.   int kernel_plane = kernel_h * kernel_w;

  125.   int pack_weight_size = oc4 * ic4 * C4NUM * C4NUM * kernel_plane;

  126.   int unit_size = C4NUM * C4NUM;

  127.   int block_size = pack_weight_size / oc4;

  128. 

  129.   for (int m = 0; m < kernel_plane; m++) {

  130.     int kernel_plane_stride = m * in_channel;

  131.     int packed_kernel_plane_stride = m * unit_size * ic4;

  132.     for (int i = 0; i < ic4; i++) {

  133.       int channel_block_stride = kernel_plane_stride + i * C4NUM;

  134.       int packed_channel_block_size = packed_kernel_plane_stride + i * unit_size;

  135.       int ic_remainder = in_channel - i * C4NUM;

  136.       int real_ic_num = ic_remainder < C4NUM ? ic_remainder : C4NUM;

  137.       for (int h = 0; h < real_ic_num; h++) {

  138.         int block_stride = channel_block_stride + h;

  139.         int packed_block_stride = packed_channel_block_size + h;

  140.         for (int j = 0; j < oc4; j++) {

  141.           int kernel_block_stride = block_stride + j * C4NUM * kernel_plane * in_channel;

  142.           int packed_kernel_block_size = packed_block_stride + j * block_size;

  143.           int oc_remainder = out_channel - j * C4NUM;

  144.           int real_oc_num = oc_remainder < C4NUM ? oc_remainder : C4NUM;

  145.           for (int k = 0; k < real_oc_num; k++) {

  146.             int8_t *origin_data_ptr = weight_data + kernel_block_stride + k * kernel_plane * in_channel;

  147.             int8_t *packed_data_ptr = packed_weight + packed_kernel_block_size + k * C4NUM;

  148.             *packed_data_ptr = origin_data_ptr[0];

  149.             if (packed_data_ptr[0] == -128) {

  150.               packed_data_ptr[0] = -127;

  151.             }

  152.             weight_sum[j * C4NUM + k] += (int32_t)(packed_data_ptr[0]);

  153.           }

  154.         }  // kernel block loop

  155.       }    // inchannel block loop

  156.     }      // channel block loop

  157.   }        // kernel plane loop

  158. }

  159. 

  160. void Conv1x1InputPack(const void *src_ptr, void *dst_ptr, ConvParameter *conv_param, int data_size) {

  161.   /* support nhwc */

  162.   char *src = (char *)src_ptr;

  163.   char *dst = (char *)dst_ptr;

  164.   for (int dst_h = 0; dst_h < conv_param->output_h_; dst_h++) {

  165.     int src_h = dst_h * conv_param->stride_h_ - conv_param->pad_u_;

  166.     if (src_h < 0 || src_h >= conv_param->input_h_) {

  167.       continue;

  168.     }

  169.     const char *src_h_ptr = src + src_h * conv_param->input_w_ * conv_param->input_channel_ * data_size;

  170.     char *dst_h_ptr = dst + dst_h * conv_param->output_w_ * conv_param->input_channel_ * data_size;

  171.     for (int dst_w = 0; dst_w < conv_param->output_w_; dst_w++) {

  172.       int src_w = dst_w * conv_param->stride_w_ - conv_param->pad_l_;

  173.       if (src_w < 0 || src_w >= conv_param->input_w_) {

  174.         continue;

  175.       }

  176.       memcpy(dst_h_ptr + dst_w * conv_param->input_channel_ * data_size,

  177.              src_h_ptr + src_w * conv_param->input_channel_ * data_size, conv_param->input_channel_ * data_size);

  178.     }

  179.   }

  180.   return;

  181. }

  182. 

  183. void Pack1x1WeightFp32(const float *weight_data, float *packed_weight, ConvParameter *conv_param) {

  184.   int c4 = UP_ROUND(conv_param->input_channel_, C4NUM);

  185.   for (int ic = 0; ic < conv_param->input_channel_; ic++) {

  186.     for (int oc = 0; oc < conv_param->output_channel_; oc++) {

  187.       int oc4mod = oc % 4;

  188.       int oc4div = oc / 4;

  189.       int dst_index = oc4div * c4 * C4NUM + ic * C4NUM + oc4mod;

  190.       int src_index = oc * conv_param->input_channel_ + ic;

  191.       packed_weight[dst_index] = weight_data[src_index];

  192.     }

  193.   }

  194.   return;

  195. }

  196. 

  197. void PackInputSum16x4PerLayer(const int8_t *src, int32_t *dst, int32_t filter_zp, size_t row4, size_t col16) {

  198.   /* optimize normal -> same layout */

  199. #ifdef ENABLE_ARM64

  200.   asm volatile(

  201.     "mov x10, %[src] \n"

  202.     "mov x11, %[dst] \n"

  203.     "dup v15.4s, %w[filter_zp]  \n"

  204. 

  205.     "mov x0, #0 \n"

  206.     "1: \n"

  207.     "cmp x0, %[row4] \n"

  208.     "beq 4f \n"

  209.     "add x0, x0, #4\n"

  210.     "dup v10.4s, wzr \n"

  211.     "mov x2, #0 \n"

  212. 

  213.     "2: \n"

  214.     "cmp x2, %[col16] \n"

  215.     "beq 3f \n"

  216.     "add x2, x2, #16\n"

  217. 

  218.     "ld1 {v0.16b}, [x10], #16\n"

  219.     "ld1 {v1.16b}, [x10], #16\n"

  220.     "ld1 {v2.16b}, [x10], #16\n"

  221.     "ld1 {v3.16b}, [x10], #16\n"

  222. 

  223.     "saddlp v4.8h, v0.16b \n"

  224.     "saddlp v5.8h, v1.16b \n"

  225.     "saddlp v6.8h, v2.16b \n"

  226.     "saddlp v7.8h, v3.16b \n"

  227. 

  228.     "saddlp v0.4S, v4.8h \n"

  229.     "saddlp v1.4S, v5.8h \n"

  230.     "saddlp v2.4S, v6.8h \n"

  231.     "saddlp v3.4S, v7.8h \n"

  232. 

  233.     "addv s4, v0.4S \n"

  234.     "addv s5, v1.4S \n"

  235.     "addv s6, v2.4S \n"

  236.     "addv s7, v3.4S \n"

  237. 

  238.     "mov v0.s[0], v4.s[0] \n"

  239.     "mov v0.s[1], v5.s[0] \n"

  240.     "mov v0.s[2], v6.s[0] \n"

  241.     "mov v0.s[3], v7.s[0] \n"

  242. 

  243.     "add v10.4s, v10.4s, v0.4s \n"

  244.     "b 2b\n"

  245. 

  246.     "3: \n"

  247.     "mul v10.4s, v10.4s, v15.4s \n"

  248.     "st1 {v10.4s}, [x11], #16 \n"

  249.     "beq 1b \n"

  250. 

  251.     "4: \n"

  252. 

  253.     :

  254.     : [ dst ] "r"(dst), [ src ] "r"(src), [ row4 ] "r"(row4), [ col16 ] "r"(col16), [ filter_zp ] "r"(filter_zp)

  255.     : "x0", "x1", "x2", "x3", "x10", "x11", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v10", "v15");

  256. #else

  257.   for (int r = 0; r < row4; r++) {

  258.     int32_t tmp_value = 0;

  259.     for (int c = 0; c < col16; c++) {

  260.       int r4div = r / C4NUM, r4mod = r % C4NUM, c16div = c / C16NUM, c16mod = c % C16NUM;

  261.       int src_index = r4div * C4NUM * col16 + c16div * C16NUM * C4NUM + r4mod * C16NUM + c16mod;

  262.       tmp_value += src[src_index];

  263.     }

  264.     dst[r] = tmp_value * filter_zp;

  265.   }

  266. #endif

  267.   return;

  268. }

  269. 

  270. void PackInputSum16x4Int8(const int8_t *input_value, int32_t *input_sum, size_t input_channel, size_t output_channel,

  271.                           size_t plane_size, ConvParameter *conv_param) {

  272.   size_t hw4 = UP_ROUND(plane_size, C4NUM);

  273.   size_t ic16 = UP_ROUND(input_channel, C16NUM);

  274.   if (conv_param->conv_quant_arg_.filter_arg_num_ == 1) {

  275.     PackInputSum16x4PerLayer(input_value, input_sum, conv_param->conv_quant_arg_.filter_quant_args_[0].zp_, hw4, ic16);

  276.   } else {

  277.     for (int ri = 0; ri < plane_size; ri++) {

  278.       int ri4div = ri / C4NUM, ri4mod = ri % C4NUM;

  279.       for (int ci = 0; ci < output_channel; ci++) {

  280.         int32_t tmp_sum_value = 0;

  281.         int ci4div = ci / C4NUM, ci4mod = ci % C4NUM;

  282.         int32_t filter_zp = conv_param->conv_quant_arg_.filter_quant_args_[ci].zp_;

  283.         for (int di = 0; di < input_channel; di++) {

  284.           size_t di16div = di / C16NUM, di16mod = di % C16NUM;

  285.           int src_index = ri4div * C4NUM * ic16 + di16div * C16NUM * C4NUM + ri4mod * C16NUM + di16mod;

  286.           tmp_sum_value += input_value[src_index];

  287.         }

  288.         int dst_index = ci4div * C4NUM * hw4 + ri * C4NUM + ci4mod;

  289.         input_sum[dst_index] = tmp_sum_value * filter_zp;

  290.       }

  291.     }

  292.   }

  293.   return;

  294. }

  295. 

  296. void PackInputSum8x4Int8(const int8_t *input_value, int32_t *input_sum, size_t input_channel, size_t output_channel,

  297.                          size_t plane_size, ConvParameter *conv_param) {

  298.   size_t hw8 = UP_ROUND(plane_size, C8NUM);

  299.   size_t ic4 = UP_ROUND(input_channel, C4NUM);

  300.   if (conv_param->conv_quant_arg_.filter_arg_num_ == 1) {

  301.     for (int r = 0; r < hw8; r++) {

  302.       int32_t tmp_value = 0;

  303.       for (int c = 0; c < ic4; c++) {

  304.         int r8div = r / C8NUM, r8mod = r % C8NUM, c4div = c / C4NUM, c4mod = c % C4NUM;

  305.         int src_index = r8div * C8NUM * ic4 + c4div * C8NUM * C4NUM + r8mod * C4NUM + c4mod;

  306.         tmp_value += input_value[src_index];

  307.       }

  308.       input_sum[r] = tmp_value * conv_param->conv_quant_arg_.filter_quant_args_[0].zp_;

  309.     }

  310.   } else {

  311.     for (int ri = 0; ri < plane_size; ri++) {

  312.       int ri8div = ri / C8NUM, ri8mod = ri % C8NUM;

  313.       for (int ci = 0; ci < output_channel; ci++) {

  314.         int32_t tmp_sum_value = 0;

  315.         int ci8div = ci / C8NUM, ci8mod = ci % C8NUM;

  316.         int32_t filter_zp = conv_param->conv_quant_arg_.filter_quant_args_[ci].zp_;

  317.         for (int di = 0; di < input_channel; di++) {

  318.           size_t di4div = di / C4NUM, di4mod = di % C4NUM;

  319.           int src_index = ri8div * C8NUM * ic4 + di4div * C8NUM * C4NUM + ri8mod * C4NUM + di4mod;

  320.           tmp_sum_value += input_value[src_index];

  321.         }

  322.         int dst_index = ci8div * C8NUM * hw8 + ri * C8NUM + ci8mod;

  323.         input_sum[dst_index] = tmp_sum_value * filter_zp;

  324.       }

  325.     }

  326.   }

  327.   return;

  328. }

  329. 

  330. void Im2ColPackUnitFp32(const float *input_data, ConvParameter *conv_param, float *packed_input, int real_cal_num,

  331.                         int block_index) {

  332.   // input format : nhwc

  333.   int kernel_h = conv_param->kernel_h_;

  334.   int kernel_w = conv_param->kernel_w_;

  335.   int stride_h = conv_param->stride_h_;

  336.   int stride_w = conv_param->stride_w_;

  337.   int pad_h = conv_param->pad_u_;

  338.   int pad_w = conv_param->pad_l_;

  339.   int dilation_h = conv_param->dilation_h_;

  340.   int dilation_w = conv_param->dilation_w_;

  341.   int in_channel = conv_param->input_channel_;

  342.   int in_h = conv_param->input_h_;

  343.   int in_w = conv_param->input_w_;

  344.   int out_w = conv_param->output_w_;

  345.   int ic4 = UP_DIV(in_channel, C4NUM);

  346.   memset(packed_input, 0, kernel_h * kernel_w * ic4 * C4NUM * TILE_NUM * sizeof(float));

  347. 

  348.   for (int i = 0; i < real_cal_num; i++) {

  349.     int block_start = block_index + i;

  350.     int input_h = block_start / out_w * stride_h - pad_h;

  351.     int input_w = block_start % out_w * stride_w - pad_w;

  352.     int input_stride = input_h * in_w * ic4 * C4NUM + input_w * ic4 * C4NUM;

  353.     int kh_s = MSMAX(0, UP_DIV(-input_h, dilation_h));

  354.     int kh_e = MSMIN(kernel_h, UP_DIV(in_h - input_h, dilation_h));

  355.     int kw_s = MSMAX(0, UP_DIV(-input_w, dilation_w));

  356.     int kw_e = MSMIN(kernel_w, UP_DIV(in_w - input_w, dilation_w));

  357.     for (int j = kh_s; j < kh_e; j++) {

  358.       int input_y_stride = j * dilation_h * in_w * ic4 * C4NUM + input_stride;

  359.       for (int n = kw_s; n < kw_e; n++) {

  360.         int input_x_stride = input_y_stride + n * dilation_w * ic4 * C4NUM;

  361.         int input_plane_offset = (j * kernel_w + n) * C8NUM * C4NUM * ic4 + i * C4NUM;

  362.         for (int m = 0; m < ic4; m++) {

  363.           int channel_block_stride = input_x_stride + m * C4NUM;

  364.           int channel_block_offset = input_plane_offset + m * C8NUM * C4NUM;

  365. #ifdef ENABLE_NEON

  366.           vst1q_f32(packed_input + channel_block_offset, vld1q_f32(input_data + channel_block_stride));

  367. #else

  368.           for (int k = 0; k < C4NUM; ++k) {

  369.             (packed_input + channel_block_offset)[k] = (input_data + channel_block_stride)[k];

  370.           }

  371. #endif

  372.         }  // channel_block loop

  373.       }    // kernel_w loop

  374.     }      // kernel_h loop

  375.   }        // tile num loop

  376. }

  377. 

  378. void Im2ColPackUnitInt8(const int8_t *input_data, int8_t *packed_input, int real_cal_num, int block_index,

  379.                         int32_t *input_sum, ConvParameter *conv_param) {

  380.   // input format : nhwc

  381.   int tile_num = conv_param->tile_num_;

  382.   QuantArg *filter_arg = conv_param->conv_quant_arg_.filter_quant_args_;

  383.   int kernel_h = conv_param->kernel_h_;

  384.   int kernel_w = conv_param->kernel_w_;

  385.   int stride_h = conv_param->stride_h_;

  386.   int stride_w = conv_param->stride_w_;

  387.   int pad_h = conv_param->pad_u_;

  388.   int pad_w = conv_param->pad_l_;

  389.   int dilation_h = conv_param->dilation_h_;

  390.   int dilation_w = conv_param->dilation_w_;

  391.   int in_channel = conv_param->input_channel_;

  392.   int in_h = conv_param->input_h_;

  393.   int in_w = conv_param->input_w_;

  394.   int ic4 = UP_DIV(in_channel, C4NUM);

  395.   int oc4 = UP_DIV(conv_param->output_channel_, C4NUM);

  396.   int out_w = conv_param->output_w_;

  397. 

  398.   for (int i = 0; i < real_cal_num; i++) {

  399.     int block_start = block_index + i;

  400.     int input_h = block_start / out_w * stride_h - pad_h;

  401.     int input_w = block_start % out_w * stride_w - pad_w;

  402.     int input_cal_num_offset = i * C4NUM * C4NUM;

  403.     int32_t input_accumulator = 0;

  404.     for (int j = 0; j < kernel_h; j++) {

  405.       int input_y = input_h + j * dilation_h;

  406.       if (input_y < 0 || input_y >= in_h) {

  407.         input_accumulator += ic4 * C4NUM * conv_param->conv_quant_arg_.input_quant_args_[0].zp_ * kernel_w;

  408.         continue;

  409.       }

  410.       int input_y_stride = input_y * in_w * ic4 * C4NUM;

  411.       for (int n = 0; n < kernel_w; n++) {

  412.         int input_x = input_w + n * dilation_w;

  413.         if (input_x < 0 || input_x >= in_w) {

  414.           input_accumulator += ic4 * C4NUM * conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  415.           continue;

  416.         }

  417.         int input_x_stride = input_y_stride + input_x * ic4 * C4NUM;

  418.         int plane_c4_block = (j * kernel_w + n) / C4NUM;

  419.         int plane_c4_res = (j * kernel_w + n) % C4NUM;

  420.         int input_plane_offset =

  421.           plane_c4_block * tile_num * C4NUM * C4NUM * ic4 + plane_c4_res * C4NUM + input_cal_num_offset;

  422.         for (int m = 0; m < ic4; m++) {

  423.           int channel_block_stride = input_x_stride + m * C4NUM;

  424.           int channel_block_offset = input_plane_offset + m * tile_num * C4NUM * C4NUM;

  425.           (packed_input + channel_block_offset)[0] = (input_data + channel_block_stride)[0];

  426.           (packed_input + channel_block_offset)[1] = (input_data + channel_block_stride)[1];

  427.           (packed_input + channel_block_offset)[2] = (input_data + channel_block_stride)[2];

  428.           (packed_input + channel_block_offset)[3] = (input_data + channel_block_stride)[3];

  429.           input_accumulator += (packed_input + channel_block_offset)[0];

  430.           input_accumulator += (packed_input + channel_block_offset)[1];

  431.           input_accumulator += (packed_input + channel_block_offset)[2];

  432.           input_accumulator += (packed_input + channel_block_offset)[3];

  433.         }  // channel_block loop

  434.       }    // kernel_w loop

  435.     }      // kernel_h loop

  436.     if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC)) {

  437.       continue;

  438.     } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  439.                (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  440.       int cal_num_offset = i * oc4 * C4NUM;

  441.       for (int l = 0; l < conv_param->output_channel_; ++l) {

  442.         input_sum[cal_num_offset + l] = input_accumulator * filter_arg[l].zp_;

  443.       }

  444.     } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  445.                !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  446.       input_sum[i] = input_accumulator * filter_arg[0].zp_;

  447.     }

  448.   }  // tile num loop

  449. }

  450. 

  451. void Im2ColPackUnitInt8Opt(const int8_t *input_data, int8_t *packed_input, int real_cal_num, int block_index,

  452.                            int32_t *input_sum, ConvParameter *conv_param) {

  453.   // input format : nhwc

  454.   int tile_num = conv_param->tile_num_;

  455.   QuantArg *filter_arg = conv_param->conv_quant_arg_.filter_quant_args_;

  456.   int kernel_h = conv_param->kernel_h_;

  457.   int kernel_w = conv_param->kernel_w_;

  458.   int stride_h = conv_param->stride_h_;

  459.   int stride_w = conv_param->stride_w_;

  460.   int pad_h = conv_param->pad_u_;

  461.   int pad_w = conv_param->pad_l_;

  462.   int dilation_h = conv_param->dilation_h_;

  463.   int dilation_w = conv_param->dilation_w_;

  464.   int in_channel = conv_param->input_channel_;

  465.   int in_h = conv_param->input_h_;

  466.   int in_w = conv_param->input_w_;

  467.   int ic4 = UP_DIV(in_channel, C4NUM);

  468.   int oc4 = UP_DIV(conv_param->output_channel_, C4NUM);

  469.   int out_w = conv_param->output_w_;

  470.   int block_size = kernel_h * kernel_w;

  471. 

  472.   for (int i = 0; i < real_cal_num; i++) {

  473.     int block_start = block_index + i;

  474.     int input_h = block_start / out_w * stride_h - pad_h;

  475.     int input_w = block_start % out_w * stride_w - pad_w;

  476.     for (int j = 0; j < kernel_h; j++) {

  477.       int input_y = input_h + j * dilation_h;

  478.       if (input_y < 0 || input_y >= in_h) {

  479.         continue;

  480.       }

  481.       int input_y_stride = input_y * in_w * ic4 * C4NUM;

  482.       for (int n = 0; n < kernel_w; n++) {

  483.         int input_x = input_w + n * dilation_w;

  484.         if (input_x < 0 || input_x >= in_w) {

  485.           continue;

  486.         }

  487.         int input_x_stride = input_y_stride + input_x * ic4 * C4NUM;

  488.         int input_plane_offset = (j * kernel_w + n) * tile_num * C4NUM * ic4 + i * C4NUM;

  489.         for (int m = 0; m < ic4; m++) {

  490.           int channel_block_stride = input_x_stride + m * C4NUM;

  491.           int channel_block_offset = input_plane_offset + m * tile_num * C4NUM;

  492.           memcpy(packed_input + channel_block_offset, input_data + channel_block_stride, 4);

  493.         }  // channel_block loop

  494.       }    // kernel_w loop

  495.     }      // kernel_h loop

  496.     int32_t input_accumulator = 0;

  497.     for (int j = 0; j < block_size; j++) {

  498.       int block_offset = j * tile_num * ic4 * C4NUM + i * C4NUM;

  499.       for (int c = 0; c < ic4; c++) {

  500.         int ic4_offset = block_offset + c * tile_num * C4NUM;

  501.         for (int k = 0; k < C4NUM; ++k) {

  502.           input_accumulator += (packed_input + ic4_offset)[k];

  503.         }

  504.       }

  505.     }

  506.     if (!(conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC)) {

  507.       continue;

  508.     } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  509.                (conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  510.       int cal_num_offset = i * oc4 * C4NUM;

  511.       for (int l = 0; l < conv_param->output_channel_; ++l) {

  512.         input_sum[cal_num_offset + l] = input_accumulator * filter_arg[l].zp_;

  513.       }

  514.     } else if ((conv_param->conv_quant_arg_.asymmetric_ & FILTER_ASYMMETRIC) &&

  515.                !(conv_param->conv_quant_arg_.per_channel_ & FILTER_PER_CHANNEL)) {

  516.       input_sum[i] = input_accumulator * filter_arg[0].zp_;

  517.     }

  518.   }  // tile num loop

  519. }

  520. 

  521. void PackInputToC8Int8(const int8_t *input_data, int16_t *packed_input, ConvParameter *conv_param) {

  522.   int in_batch = conv_param->input_batch_;

  523.   int in_channel = conv_param->input_channel_;

  524.   int in_h = conv_param->input_h_;

  525.   int in_w = conv_param->input_w_;

  526.   int ic8 = UP_DIV(in_channel, C8NUM);

  527. 

  528.   for (int b = 0; b < in_batch; b++) {

  529.     int src_batch_offset = b * in_channel * in_h * in_w;

  530.     int dst_batch_offset = b * ic8 * C8NUM * in_h * in_w;

  531.     for (int c = 0; c < in_channel; c++) {

  532.       int ic8_block = c / C8NUM;

  533.       int ic8_res = c % C8NUM;

  534.       int src_c_offset = src_batch_offset + c;

  535.       int dst_c_offset = dst_batch_offset + ic8_block * C8NUM * in_h * in_w + ic8_res;

  536.       for (int k = 0; k < in_w * in_h; k++) {

  537.         int src_plane_offset = src_c_offset + k * in_channel;

  538.         int dst_plane_offset = dst_c_offset + k * C8NUM;

  539.         (packed_input + dst_plane_offset)[0] = (int16_t)(input_data + src_plane_offset)[0];

  540.       }

  541.     }

  542.   }

  543. }

  544. 

  545. void PackWeightToC8Int8(const int8_t *origin_weight_data, int16_t *packed_weight_data, ConvParameter *conv_param) {

  546.   // origin weight format : ohwi

  547.   int input_channel = conv_param->input_channel_;

  548.   int ic8 = UP_DIV(input_channel, C8NUM);

  549.   int output_channel = conv_param->output_channel_;

  550.   QuantArg *filter_zp = conv_param->conv_quant_arg_.filter_quant_args_;

  551.   int kernel_plane = conv_param->kernel_h_ * conv_param->kernel_w_;

  552. 

  553.   for (int k = 0; k < kernel_plane; k++) {

  554.     int src_kernel_offset = k * input_channel;

  555.     int dst_kernel_offset = k * C8NUM;

  556.     for (int o = 0; o < output_channel; o++) {

  557.       int32_t zp;

  558.       if (conv_param->conv_quant_arg_.filter_arg_num_ == 1) {

  559.         zp = filter_zp[0].zp_;

  560.       } else {

  561.         zp = filter_zp[o].zp_;

  562.       }

  563.       int src_oc_offset = src_kernel_offset + o * kernel_plane * input_channel;

  564.       int dst_oc_offset = dst_kernel_offset + o * ic8 * kernel_plane * C8NUM;

  565.       for (int i = 0; i < input_channel; i++) {

  566.         int c8_block_num = i / C8NUM;

  567.         int c8_block_rem = i % C8NUM;

  568.         int src_ic_offset = src_oc_offset + i;

  569.         int dst_ic_offset = dst_oc_offset + c8_block_num * kernel_plane * C8NUM + c8_block_rem;

  570.         (packed_weight_data + dst_ic_offset)[0] = (int16_t)((origin_weight_data + src_ic_offset)[0] - zp);

  571.       }

  572.     }

  573.   }

  574. }

  575. 

  576. void PackNHWCToNC4HW4Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  577.   int c4 = UP_DIV(channel, C4NUM);

  578.   for (int b = 0; b < batch; b++) {

  579.     int src_oc_offset = b * plane * channel;

  580.     int dst_oc_offset = b * plane * c4 * C4NUM;

  581.     for (int k = 0; k < plane; k++) {

  582.       int src_kernel_offset = src_oc_offset + k * channel;

  583.       int dst_kernel_offset = dst_oc_offset + k * C4NUM;

  584.       for (int i = 0; i < channel; i++) {

  585.         int c4_block_num = i / C4NUM;

  586.         int c4_block_rem = i % C4NUM;

  587.         int src_ic_offset = src_kernel_offset + i;

  588.         int dst_ic_offset = dst_kernel_offset + c4_block_num * plane * C4NUM + c4_block_rem;

  589.         ((float *)dst + dst_ic_offset)[0] = ((float *)src + src_ic_offset)[0];

  590.       }

  591.     }

  592.   }

  593. }

  594. 

  595. void PackNCHWToNC4HW4Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  596.   int c4 = UP_DIV(channel, C4NUM);

  597.   for (int b = 0; b < batch; b++) {

  598.     int src_offset = b * plane * channel;

  599.     int dst_offset = b * plane * c4 * C4NUM;

  600.     for (int c = 0; c < channel; c++) {

  601.       int c4_block_num = c / C4NUM;

  602.       int c4_block_rem = c % C4NUM;

  603.       int src_c_offset = src_offset + c * plane;

  604.       int dst_c_offset = dst_offset + c4_block_num * plane * C4NUM;

  605.       for (int k = 0; k < plane; k++) {

  606.         int src_kernel_offset = src_c_offset + k;

  607.         int dst_kernel_offset = dst_c_offset + C4NUM * k + c4_block_rem;

  608.         ((float *)dst + dst_kernel_offset)[0] = ((float *)src + src_kernel_offset)[0];

  609.       }

  610.     }

  611.   }

  612. }

  613. 

  614. void PackNHWCToNHWC4Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  615.   int c4 = UP_DIV(channel, C4NUM);

  616.   int c4_channel = c4 * C4NUM;

  617.   int nhwc4_batch_unit_offset = c4 * C4NUM * plane;

  618.   int ic_remainder_ = channel % C4NUM;

  619.   if (ic_remainder_ != 0) {

  620.     int nhwc4_batch_offset = 0;

  621.     for (int b = 0; b < batch; b++) {

  622.       int batch_offset = b * channel * plane;

  623.       for (int i = 0; i < plane; i++) {

  624.         float *dst_per_plane = (float *)dst + nhwc4_batch_offset + i * c4_channel;

  625.         memcpy(dst_per_plane, (float *)src + batch_offset + i * channel, channel * sizeof(float));

  626.         for (int j = channel; j < c4_channel; ++j) {

  627.           dst_per_plane[j] = 0;

  628.         }

  629.       }

  630.       nhwc4_batch_offset += nhwc4_batch_unit_offset;

  631.     }

  632.   } else {

  633.     size_t ori_input_size = batch * plane * channel * sizeof(float);

  634.     memcpy((float *)dst, (float *)src, ori_input_size);

  635.   }

  636. }

  637. 

  638. void PackNHWC4ToNHWCFp32(const void *src, void *dst, int batch, int plane, int channel) {

  639.   int c4 = UP_DIV(channel, C4NUM);

  640.   int ic_remainder_ = channel % C4NUM;

  641.   if (ic_remainder_ != 0) {

  642.     int nhwc_batch_unit_offset = channel * plane;

  643.     for (int b = 0; b < batch; b++) {

  644.       int batch_offset = b * c4 * C4NUM * plane;

  645.       for (int i = 0; i < plane; i++) {

  646.         memcpy((float *)dst + b * nhwc_batch_unit_offset + i * channel, (float *)src + batch_offset + i * c4 * C4NUM,

  647.                channel * sizeof(float));

  648.       }

  649.     }

  650.   } else {

  651.     size_t ori_input_size = batch * plane * channel * sizeof(float);

  652.     memcpy((float *)dst, (float *)src, ori_input_size);

  653.   }

  654. }

  655. 

  656. void PackNCHWToNHWC4Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  657.   int nhwc4_batch_offset = 0;

  658.   int c4 = UP_DIV(channel, C4NUM);

  659.   int nhwc4_batch_unit_offset = c4 * C4NUM * plane;

  660. 

  661.   for (int b = 0; b < batch; b++) {

  662.     int batch_offset = b * channel * plane;

  663.     for (int c = 0; c < channel; c++) {

  664.       int src_c_offset = batch_offset + c * plane;

  665.       int dst_c_offset = nhwc4_batch_offset + c;

  666.       for (int i = 0; i < plane; i++) {

  667.         int src_plane_offset = src_c_offset + i;

  668.         int dst_plane_offset = dst_c_offset + i * c4 * C4NUM;

  669.         ((float *)dst)[dst_plane_offset] = ((float *)src)[src_plane_offset];

  670.       }

  671.     }

  672.     nhwc4_batch_offset += nhwc4_batch_unit_offset;

  673.   }

  674. }

  675. 

  676. void PackNC4HW4ToNHWC4Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  677.   int c4 = UP_DIV(channel, C4NUM);

  678.   for (int b = 0; b < batch; b++) {

  679.     int src_offset = b * plane * c4 * C4NUM;

  680.     int dst_offset = b * plane * channel;

  681.     for (int c = 0; c < channel; c++) {

  682.       int c4_block_num = c / C4NUM;

  683.       int c4_block_res = c % C4NUM;

  684.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  685.       int dst_c_offset = dst_offset + c4_block_num * C4NUM + c4_block_res;

  686.       for (int k = 0; k < plane; k++) {

  687.         int src_kernel_offset = src_c_offset + k * C4NUM;

  688.         int dst_kernel_offset = dst_c_offset + k * c4 * C4NUM;

  689.         ((float *)dst + dst_kernel_offset)[0] = ((float *)src + src_kernel_offset)[0];

  690.       }

  691.     }

  692.   }

  693. }

  694. 

  695. void PackNC4HW4ToNHWCFp32(const void *src, void *dst, int batch, int plane, int channel) {

  696.   int c4 = UP_DIV(channel, C4NUM);

  697.   for (int b = 0; b < batch; b++) {

  698.     int src_offset = b * plane * c4 * C4NUM;

  699.     int dst_offset = b * plane * channel;

  700.     for (int k = 0; k < plane; k++) {

  701.       int src_kernel_offset = src_offset + k * C4NUM;

  702.       int dst_kernel_offset = dst_offset + k * channel;

  703.       for (int c = 0; c < c4 - 1; c++) {

  704.         int src_c_offset = src_kernel_offset + c * plane * C4NUM;

  705.         int dst_c_offset = dst_kernel_offset + c * C4NUM;

  706. #ifdef ENABLE_NEON

  707.         vst1q_f32((float *)dst + dst_c_offset, vld1q_f32((float *)src + src_c_offset));

  708. #else

  709.         ((float *)dst + dst_c_offset)[0] = ((float *)src + src_c_offset)[0];

  710.         ((float *)dst + dst_c_offset)[1] = ((float *)src + src_c_offset)[1];

  711.         ((float *)dst + dst_c_offset)[2] = ((float *)src + src_c_offset)[2];

  712.         ((float *)dst + dst_c_offset)[3] = ((float *)src + src_c_offset)[3];

  713. #endif

  714.       }

  715.       // res part

  716.       int res_c = channel - (c4 - 1) * C4NUM;

  717.       for (int i = 0; i < res_c; i++) {

  718.         int src_res_c_offset = src_kernel_offset + (c4 - 1) * C4NUM * plane + i;

  719.         int dst_res_c_offset = dst_kernel_offset + (c4 - 1) * C4NUM + i;

  720.         ((float *)dst + dst_res_c_offset)[0] = ((float *)src + src_res_c_offset)[0];

  721.       }

  722.     }

  723.   }

  724. }

  725. 

  726. void PackNC4HW4ToNHWCReluFp32(const void *src, void *dst, int batch, int plane, int channel) {

  727.   int c4 = UP_DIV(channel, C4NUM);

  728.   for (int b = 0; b < batch; b++) {

  729.     int src_offset = b * plane * c4 * C4NUM;

  730.     int dst_offset = b * plane * channel;

  731.     for (int k = 0; k < plane; k++) {

  732.       int src_kernel_offset = src_offset + k * C4NUM;

  733.       int dst_kernel_offset = dst_offset + k * channel;

  734.       for (int c = 0; c < c4 - 1; c++) {

  735.         int src_c_offset = src_kernel_offset + c * plane * C4NUM;

  736.         int dst_c_offset = dst_kernel_offset + c * C4NUM;

  737. #ifdef ENABLE_NEON

  738.         float32x4_t input_ptr = vld1q_f32((float *)src + src_c_offset);

  739.         float32x4_t zero = vdupq_n_f32(0);

  740.         input_ptr = vmaxq_f32(zero, input_ptr);

  741.         vst1q_f32((float *)dst + dst_c_offset, input_ptr);

  742. #else

  743.         for (int i = 0; i < C4NUM; ++i) {

  744.           float input_data = ((float *)src + src_c_offset)[i];

  745.           input_data = input_data < 0 ? 0 : input_data;

  746.           ((float *)dst + dst_c_offset)[i] = input_data;

  747.         }

  748. #endif

  749.       }

  750.       // res part

  751.       int res_c = channel - (c4 - 1) * C4NUM;

  752.       for (int i = 0; i < res_c; i++) {

  753.         int src_res_c_offset = src_kernel_offset + (c4 - 1) * C4NUM * plane + i;

  754.         int dst_res_c_offset = dst_kernel_offset + (c4 - 1) * C4NUM + i;

  755.         float input_data = ((float *)src + src_res_c_offset)[0];

  756.         input_data = input_data < 0 ? 0 : input_data;

  757.         ((float *)dst + dst_res_c_offset)[0] = input_data;

  758.       }

  759.     }

  760.   }

  761. }

  762. 

  763. void PackNC4HW4ToNHWCRelu6Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  764.   int c4 = UP_DIV(channel, C4NUM);

  765.   for (int b = 0; b < batch; b++) {

  766.     int src_offset = b * plane * c4 * C4NUM;

  767.     int dst_offset = b * plane * channel;

  768.     for (int k = 0; k < plane; k++) {

  769.       int src_kernel_offset = src_offset + k * C4NUM;

  770.       int dst_kernel_offset = dst_offset + k * channel;

  771.       for (int c = 0; c < c4 - 1; c++) {

  772.         int src_c_offset = src_kernel_offset + c * plane * C4NUM;

  773.         int dst_c_offset = dst_kernel_offset + c * C4NUM;

  774. #ifdef ENABLE_NEON

  775.         float32x4_t input_ptr = vld1q_f32((float *)src + src_c_offset);

  776.         float32x4_t zero = vdupq_n_f32(0);

  777.         float32x4_t six = vdupq_n_f32(6);

  778.         input_ptr = vmaxq_f32(zero, input_ptr);

  779.         input_ptr = vminq_f32(six, input_ptr);

  780.         vst1q_f32((float *)dst + dst_c_offset, input_ptr);

  781. #else

  782.         for (int i = 0; i < C4NUM; ++i) {

  783.           float input_data = ((float *)src + src_c_offset)[i];

  784.           input_data = input_data < 0 ? 0 : input_data;

  785.           input_data = input_data > 6 ? 6 : input_data;

  786.           ((float *)dst + dst_c_offset)[i] = input_data;

  787.         }

  788. #endif

  789.       }

  790.       // res part

  791.       int res_c = channel - (c4 - 1) * C4NUM;

  792.       for (int i = 0; i < res_c; i++) {

  793.         int src_res_c_offset = src_kernel_offset + (c4 - 1) * C4NUM * plane + i;

  794.         int dst_res_c_offset = dst_kernel_offset + (c4 - 1) * C4NUM + i;

  795.         float input_data = ((float *)src + src_res_c_offset)[0];

  796.         input_data = input_data < 0 ? 0 : input_data;

  797.         input_data = input_data > 6 ? 6 : input_data;

  798.         ((float *)dst + dst_res_c_offset)[0] = input_data;

  799.       }

  800.     }

  801.   }

  802. }

  803. 

  804. void PackNC4HW4ToNHWCPreluFp32(const void *src, void *dst, const void *slope, int batch, int plane, int channel) {}

  805. 

  806. void PackNC4HW4ToNCHWFp32(const void *src, void *dst, int batch, int plane, int channel) {

  807.   int c4 = UP_DIV(channel, C4NUM);

  808.   for (int b = 0; b < batch; b++) {

  809.     int src_offset = b * plane * c4 * C4NUM;

  810.     int dst_offset = b * plane * channel;

  811.     for (int c = 0; c < channel; c++) {

  812.       int c4_block_num = c / C4NUM;

  813.       int c4_block_res = c % C4NUM;

  814.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  815.       int dst_c_offset = dst_offset + c * plane;

  816.       for (int k = 0; k < plane; k++) {

  817.         int src_kernel_offset = src_c_offset + k * C4NUM;

  818.         int dst_kernel_offset = dst_c_offset + k;

  819.         ((float *)dst + dst_kernel_offset)[0] = ((float *)src + src_kernel_offset)[0];

  820.       }

  821.     }

  822.   }

  823. }

  824. 

  825. void PackNHWCToC8HWN8Fp32(const void *src, void *dst, int batch, int plane, int channel) {

  826.   for (int n = 0; n < batch; n++) {

  827.     for (int hw = 0; hw < plane; hw++) {

  828.       for (int c = 0; c < channel; c++) {

  829.         int c8div = c / C8NUM;

  830.         int c8mod = c % C8NUM;

  831.         int src_index = n * plane * channel + hw * channel + c;

  832.         int dst_index = c8div * batch * plane * C8NUM + hw * batch * C8NUM + n * C8NUM + c8mod;

  833.         ((float *)dst)[dst_index] = ((float *)src)[src_index];

  834.       }

  835.     }

  836.   }

  837.   return;

  838. }

  839. 

  840. void PackNHWCToNHWC4Int8(const void *src, void *dst, int batch, int plane, int channel) {

  841.   int c4 = UP_DIV(channel, C4NUM);

  842.   int nhwc4_batch_unit_offset = c4 * C4NUM * plane;

  843.   int ic_remainder_ = channel % C4NUM;

  844.   if (ic_remainder_ != 0) {

  845.     int nhwc4_batch_offset = 0;

  846.     for (int b = 0; b < batch; b++) {

  847.       int batch_offset = b * channel * plane;

  848.       for (int i = 0; i < plane; i++) {

  849.         memcpy((int8_t *)dst + nhwc4_batch_offset + i * c4 * C4NUM, (int8_t *)src + batch_offset + i * channel,

  850.                channel);

  851.       }

  852.       nhwc4_batch_offset += nhwc4_batch_unit_offset;

  853.     }

  854.   } else {

  855.     size_t ori_input_size = batch * plane * channel;

  856.     memcpy((int8_t *)dst, (int8_t *)src, ori_input_size);

  857.   }

  858. }

  859. 

  860. void PackNHWC4ToNHWCInt8(const void *src, void *dst, int batch, int plane, int channel) {

  861.   int c4 = UP_DIV(channel, C4NUM);

  862.   int nhwc4_batch_unit_offset = c4 * C4NUM * plane;

  863.   int ic_remainder_ = channel % C4NUM;

  864.   if (ic_remainder_ != 0) {

  865.     for (int b = 0; b < batch; b++) {

  866.       int batch_offset = b * channel * plane;

  867.       int nhwc4_batch_offset = b * nhwc4_batch_unit_offset;

  868.       for (int i = 0; i < plane; i++) {

  869.         memcpy((int8_t *)dst + batch_offset + i * channel, (int8_t *)src + nhwc4_batch_offset + i * c4 * C4NUM,

  870.                channel);

  871.       }

  872.     }

  873.   } else {

  874.     size_t ori_input_size = batch * plane * channel;

  875.     memcpy((int8_t *)dst, (int8_t *)src, ori_input_size);

  876.   }

  877. }

  878. 

  879. void PackNCHWToNHWC4Int8(const void *src, void *dst, int batch, int plane, int channel) {

  880.   int nhwc4_batch_offset = 0;

  881.   int c4 = UP_DIV(channel, C4NUM);

  882.   int nhwc4_batch_unit_offset = c4 * C4NUM * plane;

  883. 

  884.   for (int b = 0; b < batch; b++) {

  885.     int batch_offset = b * channel * plane;

  886.     for (int c = 0; c < channel; c++) {

  887.       int src_c_offset = batch_offset + c * plane;

  888.       int dst_c_offset = nhwc4_batch_offset + c;

  889.       for (int i = 0; i < plane; i++) {

  890.         int src_plane_offset = src_c_offset + i;

  891.         int dst_plane_offset = dst_c_offset + i * c4 * C4NUM;

  892.         ((uint8_t *)dst)[dst_plane_offset] = ((uint8_t *)src)[src_plane_offset];

  893.       }

  894.     }

  895.     nhwc4_batch_offset += nhwc4_batch_unit_offset;

  896.   }

  897. }

  898. 

  899. void PackNC4HW4ToNHWC4Int8(const void *src, void *dst, int batch, int plane, int channel) {

  900.   int c4 = UP_DIV(channel, C4NUM);

  901.   for (int b = 0; b < batch; b++) {

  902.     int src_offset = b * plane * c4 * C4NUM;

  903.     int dst_offset = b * plane * channel;

  904.     for (int c = 0; c < channel; c++) {

  905.       int c4_block_num = c / C4NUM;

  906.       int c4_block_res = c % C4NUM;

  907.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  908.       int dst_c_offset = dst_offset + c4_block_num * C4NUM + c4_block_res;

  909.       for (int k = 0; k < plane; k++) {

  910.         int src_kernel_offset = src_c_offset + k * C4NUM;

  911.         int dst_kernel_offset = dst_c_offset + k * c4 * C4NUM;

  912.         ((uint8_t *)dst + dst_kernel_offset)[0] = ((uint8_t *)src + src_kernel_offset)[0];

  913.       }

  914.     }

  915.   }

  916. }

  917. 

  918. void PackNC4HW4ToNHWCInt8(const void *src, void *dst, int batch, int plane, int channel) {

  919.   int c4 = UP_DIV(channel, C4NUM);

  920.   for (int b = 0; b < batch; b++) {

  921.     int src_offset = b * plane * c4 * C4NUM;

  922.     int dst_offset = b * plane * channel;

  923.     for (int k = 0; k < plane; k++) {

  924.       int src_kernel_offset = src_offset + k * C4NUM;

  925.       int dst_kernel_offset = dst_offset + k * channel;

  926.       for (int c = 0; c < c4 - 1; c++) {

  927.         int src_c_offset = src_kernel_offset + c * plane * C4NUM;

  928.         int dst_c_offset = dst_kernel_offset + c * C4NUM;

  929.         ((int8_t *)dst + dst_c_offset)[0] = ((int8_t *)src + src_c_offset)[0];

  930.         ((int8_t *)dst + dst_c_offset)[1] = ((int8_t *)src + src_c_offset)[1];

  931.         ((int8_t *)dst + dst_c_offset)[2] = ((int8_t *)src + src_c_offset)[2];

  932.         ((int8_t *)dst + dst_c_offset)[3] = ((int8_t *)src + src_c_offset)[3];

  933.       }

  934.       // res part

  935.       int res_c = channel - (c4 - 1) * C4NUM;

  936.       for (int i = 0; i < res_c; i++) {

  937.         int src_res_c_offset = src_kernel_offset + (c4 - 1) * C4NUM * plane + i;

  938.         int dst_res_c_offset = dst_kernel_offset + (c4 - 1) * C4NUM + i;

  939.         ((int8_t *)dst + dst_res_c_offset)[0] = ((int8_t *)src + src_res_c_offset)[0];

  940.       }

  941.     }

  942.   }

  943. }

  944. 

  945. void PackNC4HW4ToNCHWInt8(const void *src, void *dst, int batch, int plane, int channel) {

  946.   int c4 = UP_DIV(channel, C4NUM);

  947.   for (int b = 0; b < batch; b++) {

  948.     int src_offset = b * plane * c4 * C4NUM;

  949.     int dst_offset = b * plane * channel;

  950.     for (int c = 0; c < channel; c++) {

  951.       int c4_block_num = c / C4NUM;

  952.       int c4_block_res = c % C4NUM;

  953.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  954.       int dst_c_offset = dst_offset + c * plane;

  955.       for (int k = 0; k < plane; k++) {

  956.         int src_kernel_offset = src_c_offset + k * C4NUM;

  957.         int dst_kernel_offset = dst_c_offset + k;

  958.         ((uint8_t *)dst + dst_kernel_offset)[0] = ((uint8_t *)src + src_kernel_offset)[0];

  959.       }

  960.     }

  961.   }

  962. }

  963. 

  964. void PackNHWCToC8HWN8Int8(const void *src, void *dst, int batch, int plane, int channel) {

  965.   for (int n = 0; n < batch; n++) {

  966.     for (int hw = 0; hw < plane; hw++) {

  967.       for (int c = 0; c < channel; c++) {

  968.         int c8div = c / C8NUM;

  969.         int c8mod = c % C8NUM;

  970.         int src_index = n * plane * channel + hw * channel + c;

  971.         int dst_index = c8div * batch * plane * C8NUM + hw * batch * C8NUM + n * C8NUM + c8mod;

  972.         ((int8_t *)dst)[dst_index] = ((int8_t *)src)[src_index];

  973.       }

  974.     }

  975.   }

  976.   return;

  977. }

  978. 

  979. void PackNHWCToNC8HW8Int8(const void *src, void *dst, int batch, int plane, int channel) {

  980.   int c8 = UP_DIV(channel, C8NUM);

  981.   for (int b = 0; b < batch; b++) {

  982.     int src_oc_offset = b * plane * channel;

  983.     int dst_oc_offset = b * plane * c8 * C8NUM;

  984.     for (int k = 0; k < plane; k++) {

  985.       int src_kernel_offset = src_oc_offset + k * channel;

  986.       int dst_kernel_offset = dst_oc_offset + k * C8NUM;

  987.       for (int i = 0; i < channel; i++) {

  988.         int c8_block_num = i / C8NUM;

  989.         int c8_block_rem = i % C8NUM;

  990.         int src_ic_offset = src_kernel_offset + i;

  991.         int dst_ic_offset = dst_kernel_offset + c8_block_num * plane * C8NUM + c8_block_rem;

  992.         ((int8_t *)dst + dst_ic_offset)[0] = ((int8_t *)src + src_ic_offset)[0];

  993.       }

  994.     }

  995.   }

  996. }

  997. 

  998. void PackNCHWToNHWCInt8(const void *src, void *dst, int batch, int plane, int channel) {

  999.   for (int n = 0; n < batch; n++) {

  1000.     for (int c = 0; c < channel; c++) {

  1001.       for (int hw = 0; hw < plane; hw++) {

  1002.         int nhwc_index = n * channel * plane + hw * channel + c;

  1003.         int nchw_index = n * channel * plane + c * plane + hw;

  1004.         ((int8_t *)(dst))[nhwc_index] = ((const int8_t *)(src))[nchw_index];

  1005.       }

  1006.     }

  1007.   }

  1008.   return;

  1009. }

  1010. 

  1011. void PackNHWCToNCHWFp32(const void *src, void *dst, int batches, int plane, int channel) {

  1012.   int hw8 = plane / C8NUM * C8NUM;

  1013.   int c8 = channel / C8NUM * C8NUM;

  1014.   int batch = plane * channel;

  1015.   for (int n = 0; n < batches; n++) {

  1016.     const float *src_batch = (const float *)src + n * batch;

  1017.     float *dst_batch = (float *)dst + n * batch;

  1018.     int hw = 0;

  1019.     for (; hw < hw8; hw += C8NUM) {

  1020.       int c = 0;

  1021.       for (; c < c8; c += C8NUM) {

  1022.         const float *src_ptr = src_batch + hw * channel + c;

  1023.         float *dst_ptr = dst_batch + c * plane + hw;

  1024. #ifdef ENABLE_ARM64

  1025.         size_t srcStride = channel * sizeof(float);

  1026.         size_t dstStride = plane * sizeof(float);

  1027.         asm volatile(

  1028.           "mov x10, %[src_ptr]\n"

  1029.           "mov x11, %[dst_ptr]\n"

  1030. 

  1031.           "ld1 {v0.4s, v1.4s}, [x10], %[srcStride]\n"

  1032.           "ld1 {v2.4s, v3.4s}, [x10], %[srcStride]\n"

  1033. 

  1034.           "zip1 v8.4s, v0.4s, v2.4s\n"

  1035.           "zip2 v9.4s, v0.4s, v2.4s\n"

  1036.           "zip1 v12.4s, v1.4s, v3.4s\n"

  1037.           "zip2 v13.4s, v1.4s, v3.4s\n"

  1038. 

  1039.           "ld1 {v4.4s, v5.4s}, [x10], %[srcStride]\n"

  1040.           "ld1 {v6.4s, v7.4s}, [x10], %[srcStride]\n"

  1041. 

  1042.           "zip1 v10.4s, v4.4s, v6.4s\n"

  1043.           "zip2 v11.4s, v4.4s, v6.4s\n"

  1044.           "zip1 v14.4s, v5.4s, v7.4s\n"

  1045.           "zip2 v15.4s, v5.4s, v7.4s\n"

  1046. 

  1047.           "ld1 {v0.4s, v1.4s}, [x10], %[srcStride]\n"

  1048.           "ld1 {v2.4s, v3.4s}, [x10], %[srcStride]\n"

  1049. 

  1050.           "trn1 v16.2d, v8.2d, v10.2d\n"

  1051.           "trn2 v18.2d, v8.2d, v10.2d\n"

  1052.           "trn1 v20.2d, v9.2d, v11.2d\n"

  1053.           "trn2 v22.2d, v9.2d, v11.2d\n"

  1054. 

  1055.           "ld1 {v4.4s, v5.4s}, [x10], %[srcStride]\n"

  1056.           "ld1 {v6.4s, v7.4s}, [x10], %[srcStride]\n"

  1057. 

  1058.           "trn1 v24.2d, v12.2d, v14.2d\n"

  1059.           "trn2 v26.2d, v12.2d, v14.2d\n"

  1060.           "trn1 v28.2d, v13.2d, v15.2d\n"

  1061.           "trn2 v30.2d, v13.2d, v15.2d\n"

  1062. 

  1063.           "zip1 v8.4s, v0.4s, v2.4s\n"

  1064.           "zip2 v9.4s, v0.4s, v2.4s\n"

  1065.           "zip1 v12.4s, v1.4s, v3.4s\n"

  1066.           "zip2 v13.4s, v1.4s, v3.4s\n"

  1067. 

  1068.           "zip1 v10.4s, v4.4s, v6.4s\n"

  1069.           "zip2 v11.4s, v4.4s, v6.4s\n"

  1070.           "zip1 v14.4s, v5.4s, v7.4s\n"

  1071.           "zip2 v15.4s, v5.4s, v7.4s\n"

  1072. 

  1073.           "trn1 v17.2d, v8.2d, v10.2d\n"

  1074.           "trn2 v19.2d, v8.2d, v10.2d\n"

  1075.           "trn1 v21.2d, v9.2d, v11.2d\n"

  1076.           "trn2 v23.2d, v9.2d, v11.2d\n"

  1077. 

  1078.           "trn1 v25.2d, v12.2d, v14.2d\n"

  1079.           "trn2 v27.2d, v12.2d, v14.2d\n"

  1080.           "trn1 v29.2d, v13.2d, v15.2d\n"

  1081.           "trn2 v31.2d, v13.2d, v15.2d\n"

  1082. 

  1083.           "st1 {v16.4s, v17.4s}, [x11], %[dstStride]\n"

  1084.           "st1 {v18.4s, v19.4s}, [x11], %[dstStride]\n"

  1085.           "st1 {v20.4s, v21.4s}, [x11], %[dstStride]\n"

  1086.           "st1 {v22.4s, v23.4s}, [x11], %[dstStride]\n"

  1087.           "st1 {v24.4s, v25.4s}, [x11], %[dstStride]\n"

  1088.           "st1 {v26.4s, v27.4s}, [x11], %[dstStride]\n"

  1089.           "st1 {v28.4s, v29.4s}, [x11], %[dstStride]\n"

  1090.           "st1 {v30.4s, v31.4s}, [x11], %[dstStride]\n"

  1091. 

  1092.           :

  1093.           :

  1094.           [ dst_ptr ] "r"(dst_ptr), [ src_ptr ] "r"(src_ptr), [ srcStride ] "r"(srcStride), [ dstStride ] "r"(dstStride)

  1095.           : "x10", "x11", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",

  1096.             "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",

  1097.             "v30", "v31");

  1098. #else

  1099.         for (int tr = 0; tr < C8NUM; tr++) {

  1100.           for (int tc = 0; tc < C8NUM; tc++) {

  1101.             dst_ptr[tc * plane + tr] = src_ptr[tr * channel + tc];

  1102.           }

  1103.         }

  1104. #endif

  1105.       }

  1106.       for (; c < channel; c++) {

  1107.         const float *src_ptr = src_batch + hw * channel + c;

  1108.         float *dst_ptr = dst_batch + c * plane + hw;

  1109.         for (size_t i = 0; i < C8NUM; i++) {

  1110.           dst_ptr[i] = src_ptr[i * channel];

  1111.         }

  1112.       }

  1113.     }

  1114.     for (; hw < plane; hw++) {

  1115.       const float *src_ptr = src_batch + hw * channel;

  1116.       float *dst_ptr = dst_batch + hw;

  1117.       for (size_t i = 0; i < channel; i++) {

  1118.         dst_ptr[i * plane] = src_ptr[i];

  1119.       }

  1120.     }

  1121.   }

  1122.   return;

  1123. }

  1124. 

  1125. void PackNHWCToNCHWInt8(const void *src, void *dst, int batch, int plane, int channel) {

  1126.   for (int n = 0; n < batch; n++) {

  1127.     for (int c = 0; c < channel; c++) {

  1128.       for (int hw = 0; hw < plane; hw++) {

  1129.         int nhwc_index = n * channel * plane + hw * channel + c;

  1130.         int nchw_index = n * channel * plane + c * plane + hw;

  1131.         ((int8_t *)dst)[nchw_index] = ((int8_t *)src)[nhwc_index];

  1132.       }

  1133.     }

  1134.   }

  1135.   return;

  1136. }

  1137. 

  1138. void PackNCHWToNHWCFp32(const void *src, void *dst, int batch, int plane, int channel) {

  1139.   return PackNHWCToNCHWFp32(src, dst, batch, channel, plane);

  1140. }

  1141. 

  1142. void MatrixPackUnit(const float *src, float *dst, size_t row, size_t col, size_t src_stride, size_t dst_stride) {

  1143.   size_t copy_size = row * C4NUM * sizeof(float);

  1144.   for (int c = 0; c < col; c++) {

  1145.     memcpy(dst + c * dst_stride, src + c * src_stride, copy_size);

  1146.   }

  1147. }

  1148. 

  1149. void MatrixPack(const float *src, float *dst, int row, int ic4, int stride) {

  1150.   int row4mod = row % 4;

  1151.   int row4div = row / 4;

  1152. 

  1153.   for (int i = 0; i < row4div; i++) {

  1154.     MatrixPackUnit(src + i * 4 * 4, dst + i * 4 * ic4 * 4, 4, ic4, stride, 16);

  1155.   }

  1156. 

  1157.   if (row4mod > 0) {

  1158.     MatrixPackUnit(src + row4div * 4 * 4, dst + row4div * 4 * ic4 * 4, row4mod, ic4, stride, row4mod * 4);

  1159.   }

  1160.   return;

  1161. }

  1162. 

  1163. void PackDepthwiseInt8Input(const int8_t *src, int16_t *dst, const ConvParameter *conv_param) {

  1164.   int input_zp = conv_param->conv_quant_arg_.input_quant_args_[0].zp_;

  1165.   int ic4 = UP_DIV(conv_param->input_channel_, C4NUM);

  1166.   int unit = conv_param->input_h_ * conv_param->input_w_;

  1167. 

  1168.   for (int b = 0; b < conv_param->input_batch_; b++) {

  1169.     const int8_t *src_b = src + b * unit * conv_param->input_channel_;

  1170.     int16_t *dst_b = dst + b * unit * ic4 * C4NUM;

  1171.     for (int k = 0; k < unit; k++) {

  1172.       const int8_t *src_k = src_b + k * conv_param->input_channel_;

  1173.       int16_t *dst_k = dst_b + k * ic4 * C4NUM;

  1174.       for (int c = 0; c < conv_param->input_channel_; c++) {

  1175.         dst_k[c] = (int16_t)(src_k[c] - input_zp);

  1176.       }

  1177.     }

  1178.   }

  1179. }

  1180. 

  1181. void PackDepthwiseInt8Weight(const int8_t *origin_weight, int16_t *packed_weight_, int plane, int channel,

  1182.                              ConvQuantArg *quant_qrg) {

  1183.   int weight_zp = quant_qrg->filter_quant_args_[0].zp_;

  1184.   for (int c = 0; c < channel; c++) {

  1185.     if (quant_qrg->per_channel_ & FILTER_PER_CHANNEL) {

  1186.       weight_zp = quant_qrg->filter_quant_args_[c].zp_;

  1187.     }

  1188.     int c4_block_num = c / C4NUM;

  1189.     int c4_block_rem = c % C4NUM;

  1190.     const int8_t *src_c = origin_weight + c * plane;

  1191.     int16_t *dst_c = packed_weight_ + c4_block_num * plane * C4NUM;

  1192.     for (int k = 0; k < plane; k++) {

  1193.       const int8_t *src_kernel = src_c + k;

  1194.       int16_t *dst_kernel = dst_c + C4NUM * k + c4_block_rem;

  1195.       *dst_kernel = (int16_t)(src_kernel[0] - weight_zp);

  1196.     }

  1197.   }

  1198. }