Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d50b324456
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd50b324456'
${ noResults }
mindspore/mindspore/lite/nnacl/fp16/pack_fp16.c

578 lines
23 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/fp16/pack_fp16.h"

  18. #include <string.h>

  19. #include <stdlib.h>

  20. 

  21. void Im2ColPackUnitFp16(float16_t *input_data, ConvParameter *conv_param, float16_t *packed_input, int real_cal_num,

  22.                         int block_index) {

  23.   // input format : nhwc

  24.   int kernel_h = conv_param->kernel_h_;

  25.   int kernel_w = conv_param->kernel_w_;

  26.   int stride_h = conv_param->stride_h_;

  27.   int stride_w = conv_param->stride_w_;

  28.   int pad_h = conv_param->pad_u_;

  29.   int pad_w = conv_param->pad_l_;

  30.   int dilation_h = conv_param->dilation_h_;

  31.   int dilation_w = conv_param->dilation_w_;

  32.   int in_channel = conv_param->input_channel_;

  33.   int in_h = conv_param->input_h_;

  34.   int in_w = conv_param->input_w_;

  35.   int out_w = conv_param->output_w_;

  36.   int ic4 = UP_DIV(in_channel, 4);

  37.   int ic4_minus = in_channel / 4;

  38.   memset(packed_input, 0, kernel_w * kernel_h * ic4 * C4NUM * 16 * sizeof(float16_t));

  39. 

  40.   for (int i = 0; i < real_cal_num; i++) {

  41.     int block_start = block_index + i;

  42.     int input_h = block_start / out_w * stride_h - pad_h;

  43.     int input_w = block_start % out_w * stride_w - pad_w;

  44.     int input_stride = (input_h * in_w + input_w) * in_channel;

  45.     int kh_s = MSMAX(0, UP_DIV(-input_h, dilation_h));

  46.     int kh_e = MSMIN(kernel_h, UP_DIV(in_h - input_h, dilation_h));

  47.     int kw_s = MSMAX(0, UP_DIV(-input_w, dilation_w));

  48.     int kw_e = MSMIN(kernel_w, UP_DIV(in_w - input_w, dilation_w));

  49.     for (int j = kh_s; j < kh_e; j++) {

  50.       int input_y_stride = j * dilation_h * in_w * in_channel + input_stride;

  51.       for (int n = kw_s; n < kw_e; n++) {

  52.         int input_x_stride = input_y_stride + n * dilation_w * in_channel;

  53.         int input_plane_offset = (j * kernel_w + n) * 16 * C4NUM * ic4 + i * C4NUM;

  54.         for (int m = 0; m < ic4_minus; m++) {

  55.           int channel_block_stride = input_x_stride + m * C4NUM;

  56.           int channel_block_offset = input_plane_offset + m * 16 * C4NUM;

  57. #ifdef ENABLE_ARM64

  58.           vst1_f16(packed_input + channel_block_offset, vld1_f16(input_data + channel_block_stride));

  59. #else

  60.           for (int l = 0; l < C4NUM; ++l) {

  61.             (packed_input + channel_block_offset)[l] = (input_data + channel_block_stride)[l];

  62.           }

  63. #endif

  64.         }  // channel_block loop

  65.         int ic_res = in_channel - ic4_minus * C4NUM;

  66.         for (int l = 0; l < ic_res; ++l) {

  67.           int channel_block_stride = input_x_stride + ic4_minus * C4NUM + l;

  68.           int channel_block_offset = input_plane_offset + ic4_minus * 16 * C4NUM + l;

  69.           packed_input[channel_block_offset] = input_data[channel_block_stride];

  70.         }

  71.       }  // kernel_w loop

  72.     }    // kernel_h loop

  73.   }      // tile num loop

  74. }

  75. 

  76. void PackWeightFp16(float16_t *weight_data, ConvParameter *conv_param, float16_t *packed_weight) {

  77.   // original weight format : ohwi

  78.   const int tile_num = 8;

  79.   const int inchannel_block = 4;

  80.   int kernel_h = conv_param->kernel_h_;

  81.   int kernel_w = conv_param->kernel_w_;

  82.   int in_channel = conv_param->input_channel_;

  83.   int out_channel = conv_param->output_channel_;

  84.   int kernel_block = UP_DIV(out_channel, tile_num);

  85.   int channel_block = UP_DIV(in_channel, inchannel_block);

  86.   int kernel_plane = kernel_h * kernel_w;

  87.   int pack_weight_size = kernel_block * channel_block * tile_num * inchannel_block * kernel_plane;

  88. 

  89.   int unit_size = tile_num * inchannel_block;

  90.   int block_size = pack_weight_size / kernel_block;

  91. 

  92.   for (int m = 0; m < kernel_plane; m++) {

  93.     int kernel_plane_stride = m * in_channel;

  94.     int packed_kernel_plane_stride = m * unit_size * channel_block;

  95.     for (int i = 0; i < channel_block; i++) {

  96.       int channel_block_stride = kernel_plane_stride + i * inchannel_block;

  97.       int packed_channel_block_size = packed_kernel_plane_stride + i * unit_size;

  98.       int ic_remainder = in_channel - i * inchannel_block;

  99.       int real_ic_num = ic_remainder < inchannel_block ? ic_remainder : inchannel_block;

  100.       for (int h = 0; h < real_ic_num; h++) {

  101.         int block_stride = channel_block_stride + h;

  102.         int packed_block_stride = packed_channel_block_size + h * tile_num;

  103.         for (int j = 0; j < kernel_block; j++) {

  104.           int kernel_block_stride = block_stride + j * tile_num * kernel_plane * in_channel;

  105.           int packed_kernel_block_size = packed_block_stride + j * block_size;

  106.           int oc_remainder = out_channel - j * tile_num;

  107.           int real_oc_num = oc_remainder < tile_num ? oc_remainder : tile_num;

  108.           for (int k = 0; k < real_oc_num; k++) {

  109.             float16_t *origin_data_ptr = weight_data + kernel_block_stride + k * kernel_plane * in_channel;

  110.             float16_t *packed_data_ptr = packed_weight + packed_kernel_block_size + k;

  111.             *packed_data_ptr = *origin_data_ptr;

  112.           }

  113.         }  // kernel block loop

  114.       }    // inchannel block loop

  115.     }      // channel block loop

  116.   }        // kernel plane loop

  117. }

  118. 

  119. void PackWeightToC8Fp16(const float16_t *origin_weight_data, float16_t *packed_weight_data, ConvParameter *conv_param) {

  120.   // origin weight format : ohwi

  121.   int input_channel = conv_param->input_channel_;

  122.   int ic8 = UP_DIV(input_channel, C8NUM);

  123.   int output_channel = conv_param->output_channel_;

  124.   int kernel_plane = conv_param->kernel_h_ * conv_param->kernel_w_;

  125. 

  126.   for (int k = 0; k < kernel_plane; k++) {

  127.     int src_kernel_offset = k * input_channel;

  128.     int dst_kernel_offset = k * C8NUM;

  129.     for (int o = 0; o < output_channel; o++) {

  130.       int src_oc_offset = src_kernel_offset + o * kernel_plane * input_channel;

  131.       int dst_oc_offset = dst_kernel_offset + o * ic8 * kernel_plane * C8NUM;

  132.       for (int i = 0; i < input_channel; i++) {

  133.         int c8_block_num = i / C8NUM;

  134.         int c8_block_rem = i % C8NUM;

  135.         int src_ic_offset = src_oc_offset + i;

  136.         int dst_ic_offset = dst_oc_offset + c8_block_num * kernel_plane * C8NUM + c8_block_rem;

  137.         (packed_weight_data + dst_ic_offset)[0] = (origin_weight_data + src_ic_offset)[0];

  138.       }

  139.     }

  140.   }

  141. }

  142. 

  143. void PackWeightToC4Fp16(const float16_t *origin_weight_data, float16_t *packed_weight_data, ConvParameter *conv_param) {

  144.   // origin weight format : ohwi

  145.   int input_channel = conv_param->input_channel_;

  146.   int ic8 = UP_DIV(input_channel, C8NUM);

  147.   int ic4 = ic8 * 2;

  148.   int output_channel = conv_param->output_channel_;

  149.   int kernel_plane = conv_param->kernel_h_ * conv_param->kernel_w_;

  150. 

  151.   for (int k = 0; k < kernel_plane; k++) {

  152.     int src_kernel_offset = k * input_channel;

  153.     int dst_kernel_offset = k * C4NUM;

  154.     for (int o = 0; o < output_channel; o++) {

  155.       int src_oc_offset = src_kernel_offset + o * kernel_plane * input_channel;

  156.       int dst_oc_offset = dst_kernel_offset + o * ic4 * kernel_plane * C4NUM;

  157.       for (int i = 0; i < input_channel; i++) {

  158.         int c4_block_num = i / C4NUM;

  159.         int c4_block_rem = i % C4NUM;

  160.         int src_ic_offset = src_oc_offset + i;

  161.         int dst_ic_offset = dst_oc_offset + c4_block_num * kernel_plane * C4NUM + c4_block_rem;

  162.         (packed_weight_data + dst_ic_offset)[0] = (origin_weight_data + src_ic_offset)[0];

  163.       }

  164.     }

  165.   }

  166. }

  167. 

  168. void PackNHWCToNC4HW4Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  169.   int c4 = UP_DIV(channel, C4NUM);

  170.   for (int b = 0; b < batch; b++) {

  171.     int src_oc_offset = b * plane * channel;

  172.     int dst_oc_offset = b * plane * c4 * C4NUM;

  173.     for (int k = 0; k < plane; k++) {

  174.       int src_kernel_offset = src_oc_offset + k * channel;

  175.       int dst_kernel_offset = dst_oc_offset + k * C4NUM;

  176.       for (int i = 0; i < channel; i++) {

  177.         int c4_block_num = i / C4NUM;

  178.         int c4_block_rem = i % C4NUM;

  179.         int src_ic_offset = src_kernel_offset + i;

  180.         int dst_ic_offset = dst_kernel_offset + c4_block_num * plane * C4NUM + c4_block_rem;

  181.         ((float16_t *)dst + dst_ic_offset)[0] = ((float16_t *)src + src_ic_offset)[0];

  182.       }

  183.     }

  184.   }

  185. }

  186. 

  187. void PackNCHWToNC4HW4Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  188.   int c4 = UP_DIV(channel, C4NUM);

  189.   for (int b = 0; b < batch; b++) {

  190.     int src_offset = b * plane * channel;

  191.     int dst_offset = b * plane * c4 * C4NUM;

  192.     for (int c = 0; c < channel; c++) {

  193.       int c4_block_num = c / C4NUM;

  194.       int c4_block_rem = c % C4NUM;

  195.       int src_c_offset = src_offset + c * plane;

  196.       int dst_c_offset = dst_offset + c4_block_num * plane * C4NUM;

  197.       for (int k = 0; k < plane; k++) {

  198.         int src_kernel_offset = src_c_offset + k;

  199.         int dst_kernel_offset = dst_c_offset + C4NUM * k + c4_block_rem;

  200.         ((float16_t *)dst + dst_kernel_offset)[0] = ((float16_t *)src + src_kernel_offset)[0];

  201.       }

  202.     }

  203.   }

  204. }

  205. 

  206. void PackNHWCToNCHWFp16(const void *src, void *dst, int batches, int plane, int channel) {

  207.   int hw16 = plane / C16NUM * C16NUM;

  208.   int c8 = channel / C8NUM * C8NUM;

  209.   int batch = plane * channel;

  210.   for (int n = 0; n < batches; n++) {

  211.     const float16_t *src_batch = (const float16_t *)src + n * batch;

  212.     float16_t *dst_batch = (float16_t *)dst + n * batch;

  213.     int hw = 0;

  214.     for (; hw < hw16; hw += C16NUM) {

  215.       int c = 0;

  216.       for (; c < c8; c += C8NUM) {

  217.         const float16_t *src_ptr = src_batch + hw * channel + c;

  218.         float16_t *dst_ptr = dst_batch + c * plane + hw;

  219. #ifdef ENABLE_ARM64

  220.         size_t srcStride = channel * sizeof(float16_t);

  221.         size_t dstStride = plane * sizeof(float16_t);

  222.         asm volatile(

  223.           "mov x10, %[src_ptr]\n"

  224.           "mov x11, %[dst_ptr]\n"

  225. 

  226.           "ld1 {v0.8h}, [x10], %[srcStride]\n"

  227.           "ld1 {v1.8h}, [x10], %[srcStride]\n"

  228.           "ld1 {v2.8h}, [x10], %[srcStride]\n"

  229.           "ld1 {v3.8h}, [x10], %[srcStride]\n"

  230.           "ld1 {v4.8h}, [x10], %[srcStride]\n"

  231.           "ld1 {v5.8h}, [x10], %[srcStride]\n"

  232.           "ld1 {v6.8h}, [x10], %[srcStride]\n"

  233.           "ld1 {v7.8h}, [x10], %[srcStride]\n"

  234. 

  235.           "zip1 v16.8h, v0.8h, v1.8h\n"

  236.           "zip1 v17.8h, v2.8h, v3.8h\n"

  237.           "zip1 v18.8h, v4.8h, v5.8h\n"

  238.           "zip1 v19.8h, v6.8h, v7.8h\n"

  239. 

  240.           "ld1 {v8.8h}, [x10], %[srcStride]\n"

  241.           "ld1 {v9.8h}, [x10], %[srcStride]\n"

  242.           "ld1 {v10.8h}, [x10], %[srcStride]\n"

  243.           "ld1 {v11.8h}, [x10], %[srcStride]\n"

  244.           "ld1 {v12.8h}, [x10], %[srcStride]\n"

  245.           "ld1 {v13.8h}, [x10], %[srcStride]\n"

  246.           "ld1 {v14.8h}, [x10], %[srcStride]\n"

  247.           "ld1 {v15.8h}, [x10], %[srcStride]\n"

  248. 

  249.           "trn1 v20.4s, v16.4s, v17.4s\n"

  250.           "trn2 v21.4s, v16.4s, v17.4s\n"

  251.           "trn1 v22.4s, v18.4s, v19.4s\n"

  252.           "trn2 v23.4s, v18.4s, v19.4s\n"

  253. 

  254.           "trn1 v24.2d, v20.2d, v22.2d\n"

  255.           "trn2 v25.2d, v20.2d, v22.2d\n"

  256.           "trn1 v26.2d, v21.2d, v23.2d\n"

  257.           "trn2 v27.2d, v21.2d, v23.2d\n"

  258. 

  259.           "zip1 v16.8h, v8.8h, v9.8h\n"

  260.           "zip1 v17.8h, v10.8h, v11.8h\n"

  261.           "zip1 v18.8h, v12.8h, v13.8h\n"

  262.           "zip1 v19.8h, v14.8h, v15.8h\n"

  263. 

  264.           "trn1 v20.4s, v16.4s, v17.4s\n"

  265.           "trn2 v21.4s, v16.4s, v17.4s\n"

  266.           "trn1 v22.4s, v18.4s, v19.4s\n"

  267.           "trn2 v23.4s, v18.4s, v19.4s\n"

  268. 

  269.           "trn1 v28.2d, v20.2d, v22.2d\n"

  270.           "trn2 v29.2d, v20.2d, v22.2d\n"

  271.           "trn1 v30.2d, v21.2d, v23.2d\n"

  272.           "trn2 v31.2d, v21.2d, v23.2d\n"

  273. 

  274.           "add x10, x11, #16\n"

  275.           "st1 {v24.8h}, [x11], %[dstStride]\n"

  276.           "st1 {v28.8h}, [x10], %[dstStride]\n"

  277.           "st1 {v26.8h}, [x11], %[dstStride]\n"

  278.           "st1 {v30.8h}, [x10], %[dstStride]\n"

  279.           "st1 {v25.8h}, [x11], %[dstStride]\n"

  280.           "st1 {v29.8h}, [x10], %[dstStride]\n"

  281.           "st1 {v27.8h}, [x11], %[dstStride]\n"

  282.           "st1 {v31.8h}, [x10], %[dstStride]\n"

  283. 

  284.           "zip2 v16.8h, v0.8h, v1.8h\n"

  285.           "zip2 v17.8h, v2.8h, v3.8h\n"

  286.           "zip2 v18.8h, v4.8h, v5.8h\n"

  287.           "zip2 v19.8h, v6.8h, v7.8h\n"

  288. 

  289.           "trn1 v20.4s, v16.4s, v17.4s\n"

  290.           "trn2 v21.4s, v16.4s, v17.4s\n"

  291.           "trn1 v22.4s, v18.4s, v19.4s\n"

  292.           "trn2 v23.4s, v18.4s, v19.4s\n"

  293. 

  294.           "trn1 v24.2d, v20.2d, v22.2d\n"

  295.           "trn2 v25.2d, v20.2d, v22.2d\n"

  296.           "trn1 v26.2d, v21.2d, v23.2d\n"

  297.           "trn2 v27.2d, v21.2d, v23.2d\n"

  298. 

  299.           "zip2 v16.8h, v8.8h, v9.8h\n"

  300.           "zip2 v17.8h, v10.8h, v11.8h\n"

  301.           "zip2 v18.8h, v12.8h, v13.8h\n"

  302.           "zip2 v19.8h, v14.8h, v15.8h\n"

  303. 

  304.           "trn1 v20.4s, v16.4s, v17.4s\n"

  305.           "trn2 v21.4s, v16.4s, v17.4s\n"

  306.           "trn1 v22.4s, v18.4s, v19.4s\n"

  307.           "trn2 v23.4s, v18.4s, v19.4s\n"

  308. 

  309.           "trn1 v28.2d, v20.2d, v22.2d\n"

  310.           "trn2 v29.2d, v20.2d, v22.2d\n"

  311.           "trn1 v30.2d, v21.2d, v23.2d\n"

  312.           "trn2 v31.2d, v21.2d, v23.2d\n"

  313. 

  314.           "st1 {v24.8h}, [x11], %[dstStride]\n"

  315.           "st1 {v28.8h}, [x10], %[dstStride]\n"

  316.           "st1 {v26.8h}, [x11], %[dstStride]\n"

  317.           "st1 {v30.8h}, [x10], %[dstStride]\n"

  318.           "st1 {v25.8h}, [x11], %[dstStride]\n"

  319.           "st1 {v29.8h}, [x10], %[dstStride]\n"

  320.           "st1 {v27.8h}, [x11], %[dstStride]\n"

  321.           "st1 {v31.8h}, [x10], %[dstStride]\n"

  322.           :

  323.           :

  324.           [ dst_ptr ] "r"(dst_ptr), [ src_ptr ] "r"(src_ptr), [ srcStride ] "r"(srcStride), [ dstStride ] "r"(dstStride)

  325.           : "x10", "x11", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",

  326.             "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",

  327.             "v30", "v31");

  328. #else

  329.         for (int tr = 0; tr < C16NUM; tr++) {

  330.           for (int tc = 0; tc < C8NUM; tc++) {

  331.             dst_ptr[tc * plane + tr] = src_ptr[tr * channel + tc];

  332.           }

  333.         }

  334. #endif

  335.       }

  336.       for (; c < channel; c++) {

  337.         const float16_t *src_ptr = src_batch + hw * channel + c;

  338.         float16_t *dst_ptr = dst_batch + c * plane + hw;

  339.         for (size_t i = 0; i < C16NUM; i++) {

  340.           dst_ptr[i] = src_ptr[i * channel];

  341.         }

  342.       }

  343.     }

  344.     for (; hw < plane; hw++) {

  345.       const float16_t *src_ptr = src_batch + hw * channel;

  346.       float16_t *dst_ptr = dst_batch + hw;

  347.       for (size_t i = 0; i < channel; i++) {

  348.         dst_ptr[i * plane] = src_ptr[i];

  349.       }

  350.     }

  351.   }

  352.   return;

  353. }

  354. 

  355. void PackNCHWToNHWCFp16(const void *src, void *dst, int batch, int plane, int channel) {

  356.   return PackNHWCToNCHWFp16(src, dst, batch, channel, plane);

  357. }

  358. 

  359. void PackNHWCToNHWC4Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  360.   int ic4 = UP_DIV(channel, C4NUM);

  361.   int c4_channel = ic4 * C4NUM;

  362.   int nhwc4_batch_unit_offset = ic4 * C4NUM * plane;

  363.   int ic_remainder_ = channel % C4NUM;

  364.   if (ic_remainder_ != 0) {

  365.     int nhwc4_batch_offset = 0;

  366.     for (int b = 0; b < batch; b++) {

  367.       int batch_offset = b * channel * plane;

  368.       for (int i = 0; i < plane; i++) {

  369.         float16_t *dst_per_plane = (float16_t *)dst + nhwc4_batch_offset + i * c4_channel;

  370.         memcpy(dst_per_plane, (float16_t *)src + batch_offset + i * channel, channel * sizeof(float16_t));

  371.         for (int j = channel; j < c4_channel; ++j) {

  372.           dst_per_plane[j] = 0;

  373.         }

  374.       }

  375.       nhwc4_batch_offset += nhwc4_batch_unit_offset;

  376.     }

  377.   } else {

  378.     size_t ori_input_size = batch * plane * channel * sizeof(float16_t);

  379.     memcpy(dst, src, ori_input_size);

  380.   }

  381. }

  382. 

  383. void PackNHWCToNHWC8Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  384.   int ic8 = UP_DIV(channel, C8NUM);

  385.   int c8_channel = ic8 * C8NUM;

  386.   int nhwc8_batch_unit_offset = ic8 * C8NUM * plane;

  387.   int ic_remainder_ = channel % C8NUM;

  388.   if (ic_remainder_ != 0) {

  389.     int nhwc8_batch_offset = 0;

  390.     for (int b = 0; b < batch; b++) {

  391.       int batch_offset = b * channel * plane;

  392.       for (int i = 0; i < plane; i++) {

  393.         float16_t *dst_per_plane = (float16_t *)dst + nhwc8_batch_offset + i * c8_channel;

  394.         memcpy(dst_per_plane, (float16_t *)src + batch_offset + i * channel, channel * sizeof(float16_t));

  395.         for (int j = channel; j < c8_channel; ++j) {

  396.           dst_per_plane[j] = 0;

  397.         }

  398.       }

  399.       nhwc8_batch_offset += nhwc8_batch_unit_offset;

  400.     }

  401.   } else {

  402.     size_t ori_input_size = batch * plane * channel * sizeof(float16_t);

  403.     memcpy(dst, src, ori_input_size);

  404.   }

  405. }

  406. 

  407. void PackNHWC4ToNHWCFp16(const void *src, void *dst, int batch, int plane, int channel) {

  408.   int c4 = UP_DIV(channel, C4NUM);

  409.   int ic_remainder_ = channel % C4NUM;

  410.   if (ic_remainder_ != 0) {

  411.     int nhwc_batch_unit_offset = channel * plane;

  412.     for (int b = 0; b < batch; b++) {

  413.       int batch_offset = b * c4 * C4NUM * plane;

  414.       for (int i = 0; i < plane; i++) {

  415.         memcpy((float16_t *)dst + b * nhwc_batch_unit_offset + i * channel,

  416.                (float16_t *)src + batch_offset + i * c4 * C4NUM, channel * sizeof(float16_t));

  417.       }

  418.     }

  419.   } else {

  420.     size_t ori_input_size = batch * plane * channel * sizeof(float16_t);

  421.     memcpy((float16_t *)dst, (float16_t *)src, ori_input_size);

  422.   }

  423. }

  424. 

  425. void PackNCHWToNHWC4Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  426.   int nhwc4_batch_offset = 0;

  427.   int ic4 = UP_DIV(channel, C4NUM);

  428.   int nhwc4_batch_unit_offset = ic4 * C4NUM * plane;

  429. 

  430.   for (int b = 0; b < batch; b++) {

  431.     int batch_offset = b * channel * plane;

  432.     for (int c = 0; c < channel; c++) {

  433.       int src_c_offset = batch_offset + c * plane;

  434.       int dst_c_offset = nhwc4_batch_offset + c;

  435.       for (int i = 0; i < plane; i++) {

  436.         int src_plane_offset = src_c_offset + i;

  437.         int dst_plane_offset = dst_c_offset + i * ic4 * C4NUM;

  438.         ((float16_t *)dst)[dst_plane_offset] = ((float16_t *)src)[src_plane_offset];

  439.       }

  440.     }

  441.     nhwc4_batch_offset += nhwc4_batch_unit_offset;

  442.   }

  443. }

  444. 

  445. void PackNC4HW4ToNHWC4Fp16(const void *src, void *dst, int batch, int plane, int channel) {

  446.   int c4 = UP_DIV(channel, C4NUM);

  447.   for (int b = 0; b < batch; b++) {

  448.     int src_offset = b * plane * c4 * C4NUM;

  449.     int dst_offset = b * plane * channel;

  450.     for (int c = 0; c < channel; c++) {

  451.       int c4_block_num = c / C4NUM;

  452.       int c4_block_res = c % C4NUM;

  453.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  454.       int dst_c_offset = dst_offset + c4_block_num * C4NUM + c4_block_res;

  455.       for (int k = 0; k < plane; k++) {

  456.         int src_kernel_offset = src_c_offset + k * C4NUM;

  457.         int dst_kernel_offset = dst_c_offset + k * c4 * C4NUM;

  458.         ((float16_t *)dst + dst_kernel_offset)[0] = ((float16_t *)src + src_kernel_offset)[0];

  459.       }

  460.     }

  461.   }

  462. }

  463. 

  464. void PackNC4HW4ToNHWCFp16(const void *src, void *dst, int batch, int plane, int channel) {

  465.   int c4 = UP_DIV(channel, C4NUM);

  466.   for (int b = 0; b < batch; b++) {

  467.     int src_offset = b * plane * c4 * C4NUM;

  468.     int dst_offset = b * plane * channel;

  469.     for (int c = 0; c < channel; c++) {

  470.       int c4_block_num = c / C4NUM;

  471.       int c4_block_res = c % C4NUM;

  472.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  473.       int dst_c_offset = dst_offset + c;

  474.       for (int k = 0; k < plane; k++) {

  475.         int src_kernel_offset = src_c_offset + k * C4NUM;

  476.         int dst_kernel_offset = dst_c_offset + k * channel;

  477.         ((float16_t *)dst + dst_kernel_offset)[0] = ((float16_t *)src + src_kernel_offset)[0];

  478.       }

  479.     }

  480.   }

  481. }

  482. 

  483. void PackNC4HW4ToNCHWFp16(const void *src, void *dst, int batch, int plane, int channel) {

  484.   int c4 = UP_DIV(channel, C4NUM);

  485.   for (int b = 0; b < batch; b++) {

  486.     int src_offset = b * plane * c4 * C4NUM;

  487.     int dst_offset = b * plane * channel;

  488.     for (int c = 0; c < channel; c++) {

  489.       int c4_block_num = c / C4NUM;

  490.       int c4_block_res = c % C4NUM;

  491.       int src_c_offset = src_offset + c4_block_num * plane * C4NUM + c4_block_res;

  492.       int dst_c_offset = dst_offset + c * plane;

  493.       for (int k = 0; k < plane; k++) {

  494.         int src_kernel_offset = src_c_offset + k * C4NUM;

  495.         int dst_kernel_offset = dst_c_offset + k;

  496.         ((float16_t *)dst + dst_kernel_offset)[0] = ((float16_t *)src + src_kernel_offset)[0];

  497.       }

  498.     }

  499.   }

  500. }

  501. 

  502. void PackNCHWFp32ToNC8HW8Fp16(float *src, float16_t *dst, int batch, int plane, int channel) {

  503.   int c8 = UP_DIV(channel, C8NUM);

  504.   for (int b = 0; b < batch; b++) {

  505.     int src_offset = b * plane * channel;

  506.     int dst_offset = b * plane * c8 * C8NUM;

  507.     for (int c = 0; c < channel; c++) {

  508.       int c8_block_num = c / C8NUM;

  509.       int c8_block_rem = c % C8NUM;

  510.       int src_c_offset = src_offset + c * plane;

  511.       int dst_c_offset = dst_offset + c8_block_num * plane * C8NUM;

  512.       for (int k = 0; k < plane; k++) {

  513.         int src_kernel_offset = src_c_offset + k;

  514.         int dst_kernel_offset = dst_c_offset + C8NUM * k + c8_block_rem;

  515.         (dst + dst_kernel_offset)[0] = (float16_t)(src + src_kernel_offset)[0];

  516.       }

  517.     }

  518.   }

  519. }

  520. 

  521. void PackNHWCFp32ToNHWC8Fp16(float *src, float16_t *dst, int batch, int plane, int channel) {

  522.   int c8_channel = UP_DIV(channel, C8NUM) * C8NUM;

  523.   for (int b = 0; b < batch; b++) {

  524.     float16_t *dst_batch = dst + b * plane * c8_channel;

  525.     float *src_batch = src + b * plane * channel;

  526.     for (int i = 0; i < plane; i++) {

  527.       float16_t *dst_plane = dst_batch + i * c8_channel;

  528.       float *src_plane = src_batch + i * channel;

  529.       for (int c = 0; c < channel; c++) {

  530.         dst_plane[c] = (float16_t)(src_plane[c]);

  531.       }

  532.     }

  533.   }

  534. }

  535. 

  536. void PackNHWCFp32ToC8HWN8Fp16(float *src, float16_t *dst, int batch, int plane, int channel) {

  537.   for (int n = 0; n < batch; n++) {

  538.     for (int hw = 0; hw < plane; hw++) {

  539.       for (int c = 0; c < channel; c++) {

  540.         int c8div = c / C8NUM;

  541.         int c8mod = c % C8NUM;

  542.         int src_index = n * plane * channel + hw * channel + c;

  543.         int dst_index = c8div * batch * plane * C8NUM + hw * batch * C8NUM + n * C8NUM + c8mod;

  544.         dst[dst_index] = (float16_t)(src[src_index]);

  545.       }

  546.     }

  547.   }

  548.   return;

  549. }

  550. 

  551. void PackNHWC8Fp16ToNHWCFp32(float16_t *src, float *dst, int batch, int plane, int channel) {

  552.   int c8_channel = UP_DIV(channel, C8NUM) * C8NUM;

  553.   for (int b = 0; b < batch; b++) {

  554.     float16_t *src_batch = src + b * plane * c8_channel;

  555.     float *dst_batch = dst + b * plane * channel;

  556.     for (int i = 0; i < plane; i++) {

  557.       float16_t *src_plane = src_batch + i * c8_channel;

  558.       float *dst_plane = dst_batch + i * channel;

  559.       for (int c = 0; c < channel; c++) {

  560.         dst_plane[c] = (float16_t)(src_plane[c]);

  561.       }

  562.     }

  563.   }

  564. }

  565. 

  566. void PackNHWC8ToNHWCFp16(float16_t *src, float16_t *dst, int batch, int plane, int channel) {

  567.   int c8_channel = UP_DIV(channel, C8NUM) * C8NUM;

  568.   for (int b = 0; b < batch; b++) {

  569.     float16_t *src_batch = src + b * plane * c8_channel;

  570.     float16_t *dst_batch = dst + b * plane * channel;

  571.     for (int i = 0; i < plane; i++) {

  572.       float16_t *src_plane = src_batch + i * c8_channel;

  573.       float16_t *dst_plane = dst_batch + i * channel;

  574.       memcpy(dst_plane, src_plane, channel * sizeof(float16_t));

  575.     }

  576.   }

  577. }