Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 96d01f17ec
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '96d01f17ec'
${ noResults }
mindspore/mindspore/lite/nnacl/int8/matmul_int8.c

419 lines
15 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/int8/matmul_int8.h"

  18. #include <limits.h>

  19. #include "nnacl/quantization/fixed_point.h"

  20. void RowMajor2Row8MajorInt8(int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  21.   for (int r = 0; r < row; r++) {

  22.     int8_t *src = src_ptr + r * col;

  23.     for (int c = 0; c < col; c++) {

  24.       int cd8 = c / 8;

  25.       int cm8 = c % 8;

  26.       dst_ptr[cd8 * 8 * row + r * 8 + cm8] = src[c];

  27.     }

  28.   }

  29. }

  30. 

  31. void RowMajor2Row4x16MajorInt8(int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  32.   int col16 = UP_ROUND(col, C16NUM);

  33.   for (int r = 0; r < row; r++) {

  34.     int rd4 = r / C4NUM;

  35.     int rm4 = r % C4NUM;

  36.     for (int c = 0; c < col; c++) {

  37.       int cd16 = c / C16NUM;

  38.       int cm16 = c % C16NUM;

  39.       int dst_index = rd4 * col16 * C4NUM + cd16 * C4NUM * C16NUM + rm4 * C16NUM + cm16;

  40.       int src_index = r * col + c;

  41.       dst_ptr[dst_index] = src_ptr[src_index];

  42.     }

  43.   }

  44. }

  45. 

  46. void RowMajor2Row2x16MajorInt8(int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  47.   int col16 = UP_ROUND(col, C16NUM);

  48.   for (int r = 0; r < row; r++) {

  49.     int rd2 = r / C2NUM;

  50.     int rm2 = r % C2NUM;

  51.     for (int c = 0; c < col; c++) {

  52.       int cd16 = c / C16NUM;

  53.       int cm16 = c % C16NUM;

  54.       int dst_index = rd2 * col16 * C2NUM + cd16 * C2NUM * C16NUM + rm2 * C16NUM + cm16;

  55.       int src_index = r * col + c;

  56.       dst_ptr[dst_index] = src_ptr[src_index];

  57.     }

  58.   }

  59. }

  60. 

  61. void RowMajor2Row8x4MajorInt8(const int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  62.   int col4 = UP_ROUND(col, C4NUM);

  63.   for (int r = 0; r < row; r++) {

  64.     int rd8 = r / C8NUM;

  65.     int rm8 = r % C8NUM;

  66.     for (int c = 0; c < col; c++) {

  67.       int cd4 = c / C4NUM;

  68.       int cm4 = c % C4NUM;

  69.       int dst_index = rd8 * col4 * C8NUM + cd4 * C8NUM * C4NUM + rm8 * C4NUM + cm4;

  70.       int src_index = r * col + c;

  71.       dst_ptr[dst_index] = src_ptr[src_index];

  72.     }

  73.   }

  74. }

  75. 

  76. void MatrixPack4x16UnitInt8(int8_t *src, int8_t *dst, int row, int col, int stride) {

  77.   for (int r = 0; r < row; r++) {

  78.     int8_t *src_r = src + r * stride;

  79.     int8_t *dst_r = dst + r * C16NUM;

  80.     memcpy(dst_r, src_r, col * sizeof(int8_t));

  81.   }

  82.   return;

  83. }

  84. 

  85. void MatrixEmptyInt8(int8_t *dst, int row, int col) {

  86.   for (int r = 0; r < row; r++) {

  87.     int8_t *dst_r = dst + r * C16NUM;

  88.     memset(dst_r, 0, col * sizeof(int8_t));

  89.   }

  90.   return;

  91. }

  92. 

  93. void RowMajor2Row4x8MajorInt8(const int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  94.   /* Row-major to row16x4-major (block row-major) */

  95.   int col4 = UP_ROUND(col, C4NUM);

  96.   for (int r = 0; r < row; r++) {

  97.     int rd8 = r / C8NUM, rm8 = r % C8NUM;

  98.     for (int c = 0; c < col; c++) {

  99.       int cd4 = c / C4NUM, cm4 = c % C4NUM;

  100.       int src_index = r * col + c;

  101.       int dst_index = rd8 * col4 * C8NUM + cd4 * C4NUM * C8NUM + rm8 * C4NUM + cm4;

  102.       dst_ptr[dst_index] = src_ptr[src_index];

  103.     }

  104.   }

  105.   return;

  106. }

  107. 

  108. void RowMajor2Row16x4MajorInt8(void *src_ptr, void *dst_ptr, int row, int col) {

  109.   /* Row-major to row16x4-major (block row-major) */

  110.   int col16 = UP_ROUND(col, C16NUM);

  111.   size_t row_4div = row / C4NUM * C4NUM;

  112.   size_t row_4res = row - row_4div;

  113.   size_t col_16div = col / C16NUM * C16NUM;

  114.   size_t col_16res = col - col_16div;

  115.   int8_t *src_r = (int8_t *)src_ptr;

  116.   int8_t *dst_r = (int8_t *)dst_ptr;

  117. 

  118.   for (int ri = 0; ri < row_4div; ri += C4NUM) {

  119.     for (int ci = 0; ci < col_16div; ci += C16NUM) {

  120.       size_t col_offset = col;

  121.       int8_t *src_c = src_r + ci;

  122.       int8_t *dst_c = dst_r + ci * C4NUM;

  123. #ifdef ENABLE_ARM64

  124.       asm volatile(

  125.         "mov x10, %[src_c] \n"

  126.         "mov x11, %[dst_c] \n"

  127. 

  128.         "ld1 {v0.16b}, [x10], %[col_offset]\n"

  129.         "ld1 {v1.16b}, [x10], %[col_offset]\n"

  130.         "ld1 {v2.16b}, [x10], %[col_offset]\n"

  131.         "ld1 {v3.16b}, [x10], %[col_offset]\n"

  132. 

  133.         "st1 {v0.16b}, [x11], #16\n"

  134.         "st1 {v1.16b}, [x11], #16\n"

  135.         "st1 {v2.16b}, [x11], #16\n"

  136.         "st1 {v3.16b}, [x11], #16\n"

  137. 

  138.         :

  139.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ col_offset ] "r"(col_offset)

  140.         : "x10", "x11", "v0", "v1", "v2", "v3");

  141. #elif ENABLE_ARM32

  142.       asm volatile(

  143.         "mov r0, %[src_c] \n"

  144.         "mov r1, %[dst_c] \n"

  145.         "mov r2, %[col_offset] \n"

  146.         "mov r3, #16 \n"

  147. 

  148.         "vld1.8 {q0}, [r0], r2 \n"

  149.         "vld1.8 {q1}, [r0], r2 \n"

  150.         "vld1.8 {q2}, [r0], r2 \n"

  151.         "vld1.8 {q3}, [r0], r2 \n"

  152. 

  153.         "vst1.32 q0, [r1], r3 \n"

  154.         "vst1.32 q1, [r1], r3 \n"

  155.         "vst1.32 q2, [r1], r3 \n"

  156.         "vst1.32 q3, [r1], r3 \n"

  157. 

  158.         :

  159.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ col_offset ] "r"(col_offset)

  160.         : "r0", "r1", "r2", "r3", "q0", "q1", "q2", "q3");

  161. #else

  162.       MatrixPack4x16UnitInt8(src_c, dst_c, C4NUM, C16NUM, col_offset);

  163. #endif

  164.     }

  165. 

  166.     if (col != col_16div) {

  167.       MatrixPack4x16UnitInt8(src_r + col_16div, dst_r + col_16div * C4NUM, C4NUM, col_16res, col);

  168.       MatrixEmptyInt8(dst_r + col_16div * C4NUM + col_16res, C4NUM, C16NUM - col_16res);

  169.     }

  170.     src_r += C4NUM * col;

  171.     dst_r += C4NUM * col16;

  172.   }

  173. 

  174.   if (row != row_4div) {

  175.     memset(dst_r, 0, C4NUM * col16);

  176. 

  177.     for (int ci = 0; ci < col_16div; ci += C16NUM) {

  178.       MatrixPack4x16UnitInt8(src_r + ci, dst_r + ci * C4NUM, row_4res, C16NUM, col);

  179.     }

  180. 

  181.     if (col != col_16div) {

  182.       MatrixPack4x16UnitInt8(src_r + col_16div, dst_r + col_16div * C4NUM, row_4res, col_16res, col);

  183.     }

  184.   }

  185.   return;

  186. }

  187. 

  188. void RowMajor2Col8MajorInt8(int8_t *src_ptr, int8_t *dst_ptr, int row, int col) {

  189.   for (int r = 0; r < row; r++) {

  190.     int rd8 = r / 8;

  191.     int rm8 = r % 8;

  192.     for (int c = 0; c < col; c++) {

  193.       dst_ptr[rd8 * col * 8 + c * 8 + rm8] = src_ptr[r * col + c];

  194.     }

  195.   }

  196. }

  197. 

  198. void MatMulInt8_16x4(const int8_t *a, const int8_t *b, int *dst, int row_4, int col_4, int deep_16,

  199.                      const int *input_sum, const int *bias) {

  200.   /*  row4x16-major * row16x4-major => row4x4-major  */

  201.   for (int r = 0; r < row_4; r++) {

  202.     for (int c = 0; c < col_4; c++) {

  203.       int r4div = r / C4NUM, r4mod = r % C4NUM;

  204.       int c4div = c / C4NUM, c4mod = c % C4NUM;

  205.       size_t ci = c4div * row_4 * C4NUM + r * C4NUM + c4mod;

  206.       int32_t value = 0;

  207.       for (int d = 0; d < deep_16; d++) {

  208.         int d16div = d / C16NUM, d16mod = d % C16NUM;

  209.         size_t ai = r4div * deep_16 * C4NUM + d16div * C4NUM * C16NUM + r4mod * C16NUM + d16mod;

  210.         size_t bi = c4div * deep_16 * C4NUM + d16div * C4NUM * C16NUM + c4mod * C16NUM + d16mod;

  211.         value = value + a[ai] * b[bi];

  212.       }

  213.       value -= input_sum[r];

  214.       value += bias[c];

  215.       ((int32_t *)dst)[ci] = value;

  216.     }

  217.   }

  218.   return;

  219. }

  220. 

  221. void MatMulInt8_16x4_r(const int8_t *a, const int8_t *b, int8_t *dst, size_t row, size_t col, size_t deep_16,

  222.                        size_t stride, const int32_t *input_sum, const int32_t *bias, int32_t *left_shift,

  223.                        int32_t *right_shift, int32_t *multiplier, int32_t output_zp, int32_t mini, int32_t maxi,

  224.                        bool peroc) {

  225.   /* support per-layer && weight per-channel */

  226.   /*  row4x16-major * row16x4-major => (int8)row-major*/

  227.   for (int r = 0; r < row; r++) {

  228.     for (int c = 0; c < col; c++) {

  229.       int r4div = r / C4NUM, r4mod = r % C4NUM;

  230.       int c4div = c / C4NUM, c4mod = c % C4NUM;

  231.       size_t ci = r * stride + c;

  232.       int32_t value = 0;

  233.       for (int d = 0; d < deep_16; d++) {

  234.         int d16div = d / C16NUM, d16mod = d % C16NUM;

  235.         size_t ai = r4div * deep_16 * C4NUM + d16div * C4NUM * C16NUM + r4mod * C16NUM + d16mod;

  236.         size_t bi = c4div * deep_16 * C4NUM + d16div * C4NUM * C16NUM + c4mod * C16NUM + d16mod;

  237.         value = value + a[ai] * b[bi];

  238.       }

  239.       int32_t cur_input_sum =

  240.         peroc ? input_sum[c4div * UP_ROUND(row, C4NUM) * C4NUM + r * C4NUM + c4mod] : input_sum[r];

  241.       value -= cur_input_sum;

  242.       value += bias[c];

  243.       int32_t cur_left_shift = peroc ? left_shift[c] : left_shift[0];

  244.       int32_t cur_right_shift = peroc ? right_shift[c] : right_shift[0];

  245.       int32_t cur_multiplier = peroc ? multiplier[c] : multiplier[0];

  246.       value = MultiplyByQuantizedMultiplier(value, cur_multiplier, cur_left_shift, cur_right_shift) + output_zp;

  247.       value = MSMIN(maxi, value);

  248.       value = MSMAX(mini, value);

  249.       dst[ci] = (int8_t)value;

  250.     }

  251.   }

  252.   return;

  253. }

  254. 

  255. void MatMulInt8_4x2_r(const int8_t *a, const int8_t *b, int8_t *dst, size_t row, size_t col, size_t deep_16,

  256.                       size_t stride, const int32_t *input_sum, const int32_t *bias, int32_t *left_shift,

  257.                       int32_t *right_shift, int32_t *multiplier, int32_t output_zp, int32_t mini, int32_t maxi,

  258.                       bool peroc) {

  259.   /* support per-layer && weight per-channel */

  260.   /*  row4x16-major * row16x2-major => (int8)row-major*/

  261.   for (int r = 0; r < row; r++) {

  262.     for (int c = 0; c < col; c++) {

  263.       int r4div = r / C4NUM, r4mod = r % C4NUM;

  264.       int c2div = c / C2NUM, c2mod = c % C2NUM;

  265.       size_t ci = r * stride + c;

  266.       int32_t value = 0;

  267.       for (int d = 0; d < deep_16; d++) {

  268.         int d16div = d / C16NUM, d16mod = d % C16NUM;

  269.         size_t ai = r4div * deep_16 * C4NUM + d16div * C4NUM * C16NUM + r4mod * C16NUM + d16mod;

  270.         size_t bi = c2div * deep_16 * C2NUM + d16div * C2NUM * C16NUM + c2mod * C16NUM + d16mod;

  271.         value = value + a[ai] * b[bi];

  272.       }

  273.       int32_t cur_input_sum =

  274.         peroc ? input_sum[c2div * UP_ROUND(row, C4NUM) * C2NUM + r * C2NUM + c2mod] : input_sum[r];

  275.       value -= cur_input_sum;

  276.       value += bias[c];

  277.       int32_t cur_left_shift = peroc ? left_shift[c] : left_shift[0];

  278.       int32_t cur_right_shift = peroc ? right_shift[c] : right_shift[0];

  279.       int32_t cur_multiplier = peroc ? multiplier[c] : multiplier[0];

  280.       value = MultiplyByQuantizedMultiplier(value, cur_multiplier, cur_left_shift, cur_right_shift) + output_zp;

  281.       value = MSMIN(maxi, value);

  282.       value = MSMAX(mini, value);

  283.       dst[ci] = (int8_t)value;

  284.     }

  285.   }

  286.   return;

  287. }

  288. 

  289. void MatMulInt8_8x8_r(const int8_t *a, const int8_t *b, int8_t *dst, size_t row, size_t col, size_t deep_4,

  290.                       size_t stride, const int32_t *input_sum, const int32_t *bias, int32_t *left_shift,

  291.                       int32_t *right_shift, int32_t *multiplier, int32_t output_zp, int32_t mini, int32_t maxi,

  292.                       size_t per_channel) {

  293.   /*  row8x4-major * row4x8-major => (int8)row-major  */

  294.   for (int r = 0; r < row; r++) {

  295.     for (int c = 0; c < col; c++) {

  296.       int r8div = r / C8NUM, r8mod = r % C8NUM;

  297.       int c8div = c / C8NUM, c8mod = c % C8NUM;

  298.       size_t ci = r * stride + c;

  299.       int32_t value = 0;

  300.       for (int d = 0; d < deep_4; d++) {

  301.         int d4div = d / C4NUM, d4mod = d % C4NUM;

  302.         size_t ai = r8div * deep_4 * C8NUM + d4div * C8NUM * C4NUM + r8mod * C4NUM + d4mod;

  303.         size_t bi = c8div * deep_4 * C8NUM + d4div * C8NUM * C4NUM + c8mod * C4NUM + d4mod;

  304.         value = value + a[ai] * b[bi];

  305.       }

  306.       int32_t cur_input_sum =

  307.         per_channel ? input_sum[c8div * UP_ROUND(row, C8NUM) * C8NUM + r * C8NUM + c8mod] : input_sum[r];

  308.       value -= cur_input_sum;

  309.       value += bias[c];

  310.       int32_t cur_left_shift = per_channel ? left_shift[c] : left_shift[0];

  311.       int32_t cur_right_shift = per_channel ? right_shift[c] : right_shift[0];

  312.       int32_t cur_multiplier = per_channel ? multiplier[c] : multiplier[0];

  313.       value = MultiplyByQuantizedMultiplier(value, cur_multiplier, cur_left_shift, cur_right_shift) + output_zp;

  314.       value = MSMIN(maxi, value);

  315.       value = MSMAX(mini, value);

  316.       dst[ci] = (int8_t)value;

  317.     }

  318.   }

  319.   return;

  320. }

  321. 

  322. /*  row4x16-major * col16x4-major => row4x4-major  */

  323. void MatmulInt8(const int8_t *a, const int8_t *b, int8_t *dst, const int *a_sums, const int *bias, int act_min,

  324.                 int act_max, int out_zp, int multiplier, int left_shift, int right_shift, int row, int col, int deep16,

  325.                 int stride) {

  326.   int8_t *output = dst;

  327.   for (int r = 0; r < row; r++) {

  328.     for (int c = 0; c < col; c++) {

  329.       int r4div = r / C4NUM;

  330.       int r4mod = r % C4NUM;

  331.       int c4div = c / C4NUM;

  332.       int c4mod = c % C4NUM;

  333.       int value = 0;

  334.       for (int d = 0; d < deep16; d++) {

  335.         int d16div = d / C16NUM;

  336.         int d16mod = d % C16NUM;

  337.         size_t ai = r4div * deep16 * C4NUM + d16div * C4NUM * C16NUM + r4mod * C16NUM + d16mod;

  338.         size_t bi = c4div * deep16 * C4NUM + d16div * C4NUM * C16NUM + c4mod * C16NUM + d16mod;

  339.         value += a[ai] * b[bi];

  340.       }

  341.       value -= a_sums[r];

  342.       value += bias[c];

  343.       value = MultiplyByQuantizedMultiplier(value, multiplier, left_shift, right_shift) + out_zp;

  344.       value = MSMIN(INT8_MAX, value);

  345.       value = MSMAX(INT8_MIN, value);

  346.       output[c] = (int8_t)value;

  347.     }

  348.     output += stride;

  349.   }

  350. }

  351. 

  352. void RowMajor2Row4x16Major(int8_t *src, int row, int col, int8_t *dst, int col_16) {

  353.   int stride = sizeof(int8_t) * 16 * 4;

  354.   for (int r = 0; r < row; ++r) {

  355.     for (int c = 0; c < col; ++c) {

  356.       int stride_n = r / 4 * (col_16 / 16) + c / 16;

  357.       int src_idx = r * col + c;

  358.       dst[stride * stride_n + r % 4 * 16 + c % 16] = src[src_idx];

  359.     }

  360.   }

  361. }

  362. 

  363. void RowMajor2Col16x4Major(int8_t *src, int row, int col, int8_t *dst, int row_16) {

  364.   int stride = sizeof(int8_t) * 16 * 4;

  365.   for (int r = 0; r < row; ++r) {

  366.     for (int c = 0; c < col; ++c) {

  367.       int stride_n = c / 4 * (row_16 / 16) + r / 16;

  368.       int src_idx = r * col + c;

  369.       dst[stride * stride_n + c % 4 * 16 + r % 16] = src[src_idx];

  370.     }

  371.   }

  372. }

  373. 

  374. // dst: weight_zp * input_row_sums

  375. void CalcInputSums(int8_t *input, int row, int col, int weight_zp, int *dst, DataOrder order) {

  376.   for (int r = 0; r < row; ++r) {

  377.     int sum = 0;

  378.     for (int c = 0; c < col; ++c) {

  379.       if (order == RowMajor) {

  380.         sum += input[r * col + c];

  381.       } else {

  382.         sum += input[c * row + r];

  383.       }

  384.     }

  385.     sum *= weight_zp;

  386.     dst[r] = sum;

  387.   }

  388. }

  389. 

  390. // dst: bias + depth*input_zp*weight_zp - input_zp*weight_col_sums

  391. void CalcWeightBiasSums(int8_t *weight, int row, int col, int input_zp, int weight_zp, int *bias, int *dst,

  392.                         DataOrder order) {

  393.   for (int c = 0; c < col; ++c) {

  394.     int sum = 0;

  395.     for (int r = 0; r < row; ++r) {

  396.       if (order == RowMajor) {

  397.         sum += weight[r * col + c];

  398.       } else {

  399.         sum += weight[c * row + r];

  400.       }

  401.     }

  402.     dst[c] = row * input_zp * weight_zp - input_zp * sum;

  403.     if (bias != NULL) {

  404.       dst[c] += bias[c];

  405.     }

  406.   }

  407. }

  408. 

  409. void Row4x4Major2RowMajor(int8_t *src, int row4, int8_t *dst, int row, int cow) {

  410.   int stride = sizeof(int8_t) * 4 * 4;

  411.   for (int r = 0; r < row; ++r) {

  412.     for (int c = 0; c < cow; ++c) {

  413.       int sride_n = c / 4 * (row4 / 4) + r / 4;

  414.       int dst_idx = r * cow + c;

  415.       dst[dst_idx] = src[stride * sride_n + r % 4 * 4 + c % 4];

  416.     }

  417.   }

  418. }