Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: a5b16c239e
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a5b16c239e'
${ noResults }
mindspore/mindspore/lite/nnacl/fp32/matmul.c

563 lines
17 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "nnacl/fp32/matmul.h"

  18. 

  19. void RowMajor2Row4Major(float *src_ptr, float *dst_ptr, int row, int col) {

  20.   for (int r = 0; r < row; r++) {

  21.     float *src = src_ptr + r * col;

  22.     for (int c = 0; c < col; c++) {

  23.       int cd8 = c / 4;

  24.       int cm8 = c % 4;

  25.       dst_ptr[cd8 * 4 * row + r * 4 + cm8] = src[c];

  26.     }

  27.   }

  28.   return;

  29. }

  30. 

  31. void RowMajor2Row8Major(float *src_ptr, float *dst_ptr, int row, int col) {

  32.   for (int r = 0; r < row; r++) {

  33.     float *src = src_ptr + r * col;

  34.     for (int c = 0; c < col; c++) {

  35.       int cd8 = c / 8;

  36.       int cm8 = c % 8;

  37.       dst_ptr[cd8 * 8 * row + r * 8 + cm8] = src[c];

  38.     }

  39.   }

  40.   return;

  41. }

  42. 

  43. void RowMajor2Row12Major(float *src_ptr, float *dst_ptr, int row, int col) {

  44.   for (int r = 0; r < row; r++) {

  45.     float *src = src_ptr + r * col;

  46.     for (int c = 0; c < col; c++) {

  47.       int cd8 = c / C12NUM;

  48.       int cm8 = c % C12NUM;

  49.       dst_ptr[cd8 * C12NUM * row + r * C12NUM + cm8] = src[c];

  50.     }

  51.   }

  52.   return;

  53. }

  54. 

  55. void RowMajor2Col12Major(float *src_ptr, float *dst_ptr, size_t row, size_t col) {

  56.   size_t row_up_12 = UP_ROUND(row, C12NUM);

  57.   size_t row12 = row / C12NUM * C12NUM;

  58.   size_t col4 = col / C4NUM * C4NUM;

  59.   float *src_r = src_ptr;

  60.   float *dst_r = dst_ptr;

  61. 

  62.   size_t ri = 0;

  63.   for (; ri < row12; ri += C12NUM) {

  64.     size_t ci = 0;

  65.     for (; ci < col4; ci += C4NUM) {

  66.       float *src_c = src_r + ci;

  67.       float *dst_c = dst_r + ci * C12NUM;

  68. 

  69.       /* 12x4 row-major to col-major */

  70. #ifdef ENABLE_ARM64

  71.       size_t stride = col * sizeof(float);

  72.       asm volatile(

  73.         "mov x10, %[src_c]\n"

  74.         "mov x11, %[dst_c]\n"

  75. 

  76.         "ld1 {v0.4s}, [x10], %[stride]\n"

  77.         "ld1 {v1.4s}, [x10], %[stride]\n"

  78.         "ld1 {v2.4s}, [x10], %[stride]\n"

  79.         "ld1 {v3.4s}, [x10], %[stride]\n"

  80. 

  81.         "ld1 {v4.4s}, [x10], %[stride]\n"

  82.         "ld1 {v5.4s}, [x10], %[stride]\n"

  83.         "ld1 {v6.4s}, [x10], %[stride]\n"

  84.         "ld1 {v7.4s}, [x10], %[stride]\n"

  85. 

  86.         "zip1 v12.4s, v0.4s, v1.4s\n"

  87.         "zip2 v13.4s, v0.4s, v1.4s\n"

  88.         "zip1 v14.4s, v2.4s, v3.4s\n"

  89.         "zip2 v15.4s, v2.4s, v3.4s\n"

  90. 

  91.         "ld1 {v8.4s}, [x10], %[stride]\n"

  92.         "ld1 {v9.4s}, [x10], %[stride]\n"

  93.         "ld1 {v10.4s}, [x10], %[stride]\n"

  94.         "ld1 {v11.4s}, [x10], %[stride]\n"

  95. 

  96.         "zip1 v16.4s, v4.4s, v5.4s\n"

  97.         "zip2 v17.4s, v4.4s, v5.4s\n"

  98.         "zip1 v18.4s, v6.4s, v7.4s\n"

  99.         "zip2 v19.4s, v6.4s, v7.4s\n"

  100. 

  101.         "trn1 v20.2d, v12.2d, v14.2d\n"

  102.         "trn2 v23.2d, v12.2d, v14.2d\n"

  103.         "trn1 v26.2d, v13.2d, v15.2d\n"

  104.         "trn2 v29.2d, v13.2d, v15.2d\n"

  105. 

  106.         "trn1 v21.2d, v16.2d, v18.2d\n"

  107.         "trn2 v24.2d, v16.2d, v18.2d\n"

  108.         "trn1 v27.2d, v17.2d, v19.2d\n"

  109.         "trn2 v30.2d, v17.2d, v19.2d\n"

  110. 

  111.         "zip1 v12.4s, v8.4s, v9.4s\n"

  112.         "zip2 v13.4s, v8.4s, v9.4s\n"

  113.         "zip1 v14.4s, v10.4s, v11.4s\n"

  114.         "zip2 v15.4s, v10.4s, v11.4s\n"

  115. 

  116.         "trn1 v22.2d, v12.2d, v14.2d\n"

  117.         "trn2 v25.2d, v12.2d, v14.2d\n"

  118.         "trn1 v28.2d, v13.2d, v15.2d\n"

  119.         "trn2 v31.2d, v13.2d, v15.2d\n"

  120. 

  121.         "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x11], #64\n"

  122.         "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x11], #64\n"

  123.         "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x11], #64\n"

  124. 

  125.         :

  126.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ stride ] "r"(stride)

  127.         : "x10", "x11", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",

  128.           "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",

  129.           "v30", "v31");

  130. #elif ENABLE_ARM32

  131.       size_t stride = col * sizeof(float);

  132.       asm volatile(

  133.         "mov r10, %[src_c]\n"

  134.         "mov r12, %[dst_c]\n"

  135. 

  136.         "vld1.32 {q0}, [r10], %[stride]\n"

  137.         "vld1.32 {q3}, [r10], %[stride]\n"

  138.         "vld1.32 {q10}, [r10], %[stride]\n"

  139.         "vld1.32 {q13}, [r10], %[stride]\n"

  140. 

  141.         "vtrn.32 d0, d6\n"

  142.         "vtrn.32 d1, d7\n"

  143.         "vtrn.32 d20, d26\n"

  144.         "vtrn.32 d21, d27\n"

  145. 

  146.         "vld1.32 {q1}, [r10], %[stride]\n"

  147.         "vld1.32 {q8}, [r10], %[stride]\n"

  148.         "vld1.32 {q11}, [r10], %[stride]\n"

  149.         "vld1.32 {q14}, [r10], %[stride]\n"

  150. 

  151.         "vswp d1, d20\n"

  152.         "vswp d7, d26\n"

  153. 

  154.         "vld1.32 {q2}, [r10], %[stride]\n"

  155.         "vld1.32 {q9}, [r10], %[stride]\n"

  156.         "vld1.32 {q12}, [r10], %[stride]\n"

  157.         "vld1.32 {q15}, [r10], %[stride]\n"

  158. 

  159.         "vtrn.32 d2, d16\n"

  160.         "vtrn.32 d3, d17\n"

  161.         "vtrn.32 d22, d28\n"

  162.         "vtrn.32 d23, d29\n"

  163. 

  164.         "vswp d3, d22\n"

  165.         "vswp d17, d28\n"

  166. 

  167.         "vtrn.32 d4, d18\n"

  168.         "vtrn.32 d5, d19\n"

  169.         "vtrn.32 d24, d30\n"

  170.         "vtrn.32 d25, d31\n"

  171. 

  172.         "vswp d5, d24\n"

  173.         "vswp d19, d30\n"

  174. 

  175.         "vst1.32 {q0, q1}, [r12]!\n"

  176.         "vst1.32 {q2, q3}, [r12]!\n"

  177.         "vst1.32 {q8, q9}, [r12]!\n"

  178.         "vst1.32 {q10, q11}, [r12]!\n"

  179.         "vst1.32 {q12, q13}, [r12]!\n"

  180.         "vst1.32 {q14, q15}, [r12]!\n"

  181. 

  182.         :

  183.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ stride ] "r"(stride)

  184.         : "r10", "r12", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  185. #else

  186.       for (int tr = 0; tr < C12NUM; tr++) {

  187.         for (int tc = 0; tc < C4NUM; tc++) {

  188.           dst_c[tc * C12NUM + tr] = src_c[tr * col + tc];

  189.         }

  190.       }

  191. #endif

  192.     }

  193.     for (; ci < col; ci++) {

  194.       float *src_c = src_r + ci;

  195.       float *dst_c = dst_r + ci * C12NUM;

  196.       for (size_t i = 0; i < C12NUM; i++) {

  197.         dst_c[i] = src_c[i * col];

  198.       }

  199.     }

  200.     src_r += C12NUM * col;

  201.     dst_r += C12NUM * col;

  202.   }

  203. 

  204.   for (; ri < row; ri++) {

  205.     for (size_t i = 0; i < col; i++) {

  206.       dst_r[i * C12NUM] = src_r[i];

  207.     }

  208.     src_r += col;

  209.     dst_r += 1;

  210.   }

  211. 

  212.   for (; ri < row_up_12; ri++) {

  213.     for (size_t i = 0; i < col; i++) {

  214.       dst_r[i * C12NUM] = 0;

  215.     }

  216.     dst_r += 1;

  217.   }

  218.   return;

  219. }

  220. 

  221. void RowMajor2Col8Major(float *src_ptr, float *dst_ptr, size_t row, size_t col) {

  222.   size_t row8 = row / C8NUM * C8NUM;

  223.   size_t col4 = col / C4NUM * C4NUM;

  224.   float *src_r = src_ptr;

  225.   float *dst_r = dst_ptr;

  226. 

  227.   size_t ri = 0;

  228.   for (; ri < row8; ri += C8NUM) {

  229.     size_t ci = 0;

  230.     for (; ci < col4; ci += C4NUM) {

  231.       float *src_c = src_r + ci;

  232.       float *dst_c = dst_r + ci * C8NUM;

  233. 

  234.       /* 8x4 row-major to col-major */

  235. #ifdef ENABLE_ARM64

  236.       size_t stride = col * sizeof(float);

  237.       asm volatile(

  238.         "mov x10, %[src_c]\n"

  239.         "mov x11, %[dst_c]\n"

  240. 

  241.         "ld1 {v0.4s}, [x10], %[stride]\n"

  242.         "ld1 {v1.4s}, [x10], %[stride]\n"

  243.         "ld1 {v2.4s}, [x10], %[stride]\n"

  244.         "ld1 {v3.4s}, [x10], %[stride]\n"

  245. 

  246.         "zip1 v4.4s, v0.4s, v1.4s\n"

  247.         "zip2 v5.4s, v0.4s, v1.4s\n"

  248.         "zip1 v6.4s, v2.4s, v3.4s\n"

  249.         "zip2 v7.4s, v2.4s, v3.4s\n"

  250. 

  251.         "ld1 {v8.4s},  [x10], %[stride]\n"

  252.         "ld1 {v9.4s},  [x10], %[stride]\n"

  253.         "ld1 {v10.4s}, [x10],  %[stride]\n"

  254.         "ld1 {v11.4s}, [x10],  %[stride]\n"

  255. 

  256.         "trn1 v0.2d, v4.2d, v6.2d\n"

  257.         "trn2 v1.2d, v4.2d, v6.2d\n"

  258.         "trn1 v2.2d, v5.2d, v7.2d\n"

  259.         "trn2 v3.2d, v5.2d, v7.2d\n"

  260. 

  261.         "zip1 v12.4s, v8.4s, v9.4s\n"

  262.         "zip2 v13.4s, v8.4s, v9.4s\n"

  263.         "zip1 v14.4s, v10.4s, v11.4s\n"

  264.         "zip2 v15.4s, v10.4s, v11.4s\n"

  265. 

  266.         "trn1 v8.2d, v12.2d, v14.2d\n"

  267.         "trn2 v9.2d, v12.2d, v14.2d\n"

  268.         "trn1 v10.2d, v13.2d, v15.2d\n"

  269.         "trn2 v11.2d, v13.2d, v15.2d\n"

  270. 

  271.         "st1 {v0.4s}, [x11],  #16\n"

  272.         "st1 {v8.4s}, [x11],  #16\n"

  273.         "st1 {v1.4s}, [x11],  #16\n"

  274.         "st1 {v9.4s}, [x11],  #16\n"

  275.         "st1 {v2.4s},  [x11],#16\n"

  276.         "st1 {v10.4s}, [x11], #16\n"

  277.         "st1 {v3.4s},  [x11],#16\n"

  278.         "st1 {v11.4s}, [x11], #16\n"

  279. 

  280.         :

  281.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ stride ] "r"(stride)

  282.         : "x10", "x11", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",

  283.           "v15");

  284. #elif ENABLE_ARM32

  285.       size_t stride = col * sizeof(float);

  286.       asm volatile(

  287.         "mov r10, %[src_c]\n"

  288.         "mov r11, %[dst_c]\n"

  289. 

  290.         "vld1.32 {q0}, [r10], %[stride]\n"

  291.         "vld1.32 {q2}, [r10], %[stride]\n"

  292.         "vld1.32 {q4}, [r10], %[stride]\n"

  293.         "vld1.32 {q6}, [r10], %[stride]\n"

  294. 

  295.         "vtrn.32 d0, d4\n"

  296.         "vtrn.32 d1, d5\n"

  297.         "vtrn.32 d8, d12\n"

  298.         "vtrn.32 d9, d13\n"

  299. 

  300.         "vld1.32 {q1}, [r10], %[stride]\n"

  301.         "vld1.32 {q3}, [r10], %[stride]\n"

  302.         "vld1.32 {q5}, [r10], %[stride]\n"

  303.         "vld1.32 {q7}, [r10], %[stride]\n"

  304. 

  305.         "vswp d1, d8\n"

  306.         "vswp d5, d12\n"

  307. 

  308.         "vtrn.32 d2, d6\n"

  309.         "vtrn.32 d3, d7\n"

  310.         "vtrn.32 d10, d14\n"

  311.         "vtrn.32 d11, d15\n"

  312. 

  313.         "vswp d3, d10\n"

  314.         "vswp d7, d14\n"

  315. 

  316.         "vst1.32 {q0, q1}, [r11]!\n"

  317.         "vst1.32 {q2, q3}, [r11]!\n"

  318.         "vst1.32 {q4, q5}, [r11]!\n"

  319.         "vst1.32 {q6, q7}, [r11]!\n"

  320. 

  321.         :

  322.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ stride ] "r"(stride)

  323.         : "r10", "r11", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7");

  324. #else

  325.       for (int tr = 0; tr < 8; tr++) {

  326.         for (int tc = 0; tc < 4; tc++) {

  327.           dst_c[tc * 8 + tr] = src_c[tr * col + tc];

  328.         }

  329.       }

  330. #endif

  331.     }

  332.     for (; ci < col; ci++) {

  333.       float *src_c = src_r + ci;

  334.       float *dst_c = dst_r + ci * C8NUM;

  335.       for (size_t i = 0; i < C8NUM; i++) {

  336.         dst_c[i] = src_c[i * col];

  337.       }

  338.     }

  339.     src_r += C8NUM * col;

  340.     dst_r += C8NUM * col;

  341.   }

  342.   for (; ri < row; ri++) {

  343.     for (size_t i = 0; i < col; i++) {

  344.       dst_r[i * C8NUM] = src_r[i];

  345.     }

  346.     src_r += col;

  347.     dst_r += 1;

  348.   }

  349.   return;

  350. }

  351. 

  352. void RowMajor2Col4Major(float *src_ptr, float *dst_ptr, size_t row, size_t col) {

  353.   size_t row8 = row / C4NUM * C4NUM;

  354.   size_t col4 = col / C4NUM * C4NUM;

  355.   float *src_r = src_ptr;

  356.   float *dst_r = dst_ptr;

  357. 

  358.   size_t ri = 0;

  359.   for (; ri < row8; ri += C4NUM) {

  360.     size_t ci = 0;

  361.     for (; ci < col4; ci += C4NUM) {

  362.       float *src_c = src_r + ci;

  363.       float *dst_c = dst_r + ci * C4NUM;

  364. 

  365.       /* 4x4 row-major to col-major */

  366. #ifdef ENABLE_ARM32

  367.       size_t stride = col * 4;

  368.       asm volatile(

  369.         "mov r10, %[src_c]\n"

  370.         "mov r12, %[dst_c]\n"

  371. 

  372.         "vld1.32 {q0}, [r10], %[stride]\n"

  373.         "vld1.32 {q1}, [r10], %[stride]\n"

  374.         "vld1.32 {q2}, [r10], %[stride]\n"

  375.         "vld1.32 {q3}, [r10], %[stride]\n"

  376. 

  377.         "vtrn.32 d0, d2\n"

  378.         "vtrn.32 d1, d3\n"

  379.         "vtrn.32 d4, d6\n"

  380.         "vtrn.32 d5, d7\n"

  381. 

  382.         "vswp d1, d4\n"

  383.         "vswp d3, d6\n"

  384. 

  385.         "vst1.32 {q0}, [r12]!\n"

  386.         "vst1.32 {q1}, [r12]!\n"

  387.         "vst1.32 {q2}, [r12]!\n"

  388.         "vst1.32 {q3}, [r12]!\n"

  389. 

  390.         :

  391.         : [ dst_c ] "r"(dst_c), [ src_c ] "r"(src_c), [ stride ] "r"(stride)

  392.         : "r10", "r12", "q0", "q1", "q2", "q3");

  393. #else

  394.       for (int tr = 0; tr < C4NUM; tr++) {

  395.         for (int tc = 0; tc < C4NUM; tc++) {

  396.           dst_c[tc * C4NUM + tr] = src_c[tr * col + tc];

  397.         }

  398.       }

  399. #endif

  400.     }

  401.     for (; ci < col; ci++) {

  402.       float *src_c = src_r + ci;

  403.       float *dst_c = dst_r + ci * C4NUM;

  404.       for (size_t i = 0; i < C4NUM; i++) {

  405.         dst_c[i] = src_c[i * col];

  406.       }

  407.     }

  408.     src_r += C4NUM * col;

  409.     dst_r += C4NUM * col;

  410.   }

  411.   for (; ri < row; ri++) {

  412.     for (size_t i = 0; i < col; i++) {

  413.       dst_r[i * C4NUM] = src_r[i];

  414.     }

  415.     src_r += col;

  416.     dst_r += 1;

  417.   }

  418.   return;

  419. }

  420. 

  421. void MatMul12x8(const float *a, const float *b, float *dst, const float *bias, ActType act_type, int deep, int row,

  422.                 int col, int stride, int out_type) {

  423.   if (out_type == OutType_Nhwc) {

  424.     for (int r = 0; r < row; r++) {

  425.       for (int c = 0; c < col; c++) {

  426.         int r12div = r / 12, r12mod = r % 12;

  427.         int c8div = c / 8, c8mod = c % 8;

  428.         size_t ci = r * stride + c;

  429.         float value = 0;

  430.         for (int d = 0; d < deep; d++) {

  431.           size_t ai = r12div * deep * 12 + d * 12 + r12mod;

  432.           size_t bi = c8div * deep * 8 + d * 8 + c8mod;

  433.           value = value + a[ai] * b[bi];

  434.         }

  435.         if (bias != NULL) value += bias[c];

  436.         if (act_type == ActType_Relu6) value = MSMIN(6.0f, value);

  437.         if (act_type != ActType_No) value = MSMAX(0.0f, value);

  438.         dst[ci] = value;

  439.       }

  440.     }

  441.   } else if (out_type == OutType_C8) {

  442.     int col_8 = UP_ROUND(col, C8NUM);

  443.     int row_12 = UP_ROUND(row, C12NUM);

  444.     for (int r = 0; r < row_12; r++) {

  445.       for (int c = 0; c < col_8; c++) {

  446.         int r12div = r / C12NUM, r12mod = r % C12NUM;

  447.         int c8div = c / C8NUM, c8mod = c % C8NUM;

  448.         size_t ci = (c8div * C8NUM * row_12 + r * C8NUM + c8mod);

  449.         float value = 0;

  450.         for (int d = 0; d < deep; d++) {

  451.           size_t ai = r12div * deep * C12NUM + d * C12NUM + r12mod;

  452.           size_t bi = c8div * deep * C8NUM + d * C8NUM + c8mod;

  453.           value = value + a[ai] * b[bi];

  454.         }

  455.         if (bias != NULL) value += bias[c];

  456.         if (act_type == ActType_Relu6) value = MSMIN(6.0f, value);

  457.         if (act_type != ActType_No) value = MSMAX(0.0f, value);

  458.         dst[ci] = value;

  459.       }

  460.     }

  461.   } else {

  462.     for (int i = 0; i < row; ++i) {

  463.       int src_r_offset = i;

  464.       int dst_r_offset = i * col * stride;

  465.       for (int j = 0; j < col; ++j) {

  466.         int c8div = j / 8, c8mod = j % 8;

  467.         size_t ci = dst_r_offset + c8div * 8 * stride + c8mod;

  468.         float value = 0;

  469.         for (int d = 0; d < deep; ++d) {

  470.           size_t ai = src_r_offset + d * C12NUM;

  471.           size_t bi = c8div * deep * 8 + d * 8 + c8mod;

  472.           value = value + a[ai] * b[bi];

  473.         }

  474.         if (bias != NULL) value += bias[j];

  475.         if (act_type == ActType_Relu6) value = MSMIN(6.0f, value);

  476.         if (act_type != ActType_No) value = MSMAX(0.0f, value);

  477.         dst[ci] = value;

  478.       }

  479.     }

  480.   }

  481.   return;

  482. }

  483. 

  484. void MatMul4x8(const float *a, const float *b, float *dst, const float *bias, ActType act_type, int deep, int row,

  485.                int col, int stride, int out_type) {

  486.   if (out_type == OutType_C8) {

  487.     int col_8 = UP_ROUND(col, C8NUM);

  488.     int row_4 = UP_ROUND(row, C4NUM);

  489.     for (int r = 0; r < row_4; r++) {

  490.       for (int c = 0; c < col_8; c++) {

  491.         int r4div = r / C4NUM, r4mod = r % C4NUM;

  492.         int c8div = c / C8NUM, c8mod = c % C8NUM;

  493.         size_t ci = (c8div * C8NUM * row_4 + r * C8NUM + c8mod);

  494.         float value = 0;

  495.         for (int d = 0; d < deep; d++) {

  496.           size_t ai = r4div * deep * C4NUM + d * C4NUM + r4mod;

  497.           size_t bi = c8div * deep * C8NUM + d * C8NUM + c8mod;

  498.           value = value + a[ai] * b[bi];

  499.         }

  500.         if (bias != NULL) value += bias[c];

  501.         if (act_type == ActType_Relu6) value = MSMIN(6.0f, value);

  502.         if (act_type != ActType_No) value = MSMAX(0.0f, value);

  503.         dst[ci] = value;

  504.       }

  505.     }

  506.   }

  507.   return;

  508. }

  509. 

  510. void MatMulOpt(const float *a, const float *b, float *c, const float *bias, ActType act_type, int deep, int row,

  511.                int col, size_t stride, int out_type) {

  512. #ifdef ENABLE_ARM64

  513.   if (out_type == OutType_C8) {

  514.     MatmulFloatNeon64(a, b, c, bias, (int)act_type, deep, row, col, stride, 0, 0);

  515.   } else {

  516.     MatmulFloatNeon64Opt(a, b, c, bias, (int)act_type, deep, row, col, stride, (int)(out_type));

  517.   }

  518. #elif ENABLE_ARM32

  519.   if (out_type == OutType_C8) {

  520.     MatmulFloatNeon32(a, b, c, bias, (int)act_type, deep, row, col, stride, 0, 0);

  521.   } else {

  522.     MatmulFloatNeon32Opt(a, b, c, bias, (int)act_type, deep, row, col, stride, (int)(out_type));

  523.   }

  524. #else

  525.   MatMul12x8(a, b, c, bias, act_type, deep, row, col, stride, out_type);

  526. #endif

  527. }

  528. 

  529. #ifdef ENABLE_NNACL_INFER_SHAPE

  530. static void SwapDims(int *dims, int index1, int index2) {

  531.   int tmp = dims[index1];

  532.   dims[index1] = dims[index2];

  533.   dims[index2] = tmp;

  534. }

  535. 

  536. int MatMulInferShape(int **in_shape, int in_num, size_t *dim_size, int *out_shape, int *in_format, int *out_format,

  537.                      int *in_datatype, int *out_datatype, OpParameter *param) {

  538.   *out_datatype = in_datatype[0];

  539.   *out_format = in_format[0];

  540.   if (dim_size[0] < 2 || dim_size[1] < 2) {

  541.     return NNACL_PARAM_INVALID;

  542.   }

  543. 

  544.   for (int i = 0; i < dim_size[0] - 2; ++i) {

  545.     if (in_shape[0][i] != in_shape[1][i]) {

  546.       return NNACL_PARAM_INVALID;

  547.     }

  548.   }

  549.   MatMulParameter *matmul_param = (MatMulParameter *)param;

  550.   if (matmul_param->a_transpose_) {

  551.     SwapDims(in_shape[0], dim_size[0] - 1, dim_size[0] - 2);

  552.   }

  553.   if (matmul_param->b_transpose_) {

  554.     SwapDims(in_shape[1], dim_size[1] - 1, dim_size[1] - 2);

  555.   }

  556.   for (int i = 0; i < dim_size[0] - 1; ++i) {

  557.     out_shape[i] = in_shape[0][i];

  558.   }

  559.   out_shape[dim_size[0] - 1] = in_shape[1][dim_size[1] - 1];

  560.   return NNACL_OK;

  561. }

  562. #endif