wangwei
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 369c2ccc5a
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '369c2ccc5a'
${ noResults }
MegEngine/dnn/src/aarch64/matrix_mul/int16/strategy_mk8_8x8.cpp

746 lines
28 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/aarch64/matrix_mul/int16/strategy_mk8_8x8.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #include "src/aarch64/matrix_mul/asm/common.h"

  13. #include "src/aarch64/matrix_mul/int16/strategy.h"

  14. #include "src/arm_common/simd_macro/marm_neon.h"

  15. #include "src/common/utils.h"

  16. 

  17. using namespace megdnn;

  18. using namespace aarch64;

  19. using namespace aarch64::matmul;

  20. 

  21. namespace {

  22. 

  23. void kern_8x1(

  24.         const dt_int16* a_ptr, const dt_int16* b_ptr, int LDB, int K,

  25.         dt_int32* output) {

  26.     //! As each load 32 number from B, but the pos add 24 * 2, so we minus 24

  27.     //! here.

  28.     LDB *= sizeof(dt_int16);

  29. 

  30.     asm volatile(

  31.             "subs %w[K], %w[K], #8\n"

  32.             "ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%[a_ptr]], 64\n"

  33.             "ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%[a_ptr]], 64\n"

  34.             "ld1 {v0.4s}, [%[b_ptr]], %x[LDB]\n"

  35. 

  36.             "smull v16.4s, v24.4h, v0.h[0]\n"

  37.             "smull2 v17.4s, v24.8h, v0.h[0]\n"

  38.             "smull v18.4s, v25.4h, v0.h[1]\n"

  39.             "smull2 v19.4s, v25.8h, v0.h[1]\n"

  40.             "smull v20.4s, v26.4h, v0.h[2]\n"

  41.             "smull2 v21.4s, v26.8h, v0.h[2]\n"

  42.             "smull v22.4s, v27.4h, v0.h[3]\n"

  43.             "smull2 v23.4s, v27.8h, v0.h[3]\n"

  44. 

  45.             "beq 2f\n"

  46. 

  47.             "1:\n"

  48.             "ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%[a_ptr]], 64\n"

  49.             "smlal v16.4s, v28.4h, v0.h[4]\n"

  50.             "smlal2 v17.4s, v28.8h, v0.h[4]\n"

  51.             "smlal v18.4s, v29.4h, v0.h[5]\n"

  52.             "smlal2 v19.4s, v29.8h, v0.h[5]\n"

  53.             "smlal v20.4s, v30.4h, v0.h[6]\n"

  54.             "smlal2 v21.4s, v30.8h, v0.h[6]\n"

  55.             "smlal v22.4s, v31.4h, v0.h[7]\n"

  56.             "smlal2 v23.4s, v31.8h, v0.h[7]\n"

  57. 

  58.             "ld1 {v0.4s}, [%[b_ptr]], %x[LDB]\n"

  59.             "ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%[a_ptr]], 64\n"

  60. 

  61.             "smlal v16.4s, v24.4h, v0.h[0]\n"

  62.             "smlal2 v17.4s, v24.8h, v0.h[0]\n"

  63.             "smlal v18.4s, v25.4h, v0.h[1]\n"

  64.             "smlal2 v19.4s, v25.8h, v0.h[1]\n"

  65.             "smlal v20.4s, v26.4h, v0.h[2]\n"

  66.             "smlal2 v21.4s, v26.8h, v0.h[2]\n"

  67.             "smlal v22.4s, v27.4h, v0.h[3]\n"

  68.             "smlal2 v23.4s, v27.8h, v0.h[3]\n"

  69. 

  70.             "subs %w[K], %w[K], #8\n"

  71.             "bne 1b\n"

  72. 

  73.             "2:\n"

  74.             "smlal v16.4s, v28.4h, v0.h[4]\n"

  75.             "smlal2 v17.4s, v28.8h, v0.h[4]\n"

  76.             "smlal v18.4s, v29.4h, v0.h[5]\n"

  77.             "smlal2 v19.4s, v29.8h, v0.h[5]\n"

  78.             "smlal v20.4s, v30.4h, v0.h[6]\n"

  79.             "smlal2 v21.4s, v30.8h, v0.h[6]\n"

  80.             "smlal v22.4s, v31.4h, v0.h[7]\n"

  81.             "smlal2 v23.4s, v31.8h, v0.h[7]\n"

  82. 

  83.             "add v16.4s, v16.4s, v18.4s\n"

  84.             "add v20.4s, v20.4s, v22.4s\n"

  85.             "add v17.4s, v17.4s, v19.4s\n"

  86.             "add v21.4s, v21.4s, v23.4s\n"

  87.             "add v16.4s, v16.4s, v20.4s\n"

  88.             "add v17.4s, v17.4s, v21.4s\n"

  89. 

  90.             "st1 {v16.4s, v17.4s}, [%[output]], 32\n"

  91. 

  92.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr), [K] "+r"(K),

  93.               [output] "+r"(output), [LDB] "+r"(LDB)

  94.             :

  95.             : "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",

  96.               "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory");

  97. }

  98. 

  99. // Overview of register layout:

  100. //

  101. // A 8x1 cell of Lhs is stored in 16bit in v24-v27

  102. // A 8x1 cell of Rhs is stored in 16bit in v0-v7

  103. // A 8x2 block of accumulators is stored in 32bit in v16-v23.

  104. //

  105. //                  Lhs +--------+

  106. //                      |v0[0-7]|

  107. //                      |v1[0-7]|

  108. //                      |v2[0-7]|

  109. //                      |v3[0-7]|

  110. //                      +--------+

  111. //      Rhs

  112. //  +---------+ - - - - -+--------+

  113. //  | v24[0-7]|          |v16[0-3]|

  114. //  | v25[0-7]|          |v17[0-3]|

  115. //  | v26[0-7]|          |v18[0-3]|

  116. //  | v27[0-7]|          |v19[0-3]|

  117. //  | v28[0-7]|          |v20[0-3]|

  118. //  | v29[0-7]|          |v21[0-3]|

  119. //  | v30[0-7]|          |v22[0-3]|

  120. //  | v31[0-7]|          |v23[0-3]|

  121. //  +---------+          +--------+

  122. //                      Accumulator

  123. void kern_8x4(

  124.         const dt_int16* a_ptr, const dt_int16* b_ptr, int LDB, int K,

  125.         dt_int32* output) {

  126.     //! As each load 32 number from B, but the pos add 24 * 2, so we minus 24

  127.     //! here.

  128.     LDB = (LDB - 24) * sizeof(dt_int16);

  129. 

  130.     asm volatile(

  131.             "subs %w[K], %w[K], #8\n"

  132. 

  133.             "ld1 {v24.4s}, [%[a_ptr]], 16\n"

  134.             "ld1 {v0.4s}, [%[b_ptr]], 16\n"

  135.             "ld1 {v1.4s}, [%[b_ptr]], 16\n"

  136.             "ld1 {v2.4s}, [%[b_ptr]], 16\n"

  137.             "ld1 {v3.4s}, [%[b_ptr]], %x[LDB]\n"

  138. 

  139.             "smull v16.4s, v24.4h, v0.h[0]\n"

  140.             "smull2 v17.4s, v24.8h, v0.h[0]\n"

  141.             "smull v18.4s, v24.4h, v1.h[0]\n"

  142.             "smull2 v19.4s, v24.8h, v1.h[0]\n"

  143. 

  144.             "ld1 {v25.4s}, [%[a_ptr]], 16\n"

  145. 

  146.             "smull v20.4s, v24.4h, v2.h[0]\n"

  147.             "smull2 v21.4s, v24.8h, v2.h[0]\n"

  148.             "smull v22.4s, v24.4h, v3.h[0]\n"

  149.             "smull2 v23.4s, v24.8h, v3.h[0]\n"

  150. 

  151.             "smlal v16.4s, v25.4h, v0.h[1]\n"

  152.             "smlal2 v17.4s, v25.8h, v0.h[1]\n"

  153.             "smlal v18.4s, v25.4h, v1.h[1]\n"

  154.             "smlal2 v19.4s, v25.8h, v1.h[1]\n"

  155. 

  156.             "ld1 {v26.4s}, [%[a_ptr]], 16\n"

  157. 

  158.             "smlal v20.4s, v25.4h, v2.h[1]\n"

  159.             "smlal2 v21.4s, v25.8h, v2.h[1]\n"

  160.             "smlal v22.4s, v25.4h, v3.h[1]\n"

  161.             "smlal2 v23.4s, v25.8h, v3.h[1]\n"

  162. 

  163.             "smlal v16.4s, v26.4h, v0.h[2]\n"

  164.             "smlal2 v17.4s, v26.8h, v0.h[2]\n"

  165.             "smlal v18.4s, v26.4h, v1.h[2]\n"

  166.             "smlal2 v19.4s, v26.8h, v1.h[2]\n"

  167. 

  168.             "ld1 {v27.4s}, [%[a_ptr]], 16\n"

  169. 

  170.             "smlal v20.4s, v26.4h, v2.h[2]\n"

  171.             "smlal2 v21.4s, v26.8h, v2.h[2]\n"

  172.             "smlal v22.4s, v26.4h, v3.h[2]\n"

  173.             "smlal2 v23.4s, v26.8h, v3.h[2]\n"

  174. 

  175.             "smlal v16.4s, v27.4h, v0.h[3]\n"

  176.             "smlal2 v17.4s, v27.8h, v0.h[3]\n"

  177.             "smlal v18.4s, v27.4h, v1.h[3]\n"

  178.             "smlal2 v19.4s, v27.8h, v1.h[3]\n"

  179. 

  180.             "ld1 {v28.4s}, [%[a_ptr]], 16\n"

  181. 

  182.             "smlal v20.4s, v27.4h, v2.h[3]\n"

  183.             "smlal2 v21.4s, v27.8h, v2.h[3]\n"

  184.             "smlal v22.4s, v27.4h, v3.h[3]\n"

  185.             "smlal2 v23.4s, v27.8h, v3.h[3]\n"

  186. 

  187.             "smlal v16.4s, v28.4h, v0.h[4]\n"

  188.             "smlal2 v17.4s, v28.8h, v0.h[4]\n"

  189.             "smlal v18.4s, v28.4h, v1.h[4]\n"

  190.             "smlal2 v19.4s, v28.8h, v1.h[4]\n"

  191. 

  192.             "ld1 {v29.4s}, [%[a_ptr]], 16\n"

  193. 

  194.             "smlal v20.4s, v28.4h, v2.h[4]\n"

  195.             "smlal2 v21.4s, v28.8h, v2.h[4]\n"

  196.             "smlal v22.4s, v28.4h, v3.h[4]\n"

  197.             "smlal2 v23.4s, v28.8h, v3.h[4]\n"

  198. 

  199.             "smlal v16.4s, v29.4h, v0.h[5]\n"

  200.             "smlal2 v17.4s, v29.8h, v0.h[5]\n"

  201.             "smlal v18.4s, v29.4h, v1.h[5]\n"

  202.             "smlal2 v19.4s, v29.8h, v1.h[5]\n"

  203. 

  204.             "ld1 {v30.4s}, [%[a_ptr]], 16\n"

  205. 

  206.             "smlal v20.4s, v29.4h, v2.h[5]\n"

  207.             "smlal2 v21.4s, v29.8h, v2.h[5]\n"

  208.             "smlal v22.4s, v29.4h, v3.h[5]\n"

  209.             "smlal2 v23.4s, v29.8h, v3.h[5]\n"

  210. 

  211.             "smlal v16.4s, v30.4h, v0.h[6]\n"

  212.             "smlal2 v17.4s, v30.8h, v0.h[6]\n"

  213.             "smlal v18.4s, v30.4h, v1.h[6]\n"

  214.             "smlal2 v19.4s, v30.8h, v1.h[6]\n"

  215. 

  216.             "ld1 {v31.4s}, [%[a_ptr]], 16\n"

  217. 

  218.             "smlal v20.4s, v30.4h, v2.h[6]\n"

  219.             "smlal2 v21.4s, v30.8h, v2.h[6]\n"

  220.             "smlal v22.4s, v30.4h, v3.h[6]\n"

  221.             "smlal2 v23.4s, v30.8h, v3.h[6]\n"

  222. 

  223.             "beq 2f\n"

  224. 

  225.             "1:\n"

  226. 

  227.             "ld1 {v24.4s}, [%[a_ptr]], 16\n"

  228. 

  229.             "smlal v16.4s, v31.4h, v0.h[7]\n"

  230.             "smlal2 v17.4s, v31.8h, v0.h[7]\n"

  231. 

  232.             "ld1 {v0.4s}, [%[b_ptr]], 16\n"

  233. 

  234.             "smlal v18.4s, v31.4h, v1.h[7]\n"

  235.             "smlal2 v19.4s, v31.8h, v1.h[7]\n"

  236. 

  237.             "ld1 {v1.4s}, [%[b_ptr]], 16\n"

  238. 

  239.             "smlal v20.4s, v31.4h, v2.h[7]\n"

  240.             "smlal2 v21.4s, v31.8h, v2.h[7]\n"

  241. 

  242.             "ld1 {v2.4s}, [%[b_ptr]], 16\n"

  243. 

  244.             "smlal v22.4s, v31.4h, v3.h[7]\n"

  245.             "smlal2 v23.4s, v31.8h, v3.h[7]\n"

  246. 

  247.             "ld1 {v3.4s}, [%[b_ptr]], %x[LDB]\n"

  248. 

  249.             "smlal v16.4s, v24.4h, v0.h[0]\n"

  250.             "smlal2 v17.4s, v24.8h, v0.h[0]\n"

  251.             "smlal v18.4s, v24.4h, v1.h[0]\n"

  252.             "smlal2 v19.4s, v24.8h, v1.h[0]\n"

  253. 

  254.             "ld1 {v25.4s}, [%[a_ptr]], 16\n"

  255. 

  256.             "smlal v20.4s, v24.4h, v2.h[0]\n"

  257.             "smlal2 v21.4s, v24.8h, v2.h[0]\n"

  258.             "smlal v22.4s, v24.4h, v3.h[0]\n"

  259.             "smlal2 v23.4s, v24.8h, v3.h[0]\n"

  260. 

  261.             "smlal v16.4s, v25.4h, v0.h[1]\n"

  262.             "smlal2 v17.4s, v25.8h, v0.h[1]\n"

  263.             "smlal v18.4s, v25.4h, v1.h[1]\n"

  264.             "smlal2 v19.4s, v25.8h, v1.h[1]\n"

  265. 

  266.             "ld1 {v26.4s}, [%[a_ptr]], 16\n"

  267. 

  268.             "smlal v20.4s, v25.4h, v2.h[1]\n"

  269.             "smlal2 v21.4s, v25.8h, v2.h[1]\n"

  270.             "smlal v22.4s, v25.4h, v3.h[1]\n"

  271.             "smlal2 v23.4s, v25.8h, v3.h[1]\n"

  272. 

  273.             "smlal v16.4s, v26.4h, v0.h[2]\n"

  274.             "smlal2 v17.4s, v26.8h, v0.h[2]\n"

  275.             "smlal v18.4s, v26.4h, v1.h[2]\n"

  276.             "smlal2 v19.4s, v26.8h, v1.h[2]\n"

  277. 

  278.             "ld1 {v27.4s}, [%[a_ptr]], 16\n"

  279. 

  280.             "smlal v20.4s, v26.4h, v2.h[2]\n"

  281.             "smlal2 v21.4s, v26.8h, v2.h[2]\n"

  282.             "smlal v22.4s, v26.4h, v3.h[2]\n"

  283.             "smlal2 v23.4s, v26.8h, v3.h[2]\n"

  284. 

  285.             "smlal v16.4s, v27.4h, v0.h[3]\n"

  286.             "smlal2 v17.4s, v27.8h, v0.h[3]\n"

  287.             "smlal v18.4s, v27.4h, v1.h[3]\n"

  288.             "smlal2 v19.4s, v27.8h, v1.h[3]\n"

  289. 

  290.             "ld1 {v28.4s}, [%[a_ptr]], 16\n"

  291. 

  292.             "smlal v20.4s, v27.4h, v2.h[3]\n"

  293.             "smlal2 v21.4s, v27.8h, v2.h[3]\n"

  294.             "smlal v22.4s, v27.4h, v3.h[3]\n"

  295.             "smlal2 v23.4s, v27.8h, v3.h[3]\n"

  296. 

  297.             "smlal v16.4s, v28.4h, v0.h[4]\n"

  298.             "smlal2 v17.4s, v28.8h, v0.h[4]\n"

  299.             "smlal v18.4s, v28.4h, v1.h[4]\n"

  300.             "smlal2 v19.4s, v28.8h, v1.h[4]\n"

  301. 

  302.             "ld1 {v29.4s}, [%[a_ptr]], 16\n"

  303. 

  304.             "smlal v20.4s, v28.4h, v2.h[4]\n"

  305.             "smlal2 v21.4s, v28.8h, v2.h[4]\n"

  306.             "smlal v22.4s, v28.4h, v3.h[4]\n"

  307.             "smlal2 v23.4s, v28.8h, v3.h[4]\n"

  308. 

  309.             "smlal v16.4s, v29.4h, v0.h[5]\n"

  310.             "smlal2 v17.4s, v29.8h, v0.h[5]\n"

  311.             "smlal v18.4s, v29.4h, v1.h[5]\n"

  312.             "smlal2 v19.4s, v29.8h, v1.h[5]\n"

  313. 

  314.             "ld1 {v30.4s}, [%[a_ptr]], 16\n"

  315. 

  316.             "smlal v20.4s, v29.4h, v2.h[5]\n"

  317.             "smlal2 v21.4s, v29.8h, v2.h[5]\n"

  318.             "smlal v22.4s, v29.4h, v3.h[5]\n"

  319.             "smlal2 v23.4s, v29.8h, v3.h[5]\n"

  320. 

  321.             "smlal v16.4s, v30.4h, v0.h[6]\n"

  322.             "smlal2 v17.4s, v30.8h, v0.h[6]\n"

  323.             "smlal v18.4s, v30.4h, v1.h[6]\n"

  324.             "smlal2 v19.4s, v30.8h, v1.h[6]\n"

  325. 

  326.             "ld1 {v31.4s}, [%[a_ptr]], 16\n"

  327. 

  328.             "smlal v20.4s, v30.4h, v2.h[6]\n"

  329.             "smlal2 v21.4s, v30.8h, v2.h[6]\n"

  330.             "smlal v22.4s, v30.4h, v3.h[6]\n"

  331.             "smlal2 v23.4s, v30.8h, v3.h[6]\n"

  332. 

  333.             "subs %w[K], %w[K], #8\n"

  334.             "bne 1b\n"

  335. 

  336.             "2:\n"

  337. 

  338.             "smlal v16.4s, v31.4h, v0.h[7]\n"

  339.             "smlal2 v17.4s, v31.8h, v0.h[7]\n"

  340.             "smlal v18.4s, v31.4h, v1.h[7]\n"

  341.             "smlal2 v19.4s, v31.8h, v1.h[7]\n"

  342.             "smlal v20.4s, v31.4h, v2.h[7]\n"

  343.             "smlal2 v21.4s, v31.8h, v2.h[7]\n"

  344.             "smlal v22.4s, v31.4h, v3.h[7]\n"

  345.             "smlal2 v23.4s, v31.8h, v3.h[7]\n"

  346. 

  347.             "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[output]], 64\n"

  348.             "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%[output]], 64\n"

  349. 

  350.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr), [K] "+r"(K),

  351.               [output] "+r"(output), [LDB] "+r"(LDB)

  352.             :

  353.             : "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20", "v21", "v22",

  354.               "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc",

  355.               "memory");

  356. }

  357. 

  358. // Overview of register layout:

  359. //

  360. // A 8x1 cell of Rhs is stored in 16bit in v8-v15

  361. // A 8x1 cell of Lhs is stored in 16bit in v0-v7

  362. // A 8x2 block of accumulators is stored in 32bit in v16-v31.

  363. //

  364. //                  Rhs +--------+

  365. //                      | v8[0-7]|

  366. //                      | v9[0-7]|

  367. //                      |v10[0-7]|

  368. //                      |v11[0-7]|

  369. //                      |v12[0-7]|

  370. //                      |v13[0-7]|

  371. //                      |v14[0-7]|

  372. //                      |v15[0-7]|

  373. //                      +--------+

  374. //      Lhs

  375. //  +--------+ - - - - -+--------+--------+

  376. //  | v0[0-7]|          |v16[0-3]|v17[0-3]|

  377. //  | v1[0-7]|          |v18[0-3]|v19[0-3]|

  378. //  | v2[0-7]|          |v20[0-3]|v21[0-3]|

  379. //  | v3[0-7]|          |v22[0-3]|v23[0-3]|

  380. //  | v4[0-7]|          |v24[0-3]|v25[0-3]|

  381. //  | v5[0-7]|          |v26[0-3]|v27[0-3]|

  382. //  | v6[0-7]|          |v28[0-3]|v29[0-3]|

  383. //  | v7[0-7]|          |v30[0-3]|v31[0-3]|

  384. //  +--------+          +--------+--------+

  385. //                      Accumulator

  386. void kern_8x8(

  387.         const dt_int16* a_ptr, const dt_int16* b_ptr, int LDB, int K,

  388.         dt_int32* output) {

  389.     //! As each load 64 number from B, but the pos add 48 * 2, so we minus 48

  390.     //! here.

  391.     LDB = (LDB - 48) * sizeof(dt_int16);

  392. 

  393.     asm volatile(

  394.             "subs %w[K], %w[K], #8\n"

  395.             "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%[a_ptr]], 64\n"

  396.             "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%[a_ptr]], 64\n"

  397. 

  398.             "ld1 {v0.4s}, [%[b_ptr]], 16\n"

  399.             "smull v16.4s, v8.4h, v0.h[0]\n"

  400.             "ld1 {v1.4s}, [%[b_ptr]], 16\n"

  401.             "smlal v16.4s, v9.4h, v0.h[1]\n"

  402.             "smull v18.4s, v8.4h, v1.h[0]\n"

  403.             "smull2 v17.4s, v8.8h, v0.h[0]\n"

  404.             "smull2 v19.4s, v8.8h, v1.h[0]\n"

  405.             "smlal v16.4s, v10.4h, v0.h[2]\n"

  406.             "smlal v18.4s, v9.4h, v1.h[1]\n"

  407.             "smlal2 v17.4s, v9.8h, v0.h[1]\n"

  408.             "smlal2 v19.4s, v9.8h, v1.h[1]\n"

  409.             "smlal v16.4s, v11.4h, v0.h[3]\n"

  410.             "smlal v18.4s, v10.4h, v1.h[2]\n"

  411.             "smlal2 v17.4s, v10.8h, v0.h[2]\n"

  412.             "smlal2 v19.4s, v10.8h, v1.h[2]\n"

  413.             "smlal v16.4s, v12.4h, v0.h[4]\n"

  414.             "smlal v18.4s, v11.4h, v1.h[3]\n"

  415.             "smlal2 v17.4s, v11.8h, v0.h[3]\n"

  416.             "smlal2 v19.4s, v11.8h, v1.h[3]\n"

  417.             "smlal v16.4s, v13.4h, v0.h[5]\n"

  418.             "smlal v18.4s, v12.4h, v1.h[4]\n"

  419.             "smlal2 v17.4s, v12.8h, v0.h[4]\n"

  420.             "smlal2 v19.4s, v12.8h, v1.h[4]\n"

  421.             "smlal2 v17.4s, v13.8h, v0.h[5]\n"

  422. 

  423.             "ld1 {v2.4s, v3.4s}, [%[b_ptr]], 32\n"

  424.             "smlal v16.4s, v14.4h, v0.h[6]\n"

  425.             "smlal v18.4s, v13.4h, v1.h[5]\n"

  426.             "smlal2 v17.4s, v14.8h, v0.h[6]\n"

  427.             "smlal2 v19.4s, v13.8h, v1.h[5]\n"

  428.             "smull v20.4s, v8.4h, v2.h[0]\n"

  429.             "smull v22.4s, v8.4h, v3.h[0]\n"

  430.             "smull2 v21.4s, v8.8h, v2.h[0]\n"

  431.             "smull2 v23.4s, v8.8h, v3.h[0]\n"

  432.             "smlal v16.4s, v15.4h, v0.h[7]\n"

  433.             "smlal v18.4s, v14.4h, v1.h[6]\n"

  434.             "smlal2 v17.4s, v15.8h, v0.h[7]\n"

  435.             "smlal2 v19.4s, v14.8h, v1.h[6]\n"

  436.             "smlal v20.4s, v9.4h, v2.h[1]\n"

  437.             "smlal v22.4s, v9.4h, v3.h[1]\n"

  438.             "smlal2 v21.4s, v9.8h, v2.h[1]\n"

  439.             "smlal2 v23.4s, v9.8h, v3.h[1]\n"

  440.             "smlal v18.4s, v15.4h, v1.h[7]\n"

  441.             "smlal v20.4s, v10.4h, v2.h[2]\n"

  442.             "smlal v22.4s, v10.4h, v3.h[2]\n"

  443.             "smlal2 v21.4s, v10.8h, v2.h[2]\n"

  444.             "smlal2 v23.4s, v10.8h, v3.h[2]\n"

  445.             "smlal2 v19.4s, v15.8h, v1.h[7]\n"

  446.             "smlal v20.4s, v11.4h, v2.h[3]\n"

  447.             "smlal v22.4s, v11.4h, v3.h[3]\n"

  448.             "smlal2 v21.4s, v11.8h, v2.h[3]\n"

  449.             "smlal2 v23.4s, v11.8h, v3.h[3]\n"

  450.             "smlal v20.4s, v12.4h, v2.h[4]\n"

  451.             "smlal v22.4s, v12.4h, v3.h[4]\n"

  452.             "smlal2 v21.4s, v12.8h, v2.h[4]\n"

  453.             "smlal2 v23.4s, v12.8h, v3.h[4]\n"

  454.             "smlal v20.4s, v13.4h, v2.h[5]\n"

  455.             "smlal v22.4s, v13.4h, v3.h[5]\n"

  456.             "smlal2 v21.4s, v13.8h, v2.h[5]\n"

  457.             "smlal2 v23.4s, v13.8h, v3.h[5]\n"

  458. 

  459.             "ld1 {v4.4s, v5.4s}, [%[b_ptr]], 32\n"

  460.             "smlal v20.4s, v14.4h, v2.h[6]\n"

  461.             "smlal v22.4s, v14.4h, v3.h[6]\n"

  462.             "smlal2 v21.4s, v14.8h, v2.h[6]\n"

  463.             "smlal2 v23.4s, v14.8h, v3.h[6]\n"

  464.             "smull v24.4s, v8.4h, v4.h[0]\n"

  465.             "smull v26.4s, v8.4h, v5.h[0]\n"

  466.             "smull2 v25.4s, v8.8h, v4.h[0]\n"

  467.             "smull2 v27.4s, v8.8h, v5.h[0]\n"

  468.             "smlal v20.4s, v15.4h, v2.h[7]\n"

  469.             "smlal v22.4s, v15.4h, v3.h[7]\n"

  470.             "smlal2 v21.4s, v15.8h, v2.h[7]\n"

  471.             "smlal2 v23.4s, v15.8h, v3.h[7]\n"

  472.             "smlal v24.4s, v9.4h, v4.h[1]\n"

  473.             "smlal v26.4s, v9.4h, v5.h[1]\n"

  474.             "smlal2 v25.4s, v9.8h, v4.h[1]\n"

  475.             "smlal2 v27.4s, v9.8h, v5.h[1]\n"

  476.             "smlal v24.4s, v10.4h, v4.h[2]\n"

  477.             "smlal v26.4s, v10.4h, v5.h[2]\n"

  478.             "smlal2 v25.4s, v10.8h, v4.h[2]\n"

  479.             "smlal2 v27.4s, v10.8h, v5.h[2]\n"

  480.             "smlal v24.4s, v11.4h, v4.h[3]\n"

  481.             "smlal v26.4s, v11.4h, v5.h[3]\n"

  482.             "smlal2 v25.4s, v11.8h, v4.h[3]\n"

  483.             "smlal2 v27.4s, v11.8h, v5.h[3]\n"

  484.             "smlal v24.4s, v12.4h, v4.h[4]\n"

  485.             "smlal v26.4s, v12.4h, v5.h[4]\n"

  486.             "smlal2 v25.4s, v12.8h, v4.h[4]\n"

  487.             "smlal2 v27.4s, v12.8h, v5.h[4]\n"

  488.             "smlal v24.4s, v13.4h, v4.h[5]\n"

  489.             "smlal v26.4s, v13.4h, v5.h[5]\n"

  490.             "smlal2 v25.4s, v13.8h, v4.h[5]\n"

  491.             "smlal2 v27.4s, v13.8h, v5.h[5]\n"

  492. 

  493.             "ld1 {v6.4s, v7.4s}, [%[b_ptr]], %x[LDB]\n"

  494.             "smlal v24.4s, v14.4h, v4.h[6]\n"

  495.             "smlal v26.4s, v14.4h, v5.h[6]\n"

  496.             "smlal2 v25.4s, v14.8h, v4.h[6]\n"

  497.             "smlal2 v27.4s, v14.8h, v5.h[6]\n"

  498.             "smull v28.4s, v8.4h, v6.h[0]\n"

  499.             "smull v30.4s, v8.4h, v7.h[0]\n"

  500.             "smull2 v29.4s, v8.8h, v6.h[0]\n"

  501.             "smull2 v31.4s, v8.8h, v7.h[0]\n"

  502.             "smlal v28.4s, v9.4h, v6.h[1]\n"

  503.             "smlal v30.4s, v9.4h, v7.h[1]\n"

  504.             "smlal2 v29.4s, v9.8h, v6.h[1]\n"

  505.             "smlal2 v31.4s, v9.8h, v7.h[1]\n"

  506.             "smlal v28.4s, v10.4h, v6.h[2]\n"

  507.             "smlal v30.4s, v10.4h, v7.h[2]\n"

  508.             "smlal2 v29.4s, v10.8h, v6.h[2]\n"

  509.             "smlal2 v31.4s, v10.8h, v7.h[2]\n"

  510.             "smlal v28.4s, v11.4h, v6.h[3]\n"

  511.             "smlal v30.4s, v11.4h, v7.h[3]\n"

  512.             "smlal2 v29.4s, v11.8h, v6.h[3]\n"

  513.             "smlal2 v31.4s, v11.8h, v7.h[3]\n"

  514.             "smlal v28.4s, v12.4h, v6.h[4]\n"

  515.             "smlal v30.4s, v12.4h, v7.h[4]\n"

  516.             "smlal2 v29.4s, v12.8h, v6.h[4]\n"

  517.             "smlal2 v31.4s, v12.8h, v7.h[4]\n"

  518.             "smlal v28.4s, v13.4h, v6.h[5]\n"

  519.             "smlal v30.4s, v13.4h, v7.h[5]\n"

  520.             "smlal2 v29.4s, v13.8h, v6.h[5]\n"

  521.             "smlal2 v31.4s, v13.8h, v7.h[5]\n"

  522. 

  523.             "beq 2f\n"

  524. 

  525.             "1:\n"

  526. 

  527.             "smlal v24.4s, v15.4h, v4.h[7]\n"

  528.             "smlal v26.4s, v15.4h, v5.h[7]\n"

  529.             "smlal2 v25.4s, v15.8h, v4.h[7]\n"

  530. 

  531.             "ld1 {v8.4s, v9.4s}, [%[a_ptr]], 32\n"

  532.             "smlal2 v27.4s, v15.8h, v5.h[7]\n"

  533.             "smlal v28.4s, v14.4h, v6.h[6]\n"

  534.             "smlal v30.4s, v14.4h, v7.h[6]\n"

  535. 

  536.             "ld1 {v10.4s, v11.4s}, [%[a_ptr]], 32\n"

  537.             "smlal2 v29.4s, v15.8h, v6.h[7]\n"

  538.             "smlal2 v31.4s, v14.8h, v7.h[6]\n"

  539.             "smlal v28.4s, v15.4h, v6.h[7]\n"

  540. 

  541.             "ld1 {v12.4s, v13.4s}, [%[a_ptr]], 32\n"

  542.             "smlal v30.4s, v15.4h, v7.h[7]\n"

  543.             "smlal2 v29.4s, v14.8h, v6.h[6]\n"

  544. 

  545.             "ld1 {v0.4s}, [%[b_ptr]], 16\n"

  546.             "smlal2 v31.4s, v15.8h, v7.h[7]\n"

  547.             "smlal v16.4s, v8.4h, v0.h[0]\n"

  548. 

  549.             "ld1 {v1.4s}, [%[b_ptr]], 16\n"

  550.             "smlal v16.4s, v9.4h, v0.h[1]\n"

  551.             "smlal2 v17.4s, v8.8h, v0.h[0]\n"

  552.             "smlal v16.4s, v10.4h, v0.h[2]\n"

  553.             "smlal v18.4s, v8.4h, v1.h[0]\n"

  554.             "smlal2 v17.4s, v9.8h, v0.h[1]\n"

  555.             "smlal2 v19.4s, v8.8h, v1.h[0]\n"

  556. 

  557.             "ld1 {v14.4s, v15.4s}, [%[a_ptr]], 32\n"

  558.             "smlal v16.4s, v11.4h, v0.h[3]\n"

  559.             "smlal v18.4s, v9.4h, v1.h[1]\n"

  560.             "smlal2 v17.4s, v10.8h, v0.h[2]\n"

  561.             "smlal2 v19.4s, v9.8h, v1.h[1]\n"

  562.             "smlal v16.4s, v12.4h, v0.h[4]\n"

  563.             "smlal v18.4s, v10.4h, v1.h[2]\n"

  564.             "smlal2 v17.4s, v11.8h, v0.h[3]\n"

  565.             "smlal2 v19.4s, v10.8h, v1.h[2]\n"

  566.             "smlal v16.4s, v13.4h, v0.h[5]\n"

  567.             "smlal v18.4s, v11.4h, v1.h[3]\n"

  568.             "smlal2 v17.4s, v12.8h, v0.h[4]\n"

  569.             "smlal2 v19.4s, v11.8h, v1.h[3]\n"

  570.             "smlal v16.4s, v14.4h, v0.h[6]\n"

  571.             "smlal v18.4s, v12.4h, v1.h[4]\n"

  572.             "smlal2 v17.4s, v13.8h, v0.h[5]\n"

  573.             "smlal2 v19.4s, v12.8h, v1.h[4]\n"

  574.             "smlal v16.4s, v15.4h, v0.h[7]\n"

  575.             "smlal v18.4s, v13.4h, v1.h[5]\n"

  576.             "smlal2 v17.4s, v14.8h, v0.h[6]\n"

  577.             "smlal2 v19.4s, v13.8h, v1.h[5]\n"

  578. 

  579.             "ld1 {v2.4s, v3.4s}, [%[b_ptr]], 32\n"

  580.             "smlal v18.4s, v14.4h, v1.h[6]\n"

  581.             "smlal2 v17.4s, v15.8h, v0.h[7]\n"

  582.             "smlal2 v19.4s, v14.8h, v1.h[6]\n"

  583.             "smlal v20.4s, v8.4h, v2.h[0]\n"

  584.             "smlal v22.4s, v8.4h, v3.h[0]\n"

  585.             "smlal2 v21.4s, v8.8h, v2.h[0]\n"

  586.             "smlal2 v23.4s, v8.8h, v3.h[0]\n"

  587.             "smlal v18.4s, v15.4h, v1.h[7]\n"

  588.             "smlal v20.4s, v9.4h, v2.h[1]\n"

  589.             "smlal v22.4s, v9.4h, v3.h[1]\n"

  590.             "smlal2 v21.4s, v9.8h, v2.h[1]\n"

  591.             "smlal2 v23.4s, v9.8h, v3.h[1]\n"

  592.             "smlal2 v19.4s, v15.8h, v1.h[7]\n"

  593.             "smlal v20.4s, v10.4h, v2.h[2]\n"

  594.             "smlal v22.4s, v10.4h, v3.h[2]\n"

  595.             "smlal2 v21.4s, v10.8h, v2.h[2]\n"

  596.             "smlal2 v23.4s, v10.8h, v3.h[2]\n"

  597.             "smlal v20.4s, v11.4h, v2.h[3]\n"

  598.             "smlal v22.4s, v11.4h, v3.h[3]\n"

  599.             "smlal2 v21.4s, v11.8h, v2.h[3]\n"

  600.             "smlal2 v23.4s, v11.8h, v3.h[3]\n"

  601.             "smlal v20.4s, v12.4h, v2.h[4]\n"

  602.             "smlal v22.4s, v12.4h, v3.h[4]\n"

  603.             "smlal2 v21.4s, v12.8h, v2.h[4]\n"

  604.             "smlal2 v23.4s, v12.8h, v3.h[4]\n"

  605.             "smlal v20.4s, v13.4h, v2.h[5]\n"

  606.             "smlal v22.4s, v13.4h, v3.h[5]\n"

  607.             "smlal2 v21.4s, v13.8h, v2.h[5]\n"

  608.             "smlal2 v23.4s, v13.8h, v3.h[5]\n"

  609. 

  610.             "ld1 {v4.4s, v5.4s}, [%[b_ptr]], 32\n"

  611.             "smlal v20.4s, v14.4h, v2.h[6]\n"

  612.             "smlal v22.4s, v14.4h, v3.h[6]\n"

  613.             "smlal2 v21.4s, v14.8h, v2.h[6]\n"

  614.             "smlal2 v23.4s, v14.8h, v3.h[6]\n"

  615.             "smlal v24.4s, v8.4h, v4.h[0]\n"

  616.             "smlal v26.4s, v8.4h, v5.h[0]\n"

  617.             "smlal2 v25.4s, v8.8h, v4.h[0]\n"

  618.             "smlal2 v27.4s, v8.8h, v5.h[0]\n"

  619.             "smlal v20.4s, v15.4h, v2.h[7]\n"

  620.             "smlal2 v21.4s, v15.8h, v2.h[7]\n"

  621.             "smlal v22.4s, v15.4h, v3.h[7]\n"

  622.             "smlal2 v23.4s, v15.8h, v3.h[7]\n"

  623.             "smlal v24.4s, v9.4h, v4.h[1]\n"

  624.             "smlal v26.4s, v9.4h, v5.h[1]\n"

  625.             "smlal2 v25.4s, v9.8h, v4.h[1]\n"

  626.             "smlal2 v27.4s, v9.8h, v5.h[1]\n"

  627.             "smlal v24.4s, v10.4h, v4.h[2]\n"

  628.             "smlal v26.4s, v10.4h, v5.h[2]\n"

  629.             "smlal2 v25.4s, v10.8h, v4.h[2]\n"

  630.             "smlal2 v27.4s, v10.8h, v5.h[2]\n"

  631.             "smlal v24.4s, v11.4h, v4.h[3]\n"

  632.             "smlal v26.4s, v11.4h, v5.h[3]\n"

  633.             "smlal2 v25.4s, v11.8h, v4.h[3]\n"

  634.             "smlal2 v27.4s, v11.8h, v5.h[3]\n"

  635.             "smlal v24.4s, v12.4h, v4.h[4]\n"

  636.             "smlal v26.4s, v12.4h, v5.h[4]\n"

  637.             "smlal2 v25.4s, v12.8h, v4.h[4]\n"

  638.             "smlal2 v27.4s, v12.8h, v5.h[4]\n"

  639.             "smlal v24.4s, v13.4h, v4.h[5]\n"

  640.             "smlal v26.4s, v13.4h, v5.h[5]\n"

  641.             "smlal2 v25.4s, v13.8h, v4.h[5]\n"

  642.             "smlal2 v27.4s, v13.8h, v5.h[5]\n"

  643. 

  644.             "ld1 {v6.4s, v7.4s}, [%[b_ptr]], %x[LDB]\n"

  645.             "smlal v24.4s, v14.4h, v4.h[6]\n"

  646.             "smlal v26.4s, v14.4h, v5.h[6]\n"

  647.             "smlal2 v25.4s, v14.8h, v4.h[6]\n"

  648.             "smlal2 v27.4s, v14.8h, v5.h[6]\n"

  649.             "smlal v28.4s, v8.4h, v6.h[0]\n"

  650.             "smlal v30.4s, v8.4h, v7.h[0]\n"

  651.             "smlal2 v29.4s, v8.8h, v6.h[0]\n"

  652.             "smlal2 v31.4s, v8.8h, v7.h[0]\n"

  653.             "smlal v28.4s, v9.4h, v6.h[1]\n"

  654.             "smlal v30.4s, v9.4h, v7.h[1]\n"

  655.             "smlal2 v29.4s, v9.8h, v6.h[1]\n"

  656.             "smlal2 v31.4s, v9.8h, v7.h[1]\n"

  657.             "smlal v28.4s, v10.4h, v6.h[2]\n"

  658.             "smlal v30.4s, v10.4h, v7.h[2]\n"

  659.             "smlal2 v29.4s, v10.8h, v6.h[2]\n"

  660.             "smlal2 v31.4s, v10.8h, v7.h[2]\n"

  661.             "smlal v28.4s, v11.4h, v6.h[3]\n"

  662.             "smlal v30.4s, v11.4h, v7.h[3]\n"

  663.             "smlal2 v29.4s, v11.8h, v6.h[3]\n"

  664.             "smlal2 v31.4s, v11.8h, v7.h[3]\n"

  665.             "smlal v28.4s, v12.4h, v6.h[4]\n"

  666.             "smlal v30.4s, v12.4h, v7.h[4]\n"

  667.             "smlal2 v29.4s, v12.8h, v6.h[4]\n"

  668.             "smlal2 v31.4s, v12.8h, v7.h[4]\n"

  669.             "smlal v28.4s, v13.4h, v6.h[5]\n"

  670.             "smlal v30.4s, v13.4h, v7.h[5]\n"

  671.             "smlal2 v29.4s, v13.8h, v6.h[5]\n"

  672.             "smlal2 v31.4s, v13.8h, v7.h[5]\n"

  673. 

  674.             "subs %w[K], %w[K], #8\n"

  675.             "bne 1b\n"

  676. 

  677.             "2:\n"

  678.             "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[output]], 64\n"

  679.             "smlal v24.4s, v15.4h, v4.h[7]\n"

  680.             "smlal v28.4s, v14.4h, v6.h[6]\n"

  681.             "smlal v30.4s, v14.4h, v7.h[6]\n"

  682.             "smlal v26.4s, v15.4h, v5.h[7]\n"

  683.             "smlal2 v25.4s, v15.8h, v4.h[7]\n"

  684.             "smlal2 v27.4s, v15.8h, v5.h[7]\n"

  685.             "smlal2 v29.4s, v14.8h, v6.h[6]\n"

  686.             "smlal2 v31.4s, v14.8h, v7.h[6]\n"

  687.             "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%[output]], 64\n"

  688.             "smlal v28.4s, v15.4h, v6.h[7]\n"

  689.             "smlal v30.4s, v15.4h, v7.h[7]\n"

  690.             "smlal2 v29.4s, v15.8h, v6.h[7]\n"

  691.             "smlal2 v31.4s, v15.8h, v7.h[7]\n"

  692.             "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%[output]], 64\n"

  693.             "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%[output]], 64\n"

  694. 

  695.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr), [K] "+r"(K),

  696.               [output] "+r"(output), [LDB] "+r"(LDB)

  697.             :

  698.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  699.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",

  700.               "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",

  701.               "cc", "memory");

  702. }

  703. 

  704. }  // anonymous namespace

  705. 

  706. MEGDNN_REG_GEMM_STRATEGY_IMPL_NOPACK(gemm_nopack_s16_8x8);

  707. 

  708. void gemm_nopack_s16_8x8::kern(

  709.         const dt_int16* A, size_t LDA, const dt_int16* B, size_t LDB, dt_int32* C,

  710.         size_t LDC, size_t M, size_t K, size_t N, const dt_int32*, void*, bool trA,

  711.         bool trB) const {

  712.     constexpr static size_t MB = 8;

  713.     constexpr static size_t KB = 8;

  714.     constexpr static size_t NB = 8;

  715.     constexpr static size_t CALCBLK = 4;

  716. 

  717.     megdnn_assert(!trA && !trB && M % MB == 0 && K % KB == 0);

  718. 

  719.     //! (m/8, k/8, 8, 8) * (k/8, n, 8) = (m/8, n, 8)

  720.     for (size_t m = 0; m < M; m += MB) {

  721.         dt_int32* output = C + (m / MB) * LDC;

  722.         const dt_int16* cur_B = B;

  723.         size_t n = 0;

  724.         for (; n + NB - 1 < N; n += NB) {

  725.             kern_8x8(A, cur_B, LDB, K, output);

  726.             cur_B += KB * NB;

  727.             output += MB * NB;

  728.         }

  729.         if (N - n >= 4) {

  730.             kern_8x4(A, cur_B, LDB, K, output);

  731.             cur_B += KB * CALCBLK;

  732.             output += MB * CALCBLK;

  733.             n += 4;

  734.         }

  735.         while (n < N) {

  736.             kern_8x1(A, cur_B, LDB, K, output);

  737.             cur_B += KB;

  738.             output += MB;

  739.             n++;

  740.         }

  741.         A += LDA;

  742.     }

  743. }

  744. 

  745. // vim: syntax=cpp.doxygen