wangwei
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 364afec033
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '364afec033'
${ noResults }
MegEngine/dnn/src/aarch64/matrix_mul/int8x8x16/kernel_mk4_8x8x8.h

1452 lines
57 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/aarch64/matrix_mul/int8x8x16/kernel_mk4_8x8x8.h

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or

  10.  * implied.

  11.  */

  12. 

  13. #include <inttypes.h>

  14. #include "src/aarch64/matrix_mul/asm/common.h"

  15. #include "src/arm_common/simd_macro/marm_neon.h"

  16. 

  17. namespace megdnn {

  18. namespace aarch64 {

  19. namespace matmul_mk4_8x8x8 {

  20. 

  21. 

  22. /**

  23.  * Overview of register layout:

  24.  *

  25.  * A 8x8 cell of Lhs is stored in 8bit in v16-v17

  26.  * B 8x8 cell of Rhs is stored in 8bit in v0-v15, v20-v23

  27.  * C 8x8 block of accumulators is stored in 16bit in v24-v31

  28.  *

  29.  *                     +---------------------------------+

  30.  *                     |  v0 ------------------------ v7 |

  31.  *                     |  v8 ------------------------ v15|

  32.  *                Rhs  +---------------------------------+

  33.  *    Lhs              |                                 |

  34.  *  +--------+ - - - - +---------------------------------+

  35.  *  | v16 |             | v24                            |

  36.  *  | v17 |             | v25                            |

  37.  *  | v16 |             | v26                            |

  38.  *  | v17 |             | v27                            |

  39.  *  | v16 |             | v28                            |

  40.  *  | v17 |             | v29                            |

  41.  *  | v16 |             | v30                            |

  42.  *  | v17 |             | v31                            |    

  43.  *  +--------+ - - - - +---------------------------------+

  44.  *

  45.  *                            Accumulator

  46.  */

  47. static void kern_8x8(const int8_t* packA, const int8_t* packB, int K,

  48.                      int16_t* output, int LDC, bool is_first_k, int m_remain,

  49.                      int n_remain) {

  50.     K /= 8;

  51.     LDC = LDC * sizeof(int16_t);

  52.     const int8_t* a_ptr = packB;//packA;

  53.     const int8_t* b_ptr = packA;//packB;

  54. // clang-format off

  55. #define LOAD_C_8 \

  56.     "ld1 {v0.8h}, [x0], #16\n"     \

  57.     "ld1 {v1.8h}, [x0], #16\n"     \

  58.     "ld1 {v2.8h}, [x0], #16\n"     \

  59.     "ld1 {v3.8h}, [x0], #16\n"     \

  60.     "ld1 {v4.8h}, [x1], #16\n"     \

  61.     "ld1 {v5.8h}, [x1], #16\n"     \

  62.     "ld1 {v6.8h}, [x1], #16\n"     \

  63.     "ld1 {v7.8h}, [x1], #16\n"     \

  64. 

  65. 

  66. #define STORE_C_8 \

  67.     "st1 {v0.8h}, [x0], #16\n"     \

  68.     "st1 {v1.8h}, [x0], #16\n"     \

  69.     "st1 {v2.8h}, [x0], #16\n"     \

  70.     "st1 {v3.8h}, [x0], #16\n"     \

  71.     "st1 {v4.8h}, [x1], #16\n"     \

  72.     "st1 {v5.8h}, [x1], #16\n"     \

  73.     "st1 {v6.8h}, [x1], #16\n"     \

  74.     "st1 {v7.8h}, [x1], #16\n"     \

  75. 

  76.     register int16_t* outptr asm("x0") = output;

  77.     asm volatile(

  78.             "add x1, x0, %x[LDC]\n"

  79. 

  80.             "eor v24.16b, v24.16b, v24.16b\n"

  81.             "PRFM PLDL1KEEP, [%[a_ptr], #512]\n"

  82.             "eor v25.16b, v25.16b, v25.16b\n"

  83.             "PRFM PLDL1KEEP, [%[b_ptr], #512]\n"

  84.             "eor v26.16b, v26.16b, v26.16b\n"

  85.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  86.             "eor v27.16b, v27.16b, v27.16b\n"

  87.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  88.             "eor v28.16b, v28.16b, v28.16b\n"

  89.             "eor v29.16b, v29.16b, v29.16b\n"

  90.             "eor v30.16b, v30.16b, v30.16b\n"

  91.             "eor v31.16b, v31.16b, v31.16b\n"

  92.             // General loop.

  93.             "1:\n"

  94.             "dup v0.8b,v20.b[0]\n"

  95.             "ld1  {v22.16b}, [%[a_ptr]],#16\n"

  96.             "dup v1.8b,v20.b[1]\n"

  97.             "ld1  {v23.16b}, [%[a_ptr]],#16\n"

  98.             "dup v2.8b,v20.b[2]\n"

  99.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  100.             "dup v3.8b,v20.b[3]\n"

  101.             "dup v4.8b,v20.b[4]\n"

  102.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  103.             "dup v5.8b,v20.b[5]\n"

  104.             "dup v6.8b,v20.b[6]\n"

  105.             "dup v7.8b,v20.b[7]\n"

  106.             

  107. 

  108.             "dup v8.8b,v20.b[8]\n"

  109.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  110.             "dup v9.8b,v20.b[9]\n"

  111.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  112.             "dup v10.8b,v20.b[10]\n"

  113.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  114.             "dup v11.8b,v20.b[11]\n"

  115.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  116.             "dup v12.8b,v20.b[12]\n"

  117.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  118.             "dup v13.8b,v20.b[13]\n"

  119.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  120.             "dup v14.8b,v20.b[14]\n"

  121.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  122.             "dup v15.8b,v20.b[15]\n"

  123.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  124. 

  125.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  126. 

  127.             "dup v0.8b,v21.b[0]\n"

  128.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  129.             "dup v1.8b,v21.b[1]\n"

  130.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  131.             "dup v2.8b,v21.b[2]\n"

  132.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  133.             "dup v3.8b,v21.b[3]\n"

  134.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  135.             "dup v4.8b,v21.b[4]\n"

  136.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  137.             "dup v5.8b,v21.b[5]\n"

  138.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  139.             "dup v6.8b,v21.b[6]\n"

  140.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  141.             "dup v7.8b,v21.b[7]\n"

  142.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  143. 

  144.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  145. 

  146.             "dup v8.8b,v21.b[8]\n"

  147.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  148.             "dup v9.8b,v21.b[9]\n"

  149.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  150.             "dup v10.8b,v21.b[10]\n"

  151.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  152.             "dup v11.8b,v21.b[11]\n"

  153.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  154.             "dup v12.8b,v21.b[12]\n"

  155.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  156.             "dup v13.8b,v21.b[13]\n"

  157.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  158.             "dup v14.8b,v21.b[14]\n"

  159.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  160.             "dup v15.8b,v21.b[15]\n"

  161.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  162. 

  163.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  164.             "dup v0.8b,v22.b[0]\n"

  165.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  166.             "dup v1.8b,v22.b[1]\n"

  167.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  168.             "dup v2.8b,v22.b[2]\n"

  169.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  170.             "dup v3.8b,v22.b[3]\n"

  171.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  172.             "dup v4.8b,v22.b[4]\n"

  173.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  174.             "dup v5.8b,v22.b[5]\n"

  175.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  176.             "dup v6.8b,v22.b[6]\n"

  177.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  178.             "dup v7.8b,v22.b[7]\n"

  179.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  180. 

  181.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  182. 

  183.             "dup v8.8b,v22.b[8]\n"

  184.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  185.             "dup v9.8b,v22.b[9]\n"

  186.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  187.             "dup v10.8b,v22.b[10]\n"

  188.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  189.             "dup v11.8b,v22.b[11]\n"

  190.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  191.             "dup v12.8b,v22.b[12]\n"

  192.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  193.             "dup v13.8b,v22.b[13]\n"

  194.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  195.             "dup v14.8b,v22.b[14]\n"

  196.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  197.             "dup v15.8b,v22.b[15]\n"

  198.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  199. 

  200.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  201.             "dup v0.8b,v23.b[0]\n"

  202.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  203.             "dup v1.8b,v23.b[1]\n"

  204.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  205.             "dup v2.8b,v23.b[2]\n"

  206.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  207.             "dup v3.8b,v23.b[3]\n"

  208.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  209.             "dup v4.8b,v23.b[4]\n"

  210.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  211.             "dup v5.8b,v23.b[5]\n"

  212.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  213.             "dup v6.8b,v23.b[6]\n"

  214.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  215.             "dup v7.8b,v23.b[7]\n"

  216.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  217. 

  218.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  219.             "dup v8.8b,v23.b[8]\n"

  220.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  221.             "dup v9.8b,v23.b[9]\n"

  222.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  223.             "dup v10.8b,v23.b[10]\n"

  224.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  225.             "dup v11.8b,v23.b[11]\n"

  226.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  227.             "dup v12.8b,v23.b[12]\n"

  228.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  229.             "dup v13.8b,v23.b[13]\n"

  230.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  231.             "dup v14.8b,v23.b[14]\n"

  232.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  233.             "dup v15.8b,v23.b[15]\n"

  234.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  235. 

  236.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  237.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  238.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  239.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  240.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  241.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  242.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  243.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  244.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  245.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  246. 

  247.             "subs %w[K], %w[K], #1\n"

  248.             "cbnz %w[K], 1b\n"

  249. 

  250.             "cmp %w[is_first_k], #1\n"

  251.             "beq 2f\n" LOAD_C_8

  252.             "b 3f                         \n"

  253.             "2:                           \n"

  254.             "eor v0.16b, v0.16b, v0.16b\n"

  255.             "eor v1.16b, v1.16b, v1.16b\n"

  256.             "eor v2.16b, v2.16b, v2.16b\n"

  257.             "eor v3.16b, v3.16b, v3.16b\n"

  258.             "eor v4.16b, v4.16b, v4.16b\n"

  259.             "eor v5.16b, v5.16b, v5.16b\n"

  260.             "eor v6.16b, v6.16b, v6.16b\n"

  261.             "eor v7.16b, v7.16b, v7.16b\n"

  262.             "3:\n"

  263.             "zip1 v8.2d,   v24.2d,  v25.2d\n"

  264.             "zip2 v9.2d,   v24.2d,  v25.2d\n"

  265.             "zip1 v10.2d,  v26.2d,  v27.2d\n"

  266.             "zip2 v11.2d,  v26.2d,  v27.2d\n"

  267.             "zip1 v12.2d,  v28.2d,  v29.2d\n"

  268.             "zip2 v13.2d,  v28.2d,  v29.2d\n"

  269.             "zip1 v14.2d,  v30.2d,  v31.2d\n"

  270.             "zip2 v15.2d,  v30.2d,  v31.2d\n"

  271.             "add v0.8h,    v0.8h,    v8.8h\n"

  272.             "add v1.8h,    v1.8h,   v10.8h\n"

  273.             "add v2.8h,    v2.8h,   v12.8h\n"

  274.             "add v3.8h,    v3.8h,   v14.8h\n"

  275.             "add v4.8h,    v4.8h,    v9.8h\n"

  276.             "add v5.8h,    v5.8h,   v11.8h\n"

  277.             "add v6.8h,    v6.8h,   v13.8h\n"

  278.             "add v7.8h,    v7.8h,   v15.8h\n"

  279. 

  280.             // Store back into memory

  281.             STORE_C_8

  282. 

  283.             :

  284.             [ a_ptr ] "+r"(a_ptr), [ b_ptr ] "+r"(b_ptr),

  285.             [ is_first_k ] "+r"(is_first_k), [ K ] "+r"(K), [ LDC ] "+r"(LDC),

  286.             [ outptr ] "+r"(outptr), [ m_remain ] "+r"(m_remain),

  287.             [ n_remain ] "+r"(n_remain)

  288.             :

  289.             : "cc", "memory", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",

  290.               "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  291.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",

  292.               "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",

  293.               "v29", "v30", "v31");

  294. // clang-format on

  295. }

  296. 

  297. static void kern_8x8_remain(const int8_t* packA, const int8_t* packB, int K,

  298.                      int16_t* output, int LDC, bool is_first_k, int m_remain,

  299.                      int n_remain) {

  300.     K /= 8;

  301.     LDC = LDC * sizeof(int16_t);

  302.     const int8_t* a_ptr = packB;

  303.     const int8_t* b_ptr = packA;

  304. //  clang-format off

  305.     register int16_t* outptr asm("x0") = output;

  306.     asm volatile(

  307.             "add x1, x0, %x[LDC]\n"

  308. 

  309.             "eor v24.16b, v24.16b, v24.16b\n"

  310.             "eor v25.16b, v25.16b, v25.16b\n"

  311.             "eor v26.16b, v26.16b, v26.16b\n"

  312.             "eor v27.16b, v27.16b, v27.16b\n"

  313.             "eor v28.16b, v28.16b, v28.16b\n"

  314.             "eor v29.16b, v29.16b, v29.16b\n"

  315.             "eor v30.16b, v30.16b, v30.16b\n"

  316.             "eor v31.16b, v31.16b, v31.16b\n"

  317.             // General loop.

  318.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  319.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  320.             "PRFM PLDL1KEEP, [%[a_ptr], #512]\n"

  321.             "PRFM PLDL1KEEP, [%[b_ptr], #512]\n"

  322.             "1:\n"

  323.             "dup v0.8b,v20.b[0]\n"

  324.             "ld1  {v22.16b}, [%[a_ptr]],#16\n"

  325.             "dup v1.8b,v20.b[1]\n"

  326.             "ld1  {v23.16b}, [%[a_ptr]],#16\n"

  327.             "dup v2.8b,v20.b[2]\n"

  328.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  329.             "dup v3.8b,v20.b[3]\n"

  330.             "dup v4.8b,v20.b[4]\n"

  331.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  332.             "dup v5.8b,v20.b[5]\n"

  333.             "dup v6.8b,v20.b[6]\n"

  334.             "dup v7.8b,v20.b[7]\n"

  335. 

  336.             "dup v8.8b,v20.b[8]\n"

  337.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  338.             "dup v9.8b,v20.b[9]\n"

  339.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  340.             "dup v10.8b,v20.b[10]\n"

  341.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  342.             "dup v11.8b,v20.b[11]\n"

  343.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  344.             "dup v12.8b,v20.b[12]\n"

  345.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  346.             "dup v13.8b,v20.b[13]\n"

  347.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  348.             "dup v14.8b,v20.b[14]\n"

  349.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  350.             "dup v15.8b,v20.b[15]\n"

  351.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  352. 

  353.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  354. 

  355.             "dup v0.8b,v21.b[0]\n"

  356.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  357.             "dup v1.8b,v21.b[1]\n"

  358.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  359.             "dup v2.8b,v21.b[2]\n"

  360.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  361.             "dup v3.8b,v21.b[3]\n"

  362.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  363.             "dup v4.8b,v21.b[4]\n"

  364.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  365.             "dup v5.8b,v21.b[5]\n"

  366.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  367.             "dup v6.8b,v21.b[6]\n"

  368.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  369.             "dup v7.8b,v21.b[7]\n"

  370.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  371. 

  372.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  373. 

  374.             "dup v8.8b,v21.b[8]\n"

  375.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  376.             "dup v9.8b,v21.b[9]\n"

  377.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  378.             "dup v10.8b,v21.b[10]\n"

  379.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  380.             "dup v11.8b,v21.b[11]\n"

  381.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  382.             "dup v12.8b,v21.b[12]\n"

  383.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  384.             "dup v13.8b,v21.b[13]\n"

  385.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  386.             "dup v14.8b,v21.b[14]\n"

  387.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  388.             "dup v15.8b,v21.b[15]\n"

  389.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  390. 

  391.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  392.             "dup v0.8b,v22.b[0]\n"

  393.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  394.             "dup v1.8b,v22.b[1]\n"

  395.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  396.             "dup v2.8b,v22.b[2]\n"

  397.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  398.             "dup v3.8b,v22.b[3]\n"

  399.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  400.             "dup v4.8b,v22.b[4]\n"

  401.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  402.             "dup v5.8b,v22.b[5]\n"

  403.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  404.             "dup v6.8b,v22.b[6]\n"

  405.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  406.             "dup v7.8b,v22.b[7]\n"

  407.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  408. 

  409.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  410. 

  411.             "dup v8.8b,v22.b[8]\n"

  412.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  413.             "dup v9.8b,v22.b[9]\n"

  414.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  415.             "dup v10.8b,v22.b[10]\n"

  416.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  417.             "dup v11.8b,v22.b[11]\n"

  418.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  419.             "dup v12.8b,v22.b[12]\n"

  420.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  421.             "dup v13.8b,v22.b[13]\n"

  422.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  423.             "dup v14.8b,v22.b[14]\n"

  424.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  425.             "dup v15.8b,v22.b[15]\n"

  426.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  427. 

  428.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  429.             "dup v0.8b,v23.b[0]\n"

  430.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  431.             "dup v1.8b,v23.b[1]\n"

  432.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  433.             "dup v2.8b,v23.b[2]\n"

  434.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  435.             "dup v3.8b,v23.b[3]\n"

  436.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  437.             "dup v4.8b,v23.b[4]\n"

  438.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  439.             "dup v5.8b,v23.b[5]\n"

  440.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  441.             "dup v6.8b,v23.b[6]\n"

  442.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  443.             "dup v7.8b,v23.b[7]\n"

  444.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  445. 

  446.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  447.             "dup v8.8b,v23.b[8]\n"

  448.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  449.             "dup v9.8b,v23.b[9]\n"

  450.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  451.             "dup v10.8b,v23.b[10]\n"

  452.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  453.             "dup v11.8b,v23.b[11]\n"

  454.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  455.             "dup v12.8b,v23.b[12]\n"

  456.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  457.             "dup v13.8b,v23.b[13]\n"

  458.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  459.             "dup v14.8b,v23.b[14]\n"

  460.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  461.             "dup v15.8b,v23.b[15]\n"

  462.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  463. 

  464.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  465.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  466.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  467.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  468.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  469.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  470.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  471.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  472.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  473.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  474. 

  475.             "subs %w[K], %w[K], #1\n"

  476.             "cbnz %w[K], 1b\n"

  477. 

  478.             "cmp %w[is_first_k], #1\n"

  479.             "beq 2f\n" 

  480.             "cmp  %x[m_remain], #8     \n"

  481.             "beq  8f                   \n"

  482.             "cmp  %x[m_remain], #4     \n"

  483.             "beq  9f                   \n"

  484.             "8:                        \n"

  485.             "cmp %x[n_remain],       #8\n"

  486.             "beq  200f                 \n"

  487.             "cmp %x[n_remain],       #7\n"

  488.             "beq  201f                 \n"

  489.             "cmp %x[n_remain],       #6\n"

  490.             "beq  202f                 \n"

  491.             "cmp %x[n_remain],       #5\n"

  492.             "beq  203f                 \n"

  493.             "cmp %x[n_remain],       #4\n"

  494.             "beq  204f                 \n"

  495.             "cmp %x[n_remain],       #3\n"

  496.             "beq  205f                 \n"

  497.             "cmp %x[n_remain],       #2\n"

  498.             "beq  206f                 \n"

  499.             "cmp %x[n_remain],       #1\n"

  500.             "beq  207f                 \n"

  501.             "200:                      \n"

  502.             "ld1 {v0.8h}, [x0], #16\n"

  503.             "ld1 {v1.8h}, [x0], #16\n"

  504.             "ld1 {v2.8h}, [x0], #16\n"

  505.             "ld1 {v3.8h}, [x0], #16\n"

  506.             "ld1 {v4.8h}, [x1], #16\n"

  507.             "ld1 {v5.8h}, [x1], #16\n"

  508.             "ld1 {v6.8h}, [x1], #16\n"

  509.             "ld1 {v7.8h}, [x1], #16\n"

  510.             "b 3f                   \n"

  511.             "201:                      \n"

  512.             "ld1 {v0.8h}, [x0], #16\n"

  513.             "ld1 {v1.8h}, [x0], #16\n"

  514.             "ld1 {v2.8h}, [x0], #16\n"

  515.             "ld1 {v3.d}[0], [x0], #8\n"

  516.             "ld1 {v4.8h}, [x1], #16\n"

  517.             "ld1 {v5.8h}, [x1], #16\n"

  518.             "ld1 {v6.8h}, [x1], #16\n"

  519.             "ld1 {v7.d}[0], [x1], #8\n"

  520.             "b 3f                   \n"

  521.             "202:                      \n"

  522.             "ld1 {v0.8h}, [x0], #16\n"

  523.             "ld1 {v1.8h}, [x0], #16\n"

  524.             "ld1 {v2.8h}, [x0], #16\n"

  525.             "ld1 {v4.8h}, [x1], #16\n"

  526.             "ld1 {v5.8h}, [x1], #16\n"

  527.             "ld1 {v6.8h}, [x1], #16\n"

  528.             "b 3f                   \n"

  529.             "203:                      \n"

  530.             "ld1 {v0.8h}, [x0], #16\n"

  531.             "ld1 {v1.8h}, [x0], #16\n"

  532.             "ld1 {v2.d}[0], [x0], #8\n"

  533.             "ld1 {v4.8h}, [x1], #16\n"

  534.             "ld1 {v5.8h}, [x1], #16\n"

  535.             "ld1 {v6.d}[0], [x1], #8\n"

  536.             "b 3f                   \n"

  537.             "204:                      \n"

  538.             "ld1 {v0.8h}, [x0], #16\n"

  539.             "ld1 {v1.8h}, [x0], #16\n"

  540.             "ld1 {v4.8h}, [x1], #16\n"

  541.             "ld1 {v5.8h}, [x1], #16\n"

  542.             "b 3f                   \n"

  543.             "205:                      \n"

  544.             "ld1 {v0.8h}, [x0], #16\n"

  545.             "ld1 {v1.d}[0], [x0], #8\n"

  546.             "ld1 {v4.8h}, [x1], #16\n"

  547.             "ld1 {v5.d}[0], [x1], #8\n"

  548.             "b 3f                   \n"

  549.             "206:                      \n"

  550.             "ld1 {v0.8h}, [x0], #16\n"

  551.             "ld1 {v4.8h}, [x1], #16\n"

  552.             "b 3f                   \n"

  553.             "207:                      \n"

  554.             "ld1 {v0.d}[0], [x0], #8\n"

  555.             "ld1 {v4.d}[0], [x1], #8\n"

  556.             "b 3f                   \n"

  557.             "9:                        \n"

  558.             "cmp %x[n_remain],       #8\n"

  559.             "beq  300f                 \n"

  560.             "cmp %x[n_remain],       #7\n"

  561.             "beq  301f                 \n"

  562.             "cmp %x[n_remain],       #6\n"

  563.             "beq  302f                 \n"

  564.             "cmp %x[n_remain],       #5\n"

  565.             "beq  303f                 \n"

  566.             "cmp %x[n_remain],       #4\n"

  567.             "beq  304f                 \n"

  568.             "cmp %x[n_remain],       #3\n"

  569.             "beq  305f                 \n"

  570.             "cmp %x[n_remain],       #2\n"

  571.             "beq  306f                 \n"

  572.             "cmp %x[n_remain],       #1\n"

  573.             "beq  307f                 \n"

  574.             "300:                      \n"

  575.             "ld1 {v0.8h}, [x0], #16\n"

  576.             "ld1 {v1.8h}, [x0], #16\n"

  577.             "ld1 {v2.8h}, [x0], #16\n"

  578.             "ld1 {v3.8h}, [x0], #16\n"

  579.             "b 3f                   \n"

  580.             "301:                      \n"

  581.             "ld1 {v0.8h}, [x0], #16\n"

  582.             "ld1 {v1.8h}, [x0], #16\n"

  583.             "ld1 {v2.8h}, [x0], #16\n"

  584.             "ld1 {v3.d}[0], [x0], #8\n"

  585.             "b 3f                   \n"

  586.             "302:                      \n"

  587.             "ld1 {v0.8h}, [x0], #16\n"

  588.             "ld1 {v1.8h}, [x0], #16\n"

  589.             "ld1 {v2.8h}, [x0], #16\n"

  590.             "b 3f                   \n"

  591.             "303:                      \n"

  592.             "ld1 {v0.8h}, [x0], #16\n"

  593.             "ld1 {v1.8h}, [x0], #16\n"

  594.             "ld1 {v2.d}[0], [x0], #8\n"

  595.             "b 3f                   \n"

  596.             "304:                      \n"

  597.             "ld1 {v0.8h}, [x0], #16\n"

  598.             "ld1 {v1.8h}, [x0], #16\n"

  599.             "b 3f                   \n"

  600.             "305:                      \n"

  601.             "ld1 {v0.8h}, [x0], #16\n"

  602.             "ld1 {v1.d}[0], [x0], #8\n"

  603.             "b 3f                   \n"

  604.             "306:                      \n"

  605.             "ld1 {v0.8h}, [x0], #16\n"

  606.             "b 3f                   \n"

  607.             "307:                      \n"

  608.             "ld1 {v0.d}[0], [x0], #8\n"

  609.             "b 3f                   \n"

  610.             "2:                           \n"

  611.             "eor v0.16b, v0.16b, v0.16b\n"

  612.             "eor v1.16b, v1.16b, v1.16b\n"

  613.             "eor v2.16b, v2.16b, v2.16b\n"

  614.             "eor v3.16b, v3.16b, v3.16b\n"

  615.             "eor v4.16b, v4.16b, v4.16b\n"

  616.             "eor v5.16b, v5.16b, v5.16b\n"

  617.             "eor v6.16b, v6.16b, v6.16b\n"

  618.             "eor v7.16b, v7.16b, v7.16b\n"

  619.             "3:\n"

  620.             "zip1 v8.2d,   v24.2d,  v25.2d\n"

  621.             "zip1 v10.2d,  v26.2d,  v27.2d\n"

  622.             "add v0.8h,  v0.8h,  v8.8h    \n"

  623.             "zip1 v12.2d,  v28.2d,  v29.2d\n"

  624.             "add v1.8h,  v1.8h,  v10.8h   \n"

  625.             "zip1 v14.2d,  v30.2d,  v31.2d\n"

  626.             "add v2.8h,  v2.8h,  v12.8h   \n"

  627.             "add v3.8h,  v3.8h,  v14.8h   \n"

  628.             "zip2 v9.2d,   v24.2d,  v25.2d\n"

  629.             "zip2 v11.2d,  v26.2d,  v27.2d  \n"

  630.             "add v4.8h,  v4.8h,  v9.8h      \n"

  631.             "zip2 v13.2d,  v28.2d,  v29.2d  \n"

  632.             "add v5.8h,  v5.8h,  v11.8h     \n"

  633.             "zip2 v15.2d,  v30.2d,  v31.2d  \n"

  634.             "add v6.8h,  v6.8h,  v13.8h     \n"

  635.             "add v7.8h,  v7.8h,  v15.8h     \n"

  636. //save to memory

  637.             "cmp  %x[m_remain], #8     \n"

  638.             "beq  4f                   \n"

  639.             "cmp  %x[m_remain], #4     \n"

  640.             "beq  5f                   \n"

  641.             "4:                        \n"

  642.             "cmp %x[n_remain],       #8\n"

  643.             "beq  100f                 \n"

  644.             "cmp %x[n_remain],       #7\n"

  645.             "beq  101f                 \n"

  646.             "cmp %x[n_remain],       #6\n"

  647.             "beq  102f                 \n"

  648.             "cmp %x[n_remain],       #5\n"

  649.             "beq  103f                 \n"

  650.             "cmp %x[n_remain],       #4\n"

  651.             "beq  104f                 \n"

  652.             "cmp %x[n_remain],       #3\n"

  653.             "beq  105f                 \n"

  654.             "cmp %x[n_remain],       #2\n"

  655.             "beq  106f                 \n"

  656.             "cmp %x[n_remain],       #1\n"

  657.             "beq  107f                 \n"

  658.             "100:                      \n"

  659.             "st1 {v0.8h}, [x0], #16\n"

  660.             "st1 {v1.8h}, [x0], #16\n"

  661.             "st1 {v2.8h}, [x0], #16\n"

  662.             "st1 {v3.8h}, [x0], #16\n"

  663.             "st1 {v4.8h}, [x1], #16\n"

  664.             "st1 {v5.8h}, [x1], #16\n"

  665.             "st1 {v6.8h}, [x1], #16\n"

  666.             "st1 {v7.8h}, [x1], #16\n"

  667.             "b 1000f                   \n"

  668.             "101:                      \n"

  669.             "st1 {v0.8h}, [x0], #16\n"

  670.             "st1 {v1.8h}, [x0], #16\n"

  671.             "st1 {v2.8h}, [x0], #16\n"

  672.             "st1 {v3.d}[0], [x0], #8\n"

  673.             "st1 {v4.8h}, [x1], #16\n"

  674.             "st1 {v5.8h}, [x1], #16\n"

  675.             "st1 {v6.8h}, [x1], #16\n"

  676.             "st1 {v7.d}[0], [x1], #8\n"

  677.             "b 1000f                   \n"

  678.             "102:                      \n"

  679.             "st1 {v0.8h}, [x0], #16\n"

  680.             "st1 {v1.8h}, [x0], #16\n"

  681.             "st1 {v2.8h}, [x0], #16\n"

  682.             "st1 {v4.8h}, [x1], #16\n"

  683.             "st1 {v5.8h}, [x1], #16\n"

  684.             "st1 {v6.8h}, [x1], #16\n"

  685.             "b 1000f                   \n"

  686.             "103:                      \n"

  687.             "st1 {v0.8h}, [x0], #16\n"

  688.             "st1 {v1.8h}, [x0], #16\n"

  689.             "st1 {v2.d}[0], [x0], #8\n"

  690.             "st1 {v4.8h}, [x1], #16\n"

  691.             "st1 {v5.8h}, [x1], #16\n"

  692.             "st1 {v6.d}[0], [x1], #8\n"

  693.             "b 1000f                   \n"

  694.             "104:                      \n"

  695.             "st1 {v0.8h}, [x0], #16\n"

  696.             "st1 {v1.8h}, [x0], #16\n"

  697.             "st1 {v4.8h}, [x1], #16\n"

  698.             "st1 {v5.8h}, [x1], #16\n"

  699.             "b 1000f                   \n"

  700.             "105:                      \n"

  701.             "st1 {v0.8h}, [x0], #16\n"

  702.             "st1 {v1.d}[0], [x0], #8\n"

  703.             "st1 {v4.8h}, [x1], #16\n"

  704.             "st1 {v5.d}[0], [x1], #8\n"

  705.             "b 1000f                   \n"

  706.             "106:                      \n"

  707.             "st1 {v0.8h}, [x0], #16\n"

  708.             "st1 {v4.8h}, [x1], #16\n"

  709.             "b 1000f                   \n"

  710.             "107:                      \n"

  711.             "st1 {v0.d}[0], [x0], #8\n"

  712.             "st1 {v4.d}[0], [x1], #8\n"

  713.             "b 1000f                   \n"

  714.             "5:                        \n"

  715.             "cmp %x[n_remain],       #8\n"

  716.             "beq  200f                 \n"

  717.             "cmp %x[n_remain],       #7\n"

  718.             "beq  201f                 \n"

  719.             "cmp %x[n_remain],       #6\n"

  720.             "beq  202f                 \n"

  721.             "cmp %x[n_remain],       #5\n"

  722.             "beq  203f                 \n"

  723.             "cmp %x[n_remain],       #4\n"

  724.             "beq  204f                 \n"

  725.             "cmp %x[n_remain],       #3\n"

  726.             "beq  205f                 \n"

  727.             "cmp %x[n_remain],       #2\n"

  728.             "beq  206f                 \n"

  729.             "cmp %x[n_remain],       #1\n"

  730.             "beq  207f                 \n"

  731.             "200:                      \n"

  732.             "st1 {v0.8h}, [x0], #16\n"

  733.             "st1 {v1.8h}, [x0], #16\n"

  734.             "st1 {v2.8h}, [x0], #16\n"

  735.             "st1 {v3.8h}, [x0], #16\n"

  736.             "b 1000f                   \n"

  737.             "201:                      \n"

  738.             "st1 {v0.8h}, [x0], #16\n"

  739.             "st1 {v1.8h}, [x0], #16\n"

  740.             "st1 {v2.8h}, [x0], #16\n"

  741.             "st1 {v3.d}[0], [x0], #8\n"

  742.             "b 1000f                   \n"

  743.             "202:                      \n"

  744.             "st1 {v0.8h}, [x0], #16\n"

  745.             "st1 {v1.8h}, [x0], #16\n"

  746.             "st1 {v2.8h}, [x0], #16\n"

  747.             "b 1000f                   \n"

  748.             "203:                      \n"

  749.             "st1 {v0.8h}, [x0], #16\n"

  750.             "st1 {v1.8h}, [x0], #16\n"

  751.             "st1 {v2.d}[0], [x0], #8\n"

  752.             "b 1000f                   \n"

  753.             "204:                      \n"

  754.             "st1 {v0.8h}, [x0], #16\n"

  755.             "st1 {v1.8h}, [x0], #16\n"

  756.             "b 1000f                   \n"

  757.             "205:                      \n"

  758.             "st1 {v0.8h}, [x0], #16\n"

  759.             "st1 {v1.d}[0], [x0], #8\n"

  760.             "b 1000f                   \n"

  761.             "206:                      \n"

  762.             "st1 {v0.8h}, [x0], #16\n"

  763.             "b 1000f                   \n"

  764.             "207:                      \n"

  765.             "st1 {v0.d}[0], [x0], #8\n"

  766.             "b 1000f                   \n"

  767. 

  768.             "1000:                     \n"

  769.             :

  770.             [ a_ptr ] "+r"(a_ptr), [ b_ptr ] "+r"(b_ptr),

  771.             [ is_first_k ] "+r"(is_first_k), [ K ] "+r"(K), [ LDC ] "+r"(LDC),

  772.             [ outptr ] "+r"(outptr), [ m_remain ] "+r"(m_remain),

  773.             [ n_remain ] "+r"(n_remain)

  774.             :

  775.             : "cc", "memory", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",

  776.               "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  777.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",

  778.               "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",

  779.               "v29", "v30", "v31");

  780. // clang-format on

  781. 

  782. #undef LOAD_C_8

  783. #undef STORE_C_8

  784. }

  785. 

  786. 

  787. static void kern_4x8(const int8_t* packA, const int8_t* packB, int K,

  788.                      int16_t* output, int LDC, bool is_first_k, int m_remain,

  789.                      int n_remain) {

  790.     K /= 8;

  791.     LDC = LDC * sizeof(int16_t);

  792.     const int8_t* a_ptr = packB;//packA;

  793.     const int8_t* b_ptr = packA;//packB;

  794. // clang-format off

  795. #define LOAD_C_4 \

  796.     "ld1 {v0.8h}, [x0], #16\n"     \

  797.     "ld1 {v1.8h}, [x0], #16\n"     \

  798.     "ld1 {v2.8h}, [x0], #16\n"     \

  799.     "ld1 {v3.8h}, [x0], #16\n"     \

  800. 

  801. 

  802. #define STORE_C_4 \

  803.     "st1 {v0.8h}, [x0], #16\n"     \

  804.     "st1 {v1.8h}, [x0], #16\n"     \

  805.     "st1 {v2.8h}, [x0], #16\n"     \

  806.     "st1 {v3.8h}, [x0], #16\n"     \

  807. 

  808.     register int16_t* outptr asm("x0") = output;

  809.     asm volatile(

  810. 

  811.             "eor v24.16b, v24.16b, v24.16b\n"

  812.             "eor v25.16b, v25.16b, v25.16b\n"

  813.             "eor v26.16b, v26.16b, v26.16b\n"

  814.             "eor v27.16b, v27.16b, v27.16b\n"

  815.             "eor v28.16b, v28.16b, v28.16b\n"

  816.             "eor v29.16b, v29.16b, v29.16b\n"

  817.             "eor v30.16b, v30.16b, v30.16b\n"

  818.             "eor v31.16b, v31.16b, v31.16b\n"

  819.             // General loop.

  820.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  821.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  822.             "PRFM PLDL1KEEP, [%[a_ptr], #512]\n"

  823.             "PRFM PLDL1KEEP, [%[b_ptr], #512]\n"

  824.             "1:\n"

  825.             "dup v0.8b,v20.b[0]\n"

  826.             "ld1  {v22.16b}, [%[a_ptr]],#16\n"

  827.             "dup v1.8b,v20.b[1]\n"

  828.             "ld1  {v23.16b}, [%[a_ptr]],#16\n"

  829.             "dup v2.8b,v20.b[2]\n"

  830.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  831.             "dup v3.8b,v20.b[3]\n"

  832.             "dup v4.8b,v20.b[4]\n"

  833.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  834.             "dup v5.8b,v20.b[5]\n"

  835.             "dup v6.8b,v20.b[6]\n"

  836.             "dup v7.8b,v20.b[7]\n"

  837.             

  838. 

  839.             "dup v8.8b,v20.b[8]\n"

  840.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  841.             "dup v9.8b,v20.b[9]\n"

  842.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  843.             "dup v10.8b,v20.b[10]\n"

  844.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  845.             "dup v11.8b,v20.b[11]\n"

  846.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  847.             "dup v12.8b,v20.b[12]\n"

  848.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  849.             "dup v13.8b,v20.b[13]\n"

  850.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  851.             "dup v14.8b,v20.b[14]\n"

  852.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  853.             "dup v15.8b,v20.b[15]\n"

  854.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  855. 

  856.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  857. 

  858.             "dup v0.8b,v21.b[0]\n"

  859.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  860.             "dup v1.8b,v21.b[1]\n"

  861.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  862.             "dup v2.8b,v21.b[2]\n"

  863.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  864.             "dup v3.8b,v21.b[3]\n"

  865.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  866.             "dup v4.8b,v21.b[4]\n"

  867.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  868.             "dup v5.8b,v21.b[5]\n"

  869.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  870.             "dup v6.8b,v21.b[6]\n"

  871.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  872.             "dup v7.8b,v21.b[7]\n"

  873.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  874. 

  875.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  876. 

  877.             "dup v8.8b,v21.b[8]\n"

  878.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  879.             "dup v9.8b,v21.b[9]\n"

  880.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  881.             "dup v10.8b,v21.b[10]\n"

  882.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  883.             "dup v11.8b,v21.b[11]\n"

  884.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  885.             "dup v12.8b,v21.b[12]\n"

  886.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  887.             "dup v13.8b,v21.b[13]\n"

  888.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  889.             "dup v14.8b,v21.b[14]\n"

  890.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  891.             "dup v15.8b,v21.b[15]\n"

  892.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  893. 

  894.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  895.             "dup v0.8b,v22.b[0]\n"

  896.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  897.             "dup v1.8b,v22.b[1]\n"

  898.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  899.             "dup v2.8b,v22.b[2]\n"

  900.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  901.             "dup v3.8b,v22.b[3]\n"

  902.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  903.             "dup v4.8b,v22.b[4]\n"

  904.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  905.             "dup v5.8b,v22.b[5]\n"

  906.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  907.             "dup v6.8b,v22.b[6]\n"

  908.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  909.             "dup v7.8b,v22.b[7]\n"

  910.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  911. 

  912.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  913. 

  914.             "dup v8.8b,v22.b[8]\n"

  915.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  916.             "dup v9.8b,v22.b[9]\n"

  917.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  918.             "dup v10.8b,v22.b[10]\n"

  919.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  920.             "dup v11.8b,v22.b[11]\n"

  921.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  922.             "dup v12.8b,v22.b[12]\n"

  923.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  924.             "dup v13.8b,v22.b[13]\n"

  925.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  926.             "dup v14.8b,v22.b[14]\n"

  927.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  928.             "dup v15.8b,v22.b[15]\n"

  929.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  930. 

  931.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  932.             "dup v0.8b,v23.b[0]\n"

  933.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  934.             "dup v1.8b,v23.b[1]\n"

  935.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  936.             "dup v2.8b,v23.b[2]\n"

  937.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  938.             "dup v3.8b,v23.b[3]\n"

  939.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  940.             "dup v4.8b,v23.b[4]\n"

  941.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  942.             "dup v5.8b,v23.b[5]\n"

  943.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  944.             "dup v6.8b,v23.b[6]\n"

  945.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  946.             "dup v7.8b,v23.b[7]\n"

  947.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  948. 

  949.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  950.             "dup v8.8b,v23.b[8]\n"

  951.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  952.             "dup v9.8b,v23.b[9]\n"

  953.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  954.             "dup v10.8b,v23.b[10]\n"

  955.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  956.             "dup v11.8b,v23.b[11]\n"

  957.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  958.             "dup v12.8b,v23.b[12]\n"

  959.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  960.             "dup v13.8b,v23.b[13]\n"

  961.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  962.             "dup v14.8b,v23.b[14]\n"

  963.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  964.             "dup v15.8b,v23.b[15]\n"

  965.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  966. 

  967.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  968.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  969.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  970.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  971.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  972.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  973.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  974.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  975.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  976.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  977. 

  978.             "subs %w[K], %w[K], #1\n"

  979.             "cbnz %w[K], 1b\n"

  980. 

  981.             "cmp %w[is_first_k], #1\n"

  982.             "beq 2f\n" LOAD_C_4

  983.             "b 3f                         \n"

  984.             "2:                           \n"

  985.             "eor v0.16b, v0.16b, v0.16b\n"

  986.             "eor v1.16b, v1.16b, v1.16b\n"

  987.             "eor v2.16b, v2.16b, v2.16b\n"

  988.             "eor v3.16b, v3.16b, v3.16b\n"

  989.             "eor v4.16b, v4.16b, v4.16b\n"

  990.             "eor v5.16b, v5.16b, v5.16b\n"

  991.             "eor v6.16b, v6.16b, v6.16b\n"

  992.             "eor v7.16b, v7.16b, v7.16b\n"

  993.             "3:\n"

  994.             "zip1 v8.2d,   v24.2d,  v25.2d\n"

  995.             "zip1 v10.2d,  v26.2d,  v27.2d\n"

  996.             "add v0.8h,  v0.8h,  v8.8h\n"

  997.             "zip1 v12.2d,  v28.2d,  v29.2d\n"

  998.             "add v1.8h,  v1.8h,  v10.8h\n"

  999.             "zip1 v14.2d,  v30.2d,  v31.2d\n"

  1000.             "add v2.8h,  v2.8h,  v12.8h\n"

  1001.             "add v3.8h,  v3.8h,  v14.8h\n"

  1002. 

  1003.             // Store back into memory

  1004.             STORE_C_4

  1005. 

  1006.             :

  1007.             [ a_ptr ] "+r"(a_ptr), [ b_ptr ] "+r"(b_ptr),

  1008.             [ is_first_k ] "+r"(is_first_k), [ K ] "+r"(K), [ LDC ] "+r"(LDC),

  1009.             [ outptr ] "+r"(outptr), [ m_remain ] "+r"(m_remain),

  1010.             [ n_remain ] "+r"(n_remain)

  1011.             :

  1012.             : "cc", "memory", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",

  1013.               "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  1014.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",

  1015.               "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",

  1016.               "v29", "v30", "v31");

  1017. // clang-format on

  1018. #undef LOAD_C_4

  1019. #undef STORE_C_4

  1020. }

  1021. static void kern_4x8_remain(const int8_t* packA, const int8_t* packB, int K,

  1022.                      int16_t* output, int LDC, bool is_first_k, int m_remain,

  1023.                      int n_remain) {

  1024.     K /= 8;

  1025.     LDC = LDC * sizeof(int16_t);

  1026.     const int8_t* a_ptr = packB;//packA;

  1027.     const int8_t* b_ptr = packA;//packB;

  1028. // clang-format off

  1029.     register int16_t* outptr asm("x0") = output;

  1030.     asm volatile(

  1031. 

  1032.             "eor v24.16b, v24.16b, v24.16b\n"

  1033.             "eor v25.16b, v25.16b, v25.16b\n"

  1034.             "eor v26.16b, v26.16b, v26.16b\n"

  1035.             "eor v27.16b, v27.16b, v27.16b\n"

  1036.             "eor v28.16b, v28.16b, v28.16b\n"

  1037.             "eor v29.16b, v29.16b, v29.16b\n"

  1038.             "eor v30.16b, v30.16b, v30.16b\n"

  1039.             "eor v31.16b, v31.16b, v31.16b\n"

  1040.             // General loop.

  1041.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  1042.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  1043.             "PRFM PLDL1KEEP, [%[a_ptr], #512]\n"

  1044.             "PRFM PLDL1KEEP, [%[b_ptr], #512]\n"

  1045.             "1:\n"

  1046.             "dup v0.8b,v20.b[0]\n"

  1047.             "ld1  {v22.16b}, [%[a_ptr]],#16\n"

  1048.             "dup v1.8b,v20.b[1]\n"

  1049.             "ld1  {v23.16b}, [%[a_ptr]],#16\n"

  1050.             "dup v2.8b,v20.b[2]\n"

  1051.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  1052.             "dup v3.8b,v20.b[3]\n"

  1053.             "dup v4.8b,v20.b[4]\n"

  1054.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  1055.             "dup v5.8b,v20.b[5]\n"

  1056.             "dup v6.8b,v20.b[6]\n"

  1057.             "dup v7.8b,v20.b[7]\n"

  1058. 

  1059.             "dup v8.8b,v20.b[8]\n"

  1060.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  1061.             "dup v9.8b,v20.b[9]\n"

  1062.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  1063.             "dup v10.8b,v20.b[10]\n"

  1064.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  1065.             "dup v11.8b,v20.b[11]\n"

  1066.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  1067.             "dup v12.8b,v20.b[12]\n"

  1068.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  1069.             "dup v13.8b,v20.b[13]\n"

  1070.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  1071.             "dup v14.8b,v20.b[14]\n"

  1072.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  1073.             "dup v15.8b,v20.b[15]\n"

  1074.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  1075. 

  1076.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  1077. 

  1078.             "dup v0.8b,v21.b[0]\n"

  1079.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  1080.             "dup v1.8b,v21.b[1]\n"

  1081.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  1082.             "dup v2.8b,v21.b[2]\n"

  1083.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  1084.             "dup v3.8b,v21.b[3]\n"

  1085.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  1086.             "dup v4.8b,v21.b[4]\n"

  1087.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  1088.             "dup v5.8b,v21.b[5]\n"

  1089.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  1090.             "dup v6.8b,v21.b[6]\n"

  1091.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  1092.             "dup v7.8b,v21.b[7]\n"

  1093.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  1094. 

  1095.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  1096. 

  1097.             "dup v8.8b,v21.b[8]\n"

  1098.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  1099.             "dup v9.8b,v21.b[9]\n"

  1100.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  1101.             "dup v10.8b,v21.b[10]\n"

  1102.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  1103.             "dup v11.8b,v21.b[11]\n"

  1104.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  1105.             "dup v12.8b,v21.b[12]\n"

  1106.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  1107.             "dup v13.8b,v21.b[13]\n"

  1108.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  1109.             "dup v14.8b,v21.b[14]\n"

  1110.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  1111.             "dup v15.8b,v21.b[15]\n"

  1112.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  1113. 

  1114.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  1115.             "dup v0.8b,v22.b[0]\n"

  1116.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  1117.             "dup v1.8b,v22.b[1]\n"

  1118.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  1119.             "dup v2.8b,v22.b[2]\n"

  1120.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  1121.             "dup v3.8b,v22.b[3]\n"

  1122.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  1123.             "dup v4.8b,v22.b[4]\n"

  1124.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  1125.             "dup v5.8b,v22.b[5]\n"

  1126.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  1127.             "dup v6.8b,v22.b[6]\n"

  1128.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  1129.             "dup v7.8b,v22.b[7]\n"

  1130.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  1131. 

  1132.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  1133. 

  1134.             "dup v8.8b,v22.b[8]\n"

  1135.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  1136.             "dup v9.8b,v22.b[9]\n"

  1137.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  1138.             "dup v10.8b,v22.b[10]\n"

  1139.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  1140.             "dup v11.8b,v22.b[11]\n"

  1141.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  1142.             "dup v12.8b,v22.b[12]\n"

  1143.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  1144.             "dup v13.8b,v22.b[13]\n"

  1145.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  1146.             "dup v14.8b,v22.b[14]\n"

  1147.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  1148.             "dup v15.8b,v22.b[15]\n"

  1149.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  1150. 

  1151.             "ld1  {v16.8b}, [%[b_ptr]], 8\n"

  1152.             "dup v0.8b,v23.b[0]\n"

  1153.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  1154.             "dup v1.8b,v23.b[1]\n"

  1155.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  1156.             "dup v2.8b,v23.b[2]\n"

  1157.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  1158.             "dup v3.8b,v23.b[3]\n"

  1159.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  1160.             "dup v4.8b,v23.b[4]\n"

  1161.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  1162.             "dup v5.8b,v23.b[5]\n"

  1163.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  1164.             "dup v6.8b,v23.b[6]\n"

  1165.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  1166.             "dup v7.8b,v23.b[7]\n"

  1167.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  1168. 

  1169.             "ld1  {v17.8b}, [%[b_ptr]], 8\n"

  1170.             "dup v8.8b,v23.b[8]\n"

  1171.             "smlal   v24.8h,  v0.8b,  v16.8b\n"

  1172.             "dup v9.8b,v23.b[9]\n"

  1173.             "smlal   v25.8h,  v1.8b,  v16.8b\n"

  1174.             "dup v10.8b,v23.b[10]\n"

  1175.             "smlal   v26.8h,  v2.8b,  v16.8b\n"

  1176.             "dup v11.8b,v23.b[11]\n"

  1177.             "smlal   v27.8h,  v3.8b,  v16.8b\n"

  1178.             "dup v12.8b,v23.b[12]\n"

  1179.             "smlal   v28.8h,  v4.8b,  v16.8b\n"

  1180.             "dup v13.8b,v23.b[13]\n"

  1181.             "smlal   v29.8h,  v5.8b,  v16.8b\n"

  1182.             "dup v14.8b,v23.b[14]\n"

  1183.             "smlal   v30.8h,  v6.8b,  v16.8b\n"

  1184.             "dup v15.8b,v23.b[15]\n"

  1185.             "smlal   v31.8h,  v7.8b,  v16.8b\n"

  1186. 

  1187.             "ld1  {v20.16b}, [%[a_ptr]],#16\n"

  1188.             "smlal   v24.8h,  v8.8b,   v17.8b\n"

  1189.             "smlal   v25.8h,  v9.8b,   v17.8b\n"

  1190.             "smlal   v26.8h,  v10.8b,  v17.8b\n"

  1191.             "smlal   v27.8h,  v11.8b,  v17.8b\n"

  1192.             "ld1  {v21.16b}, [%[a_ptr]],#16\n"

  1193.             "smlal   v28.8h,  v12.8b,  v17.8b\n"

  1194.             "smlal   v29.8h,  v13.8b,  v17.8b\n"

  1195.             "smlal   v30.8h,  v14.8b,  v17.8b\n"

  1196.             "smlal   v31.8h,  v15.8b,  v17.8b\n"

  1197. 

  1198.             "subs %w[K], %w[K], #1     \n"

  1199.             "cbnz %w[K], 1b            \n"

  1200.             "cmp %w[is_first_k], #1    \n"

  1201.             "beq 2f                    \n"

  1202.             "cmp %w[n_remain],#7       \n"

  1203.             "beq 200f                  \n"

  1204.             "cmp %w[n_remain],#6       \n"

  1205.             "beq 201f                  \n"

  1206.             "cmp %w[n_remain],#5       \n"

  1207.             "beq 202f                  \n"

  1208.             "cmp %w[n_remain],#4       \n"

  1209.             "beq 203f                  \n"

  1210.             "cmp %w[n_remain],#3       \n"

  1211.             "beq 204f                  \n"

  1212.             "cmp %w[n_remain],#2       \n"

  1213.             "beq 205f                  \n"

  1214.             "cmp %w[n_remain],#1       \n"

  1215.             "beq 206f                  \n"

  1216.             "200:                      \n"

  1217.             "ld1 {v0.8h}, [x0],#16     \n"

  1218.             "ld1 {v1.8h}, [x0],#16     \n"

  1219.             "ld1 {v2.8h}, [x0],#16     \n"

  1220.             "ld1 {v3.d}[0], [x0],#8    \n"

  1221.             "b 3f                      \n"

  1222.             "201:                      \n"

  1223.             "ld1 {v0.8h}, [x0],#16     \n"

  1224.             "ld1 {v1.8h}, [x0],#16     \n"

  1225.             "ld1 {v2.8h}, [x0],#16     \n"

  1226.             "b 3f                      \n"

  1227.             "202:                      \n"

  1228.             "ld1 {v0.8h}, [x0],#16     \n"

  1229.             "ld1 {v1.8h}, [x0],#16     \n"

  1230.             "ld1 {v2.d}[0], [x0],#8    \n"

  1231.             "b 3f                      \n"

  1232.             "203:                      \n"

  1233.             "ld1 {v0.8h}, [x0],#16     \n"

  1234.             "ld1 {v1.8h}, [x0],#16     \n"

  1235.             "b 3f                      \n"

  1236.             "204:                      \n"

  1237.             "ld1 {v0.8h}, [x0],#16     \n"

  1238.             "ld1 {v1.d}[0], [x0],#8    \n"

  1239.             "b 3f                      \n"

  1240.             "205:                      \n"

  1241.             "ld1 {v0.8h}, [x0],#16     \n"

  1242.             "b 3f                      \n"

  1243.             "206:                      \n"

  1244.             "ld1 {v0.d}[0], [x0],#8    \n"

  1245.             "b 3f                      \n"

  1246.             "2:                        \n"

  1247.             "eor v0.16b, v0.16b, v0.16b\n"

  1248.             "eor v1.16b, v1.16b, v1.16b\n"

  1249.             "eor v2.16b, v2.16b, v2.16b\n"

  1250.             "eor v3.16b, v3.16b, v3.16b\n"

  1251.             "eor v4.16b, v4.16b, v4.16b\n"

  1252.             "eor v5.16b, v5.16b, v5.16b\n"

  1253.             "eor v6.16b, v6.16b, v6.16b\n"

  1254.             "eor v7.16b, v7.16b, v7.16b\n"

  1255.             "3:                           \n"

  1256.             "zip1 v8.2d,  v24.2d, v25.2d\n"

  1257.             "zip1 v10.2d, v26.2d, v27.2d\n"

  1258.             "add  v0.8h,  v0.8h,  v8.8h \n"

  1259.             "zip1 v12.2d, v28.2d, v29.2d\n"

  1260.             "add  v1.8h,  v1.8h,  v10.8h\n"

  1261.             "zip1 v14.2d, v30.2d, v31.2d\n"

  1262.             "add  v2.8h,  v2.8h,  v12.8h\n"

  1263.             "add  v3.8h,  v3.8h,  v14.8h\n"

  1264. 

  1265.             // Store back into memory

  1266.             "cmp %w[n_remain],#7      \n"

  1267.             "beq 100f                 \n"

  1268.             "cmp %w[n_remain],#6      \n"

  1269.             "beq 101f                 \n"

  1270.             "cmp %w[n_remain],#5      \n"

  1271.             "beq 102f                 \n"

  1272.             "cmp %w[n_remain],#4      \n"

  1273.             "beq 103f                 \n"

  1274.             "cmp %w[n_remain],#3      \n"

  1275.             "beq 104f                 \n"

  1276.             "cmp %w[n_remain],#2      \n"

  1277.             "beq 105f                 \n"

  1278.             "cmp %w[n_remain],#1      \n"

  1279.             "beq 106f                 \n"

  1280.             "100:                     \n"

  1281.             "st1 {v0.8h}, [x0],#16    \n"

  1282.             "st1 {v1.8h}, [x0],#16    \n"

  1283.             "st1 {v2.8h}, [x0],#16    \n"

  1284.             "st1 {v3.d}[0], [x0],#8   \n"

  1285.             "b 1000f                  \n"

  1286.             "101:                     \n"

  1287.             "st1 {v0.8h}, [x0],#16    \n"

  1288.             "st1 {v1.8h}, [x0],#16    \n"

  1289.             "st1 {v2.8h}, [x0],#16    \n"

  1290.             "b 1000f                  \n"

  1291.             "102:                     \n"

  1292.             "st1 {v0.8h}, [x0],#16    \n"

  1293.             "st1 {v1.8h}, [x0],#16    \n"

  1294.             "st1 {v2.d}[0], [x0],#8   \n"

  1295.             "b 1000f                  \n"

  1296.             "103:                     \n"

  1297.             "st1 {v0.8h}, [x0],#16    \n"

  1298.             "st1 {v1.8h}, [x0],#16    \n"

  1299.             "b 1000f                  \n"

  1300.             "104:                     \n"

  1301.             "st1 {v0.8h}, [x0],#16    \n"

  1302.             "st1 {v1.d}[0], [x0],#8   \n"

  1303.             "b 1000f                  \n"

  1304.             "105:                     \n"

  1305.             "st1 {v0.8h}, [x0],#16    \n"

  1306.             "b 1000f                  \n"

  1307.             "106:                     \n"

  1308.             "st1 {v0.d}[0], [x0],#8   \n"

  1309.             "b 1000f                  \n"

  1310.             "1000:                    \n"

  1311.             :

  1312.             [ a_ptr ] "+r"(a_ptr), [ b_ptr ] "+r"(b_ptr),

  1313.             [ is_first_k ] "+r"(is_first_k), [ K ] "+r"(K), [ LDC ] "+r"(LDC),

  1314.             [ outptr ] "+r"(outptr), [ m_remain ] "+r"(m_remain),

  1315.             [ n_remain ] "+r"(n_remain)

  1316.             :

  1317.             : "cc", "memory", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",

  1318.               "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  1319.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",

  1320.               "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",

  1321.               "v29", "v30", "v31");

  1322. // clang-format on

  1323. #undef LOAD_C_4

  1324. #undef STORE_C_4

  1325. }

  1326. 

  1327. 

  1328. //! pack to icxoc

  1329. //! (M/4,K/4,4(K),4(M)) pack to (M/8,k/8,8(K_ic_0~3_ic_4~7),8(M_oc0~3_OC_4~7))

  1330. //! if M K is not times of 8,pack 0 instead 

  1331. static void gemm_s8x8x16_mk4_8x8x8_pack_A(dt_int8* outptr,

  1332.                                           const dt_int8* inptr, int ldin,

  1333.                                           int m0, int mmax, int k0, int kmax) {

  1334.     megdnn_assert(m0 % 4 == 0 && mmax % 4 == 0, "M must be time of 4");

  1335.     megdnn_assert(k0 % 4 == 0 && kmax % 4 == 0, "K must be time of 4");

  1336.     constexpr int pack_m = 8;

  1337.     constexpr int pack_k = 8;

  1338.     constexpr int pack_size = 4;

  1339.     int8_t tmpbuff0[pack_m * pack_size] = {0};

  1340.     int8_t tmpbuff1[pack_m * pack_size] = {0};

  1341.     int8_t zerobuff[pack_m * pack_size] = {0};

  1342.     const int m_size = mmax - m0;

  1343.     const int m_end = m_size / pack_m * pack_m + m0;

  1344.     int remain_m = mmax - m_end;

  1345. 

  1346.     for (int m_idx = m0; m_idx < m_end; m_idx += pack_m) {

  1347.         const int8_t* inptr0 = inptr + m_idx / pack_size * ldin + k0;

  1348.         const int8_t* inptr1 = inptr0 + ldin;

  1349.         prefetch_2x(inptr0);

  1350.         prefetch_2x(inptr1);

  1351.         int k_idx = k0;

  1352.         for ( ; k_idx + 7 < kmax; k_idx += pack_k) {

  1353.             interleave_8x8_mk4_b(inptr0,inptr1,outptr);

  1354.         }

  1355. 

  1356.         if (k_idx < kmax) {

  1357.             memcpy(tmpbuff0, inptr0, sizeof(int8_t) * (kmax - k_idx) * pack_size);

  1358.             memcpy(tmpbuff1, inptr1, sizeof(int8_t) * (kmax - k_idx) * pack_size);

  1359.             inptr0 = tmpbuff0;

  1360.             inptr1 = tmpbuff1;

  1361.             interleave_8x8_mk4_b(inptr0, inptr1, outptr);

  1362.         }

  1363.     }

  1364.     int m_idx = m_end;

  1365.     if (remain_m == 4) {

  1366.         const int8_t* inptr0 = inptr + m_idx / pack_size * ldin + k0;

  1367.         const int8_t* inptr1 = inptr0 + ldin;

  1368.         prefetch_2x(inptr0);

  1369.         prefetch_2x(inptr1);

  1370.         int k_idx = k0;

  1371.         for ( ; k_idx + 7 < kmax; k_idx += pack_k) {

  1372.             inptr1 = zerobuff;

  1373.             interleave_8x8_mk4_b(inptr0,inptr1,outptr);

  1374.         }

  1375. 

  1376.         if (k_idx < kmax) {

  1377.             memcpy(tmpbuff0, inptr0, sizeof(int8_t) * (kmax - k_idx) * pack_size);

  1378.             inptr0 = tmpbuff0;

  1379.             inptr1 = zerobuff;

  1380.             interleave_8x8_mk4_b(inptr0, inptr1, outptr);

  1381.         }

  1382.     }

  1383. }

  1384. //! pack to nxic

  1385. //! (K/4,N,4) pack to K/8,N,8(ic0~7) ,K is not times of 8 ,pack 0 instead.

  1386. static void gemm_s8x8x16_mk4_8x8x8_pack_B(dt_int8* out, const dt_int8* in,

  1387.                                           int ldin, int n0, int nmax, int k0,

  1388.                                           int kmax) {

  1389.     megdnn_assert(k0 % 4 == 0 && kmax % 4 == 0, "K must be time of 4");

  1390. 

  1391.     constexpr int pack_n = 8;

  1392.     constexpr int pack_k = 8;

  1393.     constexpr int pack_size = 4;

  1394.     int8_t tmpbuff0[pack_n * pack_size] = {0};

  1395.     int8_t tmpbuff1[pack_n * pack_size] = {0};

  1396.     int8_t zerobuff[pack_n * pack_size] = {0};

  1397.     const int ksize = round_up<int>((kmax - k0),8);

  1398.     const int nsize = nmax - n0;

  1399.     const int n_end = nsize / pack_n * pack_n + n0;

  1400.     const int remain_n = nsize % pack_n;

  1401.     int output_stride = ksize * pack_n;

  1402.     int8_t* outptr_base = out;

  1403.     int k_idx = k0;

  1404.     for ( ; k_idx + 7 < kmax; k_idx += pack_k) {

  1405.         const int8_t* inptr0 = in + k_idx / pack_size * ldin + n0 * pack_size;

  1406.         const int8_t* inptr1 = inptr0 + ldin;

  1407.         prefetch_3x(inptr0);

  1408.         prefetch_3x(inptr1);

  1409. 

  1410.         auto outptr = outptr_base;

  1411.         for (int n_idx = n0; n_idx < n_end; n_idx += pack_n) {

  1412.             transpose_8x8_mk4_b(inptr0, inptr1, outptr);

  1413.            outptr += output_stride;

  1414.         }

  1415.         if (remain_n > 0) {

  1416.             memcpy(tmpbuff0, inptr0, sizeof(int8_t) * remain_n * pack_size);

  1417.             memcpy(tmpbuff1, inptr1, sizeof(int8_t) * remain_n * pack_size);

  1418.             inptr0 = tmpbuff0;

  1419.             inptr1 = tmpbuff1;

  1420.             transpose_8x8_mk4_b(inptr0, inptr1, outptr);

  1421.             outptr += output_stride;

  1422.         }

  1423.         outptr_base += pack_n * pack_k;

  1424.     }

  1425.     

  1426.     if(k_idx < kmax){

  1427.         const int8_t* inptr0 = in + k_idx / pack_size * ldin + n0 * pack_size;

  1428.         const int8_t* inptr1 = nullptr;

  1429.         prefetch_3x(inptr0);

  1430.         auto outptr = outptr_base;

  1431.         for (int n_idx = n0; n_idx < n_end; n_idx += pack_n) {

  1432.             inptr1 = zerobuff;

  1433.             transpose_8x8_mk4_b(inptr0, inptr1, outptr);

  1434.             outptr += output_stride;

  1435.         }

  1436.         if (remain_n > 0) {

  1437.             memcpy(tmpbuff0, inptr0, sizeof(int8_t) * remain_n * pack_size);

  1438.             inptr1 = zerobuff;

  1439.             inptr0 = tmpbuff0;

  1440.             transpose_8x8_mk4_b(inptr0, inptr1, outptr);

  1441.             outptr += output_stride;

  1442.         }

  1443.         outptr_base += pack_n * pack_size;

  1444.     }

  1445. }

  1446. 

  1447. }  // namespace matmul_mk4_16x12x4_a53

  1448. }  // namespace aarch64

  1449. }  // namespace megdnn

  1450. 

  1451. // vim: syntax=cpp.doxygen