nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 2cfec9032b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2cfec9032b'
${ noResults }
ncnn/src/layer/arm/convolution_3x3_pack4_bf16s.h

4809 lines
227 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. static void conv3x3s1_winograd64_pack4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias, const Option& opt)

  16. {

  17.     int w = bottom_blob.w;

  18.     int h = bottom_blob.h;

  19.     int inch = bottom_blob.c;

  20.     size_t elemsize = bottom_blob.elemsize;

  21.     int elempack = bottom_blob.elempack;

  22. 

  23.     int outw = top_blob.w;

  24.     int outh = top_blob.h;

  25.     int outch = top_blob.c;

  26. 

  27.     // pad to 6n+2

  28.     Mat bottom_blob_bordered = bottom_blob;

  29. 

  30.     outw = (outw + 5) / 6 * 6;

  31.     outh = (outh + 5) / 6 * 6;

  32. 

  33.     w = outw + 2;

  34.     h = outh + 2;

  35.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt);

  36. 

  37.     const float* bias = _bias;

  38. 

  39.     // BEGIN transform input

  40.     Mat bottom_blob_tm;

  41.     {

  42.         int w_tm = outw / 6 * 8;

  43.         int h_tm = outh / 6 * 8;

  44. 

  45.         const int tiles = w_tm / 8 * h_tm / 8;

  46. 

  47.         //         bottom_blob_tm.create(tiles, 64, inch, elemsize, elempack, opt.workspace_allocator);

  48.         bottom_blob_tm.create(tiles, 64, inch, 4u * elempack, elempack, opt.workspace_allocator);

  49. 

  50.         //         const float itm[8][8] = {

  51.         //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  52.         //

  53.         //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  54.         //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  55.         //

  56.         //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  57.         //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  58.         //

  59.         //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  60.         //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  61.         //

  62.         //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  63.         //         };

  64. 

  65.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  66.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  67. 

  68.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  69.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  70. 

  71.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  72.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  73. 

  74.         // reuse r04 * 1.25

  75.         // reuse r03 * 2.5

  76.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  77.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  78. 

  79.         #pragma omp parallel for num_threads(opt.num_threads)

  80.         for (int q = 0; q < inch; q++)

  81.         {

  82.             const Mat img0 = bottom_blob_bordered.channel(q);

  83.             Mat img0_tm = bottom_blob_tm.channel(q);

  84. 

  85.             float tmp[8][8][4];

  86. 

  87.             // tile

  88.             for (int i = 0; i < h_tm / 8; i++)

  89.             {

  90.                 for (int j = 0; j < w_tm / 8; j++)

  91.                 {

  92.                     const unsigned short* r0 = img0.row<const unsigned short>(i * 6) + (j * 6) * 4;

  93. 

  94.                     for (int m = 0; m < 8; m++)

  95.                     {

  96.                         float32x4_t _r00 = vcvt_f32_bf16(vld1_u16(r0));

  97.                         float32x4_t _r01 = vcvt_f32_bf16(vld1_u16(r0 + 4));

  98.                         float32x4_t _r02 = vcvt_f32_bf16(vld1_u16(r0 + 8));

  99.                         float32x4_t _r03 = vcvt_f32_bf16(vld1_u16(r0 + 12));

  100.                         float32x4_t _r04 = vcvt_f32_bf16(vld1_u16(r0 + 16));

  101.                         float32x4_t _r05 = vcvt_f32_bf16(vld1_u16(r0 + 20));

  102.                         float32x4_t _r06 = vcvt_f32_bf16(vld1_u16(r0 + 24));

  103.                         float32x4_t _r07 = vcvt_f32_bf16(vld1_u16(r0 + 28));

  104. 

  105.                         float32x4_t _tmp0m = vmlaq_n_f32(vsubq_f32(_r00, _r06), vsubq_f32(_r04, _r02), 5.25f);

  106.                         float32x4_t _tmp7m = vmlaq_n_f32(vsubq_f32(_r07, _r01), vsubq_f32(_r03, _r05), 5.25f);

  107.                         vst1q_f32(tmp[0][m], _tmp0m);

  108.                         vst1q_f32(tmp[7][m], _tmp7m);

  109. 

  110.                         //                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25;

  111.                         //                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25;

  112. 

  113.                         float32x4_t _tmp12a = vmlsq_n_f32(vaddq_f32(_r02, _r06), _r04, 4.25f);

  114.                         float32x4_t _tmp12b = vmlsq_n_f32(vaddq_f32(_r01, _r05), _r03, 4.25f);

  115. 

  116.                         //                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25);

  117.                         //                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25);

  118. 

  119.                         float32x4_t _tmp1m = vaddq_f32(_tmp12a, _tmp12b);

  120.                         float32x4_t _tmp2m = vsubq_f32(_tmp12a, _tmp12b);

  121.                         vst1q_f32(tmp[1][m], _tmp1m);

  122.                         vst1q_f32(tmp[2][m], _tmp2m);

  123. 

  124.                         //                         tmp[1][m] = tmp12a + tmp12b;

  125.                         //                         tmp[2][m] = tmp12a - tmp12b;

  126. 

  127.                         float32x4_t _tmp34a = vmlsq_n_f32(vmlaq_n_f32(_r06, _r02, 0.25f), _r04, 1.25f);

  128.                         float32x4_t _tmp34b = vmlaq_n_f32(vmlsq_n_f32(vmulq_n_f32(_r01, 0.5f), _r03, 2.5f), _r05, 2.f);

  129. 

  130.                         //                         float tmp34a = (r0[6] + r0[2] * 0.25 - r0[4] * 1.25);

  131.                         //                         float tmp34b = (r0[1] * 0.5 - r0[3] * 2.5 + r0[5] * 2);

  132. 

  133.                         float32x4_t _tmp3m = vaddq_f32(_tmp34a, _tmp34b);

  134.                         float32x4_t _tmp4m = vsubq_f32(_tmp34a, _tmp34b);

  135.                         vst1q_f32(tmp[3][m], _tmp3m);

  136.                         vst1q_f32(tmp[4][m], _tmp4m);

  137. 

  138.                         //                         tmp[3][m] = tmp34a + tmp34b;

  139.                         //                         tmp[4][m] = tmp34a - tmp34b;

  140. 

  141.                         float32x4_t _tmp56a = vmlaq_n_f32(_r06, vmlsq_n_f32(_r02, _r04, 1.25f), 4.f);

  142.                         float32x4_t _tmp56b = vmlaq_n_f32(vmlsq_n_f32(vmulq_n_f32(_r01, 2.f), _r03, 2.5f), _r05, 0.5f);

  143. 

  144.                         //                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25) * 4);

  145.                         //                         float tmp56b = (r0[1] * 2 - r0[3] * 2.5 + r0[5] * 0.5);

  146. 

  147.                         float32x4_t _tmp5m = vaddq_f32(_tmp56a, _tmp56b);

  148.                         float32x4_t _tmp6m = vsubq_f32(_tmp56a, _tmp56b);

  149.                         vst1q_f32(tmp[5][m], _tmp5m);

  150.                         vst1q_f32(tmp[6][m], _tmp6m);

  151. 

  152.                         //                         tmp[5][m] = tmp56a + tmp56b;

  153.                         //                         tmp[6][m] = tmp56a - tmp56b;

  154. 

  155.                         r0 += w * 4;

  156.                     }

  157. 

  158.                     float* r0_tm_0 = (float*)img0_tm + (i * w_tm / 8 + j) * 4;

  159.                     float* r0_tm_1 = r0_tm_0 + tiles * 4;

  160.                     float* r0_tm_2 = r0_tm_0 + tiles * 8;

  161.                     float* r0_tm_3 = r0_tm_0 + tiles * 12;

  162.                     float* r0_tm_4 = r0_tm_0 + tiles * 16;

  163.                     float* r0_tm_5 = r0_tm_0 + tiles * 20;

  164.                     float* r0_tm_6 = r0_tm_0 + tiles * 24;

  165.                     float* r0_tm_7 = r0_tm_0 + tiles * 28;

  166. 

  167.                     for (int m = 0; m < 8; m++)

  168.                     {

  169.                         float32x4_t _tmp00 = vld1q_f32(tmp[m][0]);

  170.                         float32x4_t _tmp01 = vld1q_f32(tmp[m][1]);

  171.                         float32x4_t _tmp02 = vld1q_f32(tmp[m][2]);

  172.                         float32x4_t _tmp03 = vld1q_f32(tmp[m][3]);

  173.                         float32x4_t _tmp04 = vld1q_f32(tmp[m][4]);

  174.                         float32x4_t _tmp05 = vld1q_f32(tmp[m][5]);

  175.                         float32x4_t _tmp06 = vld1q_f32(tmp[m][6]);

  176.                         float32x4_t _tmp07 = vld1q_f32(tmp[m][7]);

  177. 

  178.                         float32x4_t _r0tm0 = vmlaq_n_f32(vsubq_f32(_tmp00, _tmp06), vsubq_f32(_tmp04, _tmp02), 5.25f);

  179.                         float32x4_t _r0tm7 = vmlaq_n_f32(vsubq_f32(_tmp07, _tmp01), vsubq_f32(_tmp03, _tmp05), 5.25f);

  180. 

  181.                         //                         r0_tm[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25;

  182.                         //                         r0_tm[7] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25;

  183. 

  184.                         float32x4_t _tmp12a = vmlsq_n_f32(vaddq_f32(_tmp02, _tmp06), _tmp04, 4.25f);

  185.                         float32x4_t _tmp12b = vmlsq_n_f32(vaddq_f32(_tmp01, _tmp05), _tmp03, 4.25f);

  186. 

  187.                         //                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25);

  188.                         //                         float tmp12b = (tmp0[1] + tmp0[5] - tmp0[3] * 4.25);

  189. 

  190.                         float32x4_t _r0tm1 = vaddq_f32(_tmp12a, _tmp12b);

  191.                         float32x4_t _r0tm2 = vsubq_f32(_tmp12a, _tmp12b);

  192. 

  193.                         //                         r0_tm[1] = tmp12a + tmp12b;

  194.                         //                         r0_tm[2] = tmp12a - tmp12b;

  195. 

  196.                         float32x4_t _tmp34a = vmlsq_n_f32(vmlaq_n_f32(_tmp06, _tmp02, 0.25f), _tmp04, 1.25f);

  197.                         float32x4_t _tmp34b = vmlaq_n_f32(vmlsq_n_f32(vmulq_n_f32(_tmp01, 0.5f), _tmp03, 2.5f), _tmp05, 2.f);

  198. 

  199.                         //                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25 - tmp0[4] * 1.25);

  200.                         //                         float tmp34b = (tmp0[1] * 0.5 - tmp0[3] * 2.5 + tmp0[5] * 2);

  201. 

  202.                         float32x4_t _r0tm3 = vaddq_f32(_tmp34a, _tmp34b);

  203.                         float32x4_t _r0tm4 = vsubq_f32(_tmp34a, _tmp34b);

  204. 

  205.                         //                         r0_tm[3] = tmp34a + tmp34b;

  206.                         //                         r0_tm[4] = tmp34a - tmp34b;

  207. 

  208.                         float32x4_t _tmp56a = vmlaq_n_f32(_tmp06, vmlsq_n_f32(_tmp02, _tmp04, 1.25f), 4.f);

  209.                         float32x4_t _tmp56b = vmlaq_n_f32(vmlsq_n_f32(vmulq_n_f32(_tmp01, 2.f), _tmp03, 2.5f), _tmp05, 0.5f);

  210. 

  211.                         //                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25) * 4);

  212.                         //                         float tmp56b = (tmp0[1] * 2 - tmp0[3] * 2.5 + tmp0[5] * 0.5);

  213. 

  214.                         float32x4_t _r0tm5 = vaddq_f32(_tmp56a, _tmp56b);

  215.                         float32x4_t _r0tm6 = vsubq_f32(_tmp56a, _tmp56b);

  216. 

  217.                         //                         r0_tm[5] = tmp56a + tmp56b;

  218.                         //                         r0_tm[6] = tmp56a - tmp56b;

  219. 

  220.                         vst1q_f32(r0_tm_0, _r0tm0);

  221.                         vst1q_f32(r0_tm_1, _r0tm1);

  222.                         vst1q_f32(r0_tm_2, _r0tm2);

  223.                         vst1q_f32(r0_tm_3, _r0tm3);

  224.                         vst1q_f32(r0_tm_4, _r0tm4);

  225.                         vst1q_f32(r0_tm_5, _r0tm5);

  226.                         vst1q_f32(r0_tm_6, _r0tm6);

  227.                         vst1q_f32(r0_tm_7, _r0tm7);

  228. 

  229.                         r0_tm_0 += tiles * 32;

  230.                         r0_tm_1 += tiles * 32;

  231.                         r0_tm_2 += tiles * 32;

  232.                         r0_tm_3 += tiles * 32;

  233.                         r0_tm_4 += tiles * 32;

  234.                         r0_tm_5 += tiles * 32;

  235.                         r0_tm_6 += tiles * 32;

  236.                         r0_tm_7 += tiles * 32;

  237.                     }

  238.                 }

  239.             }

  240.         }

  241.     }

  242.     bottom_blob_bordered = Mat();

  243.     // END transform input

  244. 

  245.     // BEGIN dot

  246.     Mat top_blob_tm;

  247.     {

  248.         int w_tm = outw / 6 * 8;

  249.         int h_tm = outh / 6 * 8;

  250. 

  251.         const int tiles = h_tm / 8 * w_tm / 8;

  252. 

  253.         // permute

  254.         //         bottom_blob_tm.create(tiles, 64, inch, elemsize, elempack, opt.workspace_allocator);

  255.         Mat bottom_blob_tm2;

  256. #if __aarch64__

  257.         if (tiles >= 12)

  258.             bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  259.         else if (tiles >= 8)

  260.             bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  261.         else if (tiles >= 4)

  262.             bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  263.         else if (tiles >= 2)

  264.             bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  265.         else // if (tiles >= 1)

  266.             bottom_blob_tm2.create(1 * inch, tiles, 64, 4u * elempack, elempack, opt.workspace_allocator);

  267. #else

  268.         if (tiles >= 8)

  269.             bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  270.         else if (tiles >= 4)

  271.             bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  272.         else if (tiles >= 2)

  273.             bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator);

  274.         else // if (tiles >= 1)

  275.             bottom_blob_tm2.create(1 * inch, tiles, 64, 4u * elempack, elempack, opt.workspace_allocator);

  276. #endif

  277. 

  278.         #pragma omp parallel for num_threads(opt.num_threads)

  279.         for (int r = 0; r < 64; r++)

  280.         {

  281.             Mat tm2 = bottom_blob_tm2.channel(r);

  282. 

  283.             // tile

  284.             int i = 0;

  285. #if __aarch64__

  286.             for (; i + 11 < tiles; i += 12)

  287.             {

  288.                 float* tm2p = tm2.row(i / 12);

  289. 

  290.                 const float* r0 = bottom_blob_tm;

  291. 

  292.                 r0 += (r * tiles + i) * 4;

  293. 

  294.                 for (int q = 0; q < inch; q++)

  295.                 {

  296.                     asm volatile(

  297.                         "prfm   pldl1keep, [%0, #512]       \n"

  298.                         "ld4    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n"

  299.                         "prfm   pldl1keep, [%0, #512]       \n"

  300.                         "ld4    {v4.4s, v5.4s, v6.4s, v7.4s}, [%0], #64 \n"

  301.                         "prfm   pldl1keep, [%0, #512]       \n"

  302.                         "ld4    {v8.4s, v9.4s, v10.4s, v11.4s}, [%0] \n"

  303.                         "st1    {v0.4s}, [%1], #16          \n"

  304.                         "st1    {v4.4s}, [%1], #16          \n"

  305.                         "st1    {v8.4s}, [%1], #16          \n"

  306.                         "sub    %0, %0, #128                \n"

  307.                         "st1    {v1.4s}, [%1], #16          \n"

  308.                         "st1    {v5.4s}, [%1], #16          \n"

  309.                         "st1    {v9.4s}, [%1], #16          \n"

  310.                         "st1    {v2.4s}, [%1], #16          \n"

  311.                         "st1    {v6.4s}, [%1], #16          \n"

  312.                         "st1    {v10.4s}, [%1], #16         \n"

  313.                         "st1    {v3.4s}, [%1], #16          \n"

  314.                         "st1    {v7.4s}, [%1], #16          \n"

  315.                         "st1    {v11.4s}, [%1], #16         \n"

  316.                         : "=r"(r0),  // %0

  317.                         "=r"(tm2p) // %1

  318.                         : "0"(r0),

  319.                         "1"(tm2p)

  320.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11");

  321.                     r0 += bottom_blob_tm.cstep * 4;

  322.                 }

  323.             }

  324. #endif

  325.             for (; i + 7 < tiles; i += 8)

  326.             {

  327. #if __aarch64__

  328.                 float* tm2p = tm2.row(i / 12 + (i % 12) / 8);

  329. #else

  330.                 float* tm2p = tm2.row(i / 8);

  331. #endif

  332. 

  333.                 const float* r0 = bottom_blob_tm;

  334. 

  335.                 r0 += (r * tiles + i) * 4;

  336. 

  337.                 for (int q = 0; q < inch; q++)

  338.                 {

  339. #if __aarch64__

  340.                     asm volatile(

  341.                         "prfm   pldl1keep, [%0, #512]       \n"

  342.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n"

  343.                         "prfm   pldl1keep, [%0, #512]       \n"

  344.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%0] \n"

  345.                         "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n"

  346.                         "sub    %0, %0, #64                 \n"

  347.                         "st1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n"

  348.                         : "=r"(r0),  // %0

  349.                         "=r"(tm2p) // %1

  350.                         : "0"(r0),

  351.                         "1"(tm2p)

  352.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");

  353. #else

  354.                     asm volatile(

  355.                         "pld        [%0, #512]          \n"

  356.                         "vldm       %0!, {d0-d7}        \n"

  357.                         "pld        [%0, #512]          \n"

  358.                         "vldm       %0, {d16-d23}       \n"

  359. 

  360.                         // transpose 8x4

  361.                         "vtrn.32    q0, q1              \n"

  362.                         "vtrn.32    q2, q3              \n"

  363.                         "vtrn.32    q8, q9              \n"

  364.                         "vtrn.32    q10, q11            \n"

  365.                         "vswp       d1, d4              \n"

  366.                         "vswp       d3, d6              \n"

  367.                         "vswp       d17, d20            \n"

  368.                         "vswp       d19, d22            \n"

  369.                         "vswp       q1, q8              \n"

  370.                         "vswp       q3, q10             \n"

  371. 

  372.                         "vst1.f32   {d0-d3}, [%1 :128]! \n"

  373.                         "vst1.f32   {d16-d19}, [%1 :128]! \n"

  374.                         "sub        %0, %0, #64         \n"

  375.                         "vst1.f32   {d4-d7}, [%1 :128]! \n"

  376.                         "vst1.f32   {d20-d23}, [%1 :128]! \n"

  377.                         : "=r"(r0),  // %0

  378.                         "=r"(tm2p) // %1

  379.                         : "0"(r0),

  380.                         "1"(tm2p)

  381.                         : "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");

  382. #endif

  383.                     r0 += bottom_blob_tm.cstep * 4;

  384.                 }

  385.             }

  386.             for (; i + 3 < tiles; i += 4)

  387.             {

  388. #if __aarch64__

  389.                 float* tm2p = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);

  390. #else

  391.                 float* tm2p = tm2.row(i / 8 + (i % 8) / 4);

  392. #endif

  393. 

  394.                 const float* r0 = bottom_blob_tm;

  395. 

  396.                 r0 += (r * tiles + i) * 4;

  397. 

  398.                 for (int q = 0; q < inch; q++)

  399.                 {

  400. #if __aarch64__

  401.                     asm volatile(

  402.                         "prfm   pldl1keep, [%0, #512]       \n"

  403.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0] \n"

  404.                         "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n"

  405.                         : "=r"(r0),  // %0

  406.                         "=r"(tm2p) // %1

  407.                         : "0"(r0),

  408.                         "1"(tm2p)

  409.                         : "memory", "v0", "v1", "v2", "v3");

  410. #else

  411.                     asm volatile(

  412.                         "pld        [%0, #512]          \n"

  413.                         "vldm       %0, {d0-d7}         \n"

  414.                         "vstm       %1!, {d0-d7}        \n"

  415.                         : "=r"(r0),  // %0

  416.                         "=r"(tm2p) // %1

  417.                         : "0"(r0),

  418.                         "1"(tm2p)

  419.                         : "memory", "q0", "q1", "q2", "q3");

  420. #endif // __aarch64__

  421.                     r0 += bottom_blob_tm.cstep * 4;

  422.                 }

  423.             }

  424.             for (; i + 1 < tiles; i += 2)

  425.             {

  426. #if __aarch64__

  427.                 float* tm2p = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);

  428. #else

  429.                 float* tm2p = tm2.row(i / 8 + (i % 8) / 4 + (i % 4) / 2);

  430. #endif

  431. 

  432.                 const float* r0 = bottom_blob_tm;

  433. 

  434.                 r0 += (r * tiles + i) * 4;

  435. 

  436.                 for (int q = 0; q < inch; q++)

  437.                 {

  438. #if __aarch64__

  439.                     asm volatile(

  440.                         "prfm   pldl1keep, [%0, #256]       \n"

  441.                         "ld1    {v0.4s, v1.4s}, [%0]        \n"

  442.                         "st1    {v0.4s, v1.4s}, [%1], #32   \n"

  443.                         : "=r"(r0),  // %0

  444.                         "=r"(tm2p) // %1

  445.                         : "0"(r0),

  446.                         "1"(tm2p)

  447.                         : "memory", "v0", "v1");

  448. #else

  449.                     asm volatile(

  450.                         "pld        [%0, #256]          \n"

  451.                         "vld1.f32   {d0-d3}, [%0 :128]  \n"

  452.                         "vst1.f32   {d0-d3}, [%1 :128]! \n"

  453.                         : "=r"(r0),  // %0

  454.                         "=r"(tm2p) // %1

  455.                         : "0"(r0),

  456.                         "1"(tm2p)

  457.                         : "memory", "q0", "q1");

  458. #endif // __aarch64__

  459.                     r0 += bottom_blob_tm.cstep * 4;

  460.                 }

  461.             }

  462.             for (; i < tiles; i++)

  463.             {

  464. #if __aarch64__

  465.                 float* tm2p = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);

  466. #else

  467.                 float* tm2p = tm2.row(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);

  468. #endif

  469. 

  470.                 const float* r0 = bottom_blob_tm;

  471. 

  472.                 r0 += (r * tiles + i) * 4;

  473. 

  474.                 for (int q = 0; q < inch; q++)

  475.                 {

  476. #if __aarch64__

  477.                     asm volatile(

  478.                         "prfm   pldl1keep, [%0, #128]       \n"

  479.                         "ld1    {v0.4s}, [%0]               \n"

  480.                         "st1    {v0.4s}, [%1], #16          \n"

  481.                         : "=r"(r0),  // %0

  482.                         "=r"(tm2p) // %1

  483.                         : "0"(r0),

  484.                         "1"(tm2p)

  485.                         : "memory", "v0");

  486. #else

  487.                     asm volatile(

  488.                         "pld        [%0, #128]          \n"

  489.                         "vld1.f32   {d0-d1}, [%0 :128]  \n"

  490.                         "vst1.f32   {d0-d1}, [%1 :128]! \n"

  491.                         : "=r"(r0),  // %0

  492.                         "=r"(tm2p) // %1

  493.                         : "0"(r0),

  494.                         "1"(tm2p)

  495.                         : "memory", "q0");

  496. #endif // __aarch64__

  497.                     r0 += bottom_blob_tm.cstep * 4;

  498.                 }

  499.             }

  500.         }

  501. 

  502.         bottom_blob_tm = Mat();

  503.         // permute end

  504. 

  505.         top_blob_tm.create(tiles, 64, outch, 4u * elempack, elempack, opt.workspace_allocator);

  506. 

  507.         int remain_outch_start = 0;

  508. 

  509. #if __ARM_NEON && __aarch64__

  510.         int nn_outch = 0;

  511.         nn_outch = outch >> 1;

  512.         remain_outch_start = nn_outch << 1;

  513. 

  514.         #pragma omp parallel for num_threads(opt.num_threads)

  515.         for (int pp = 0; pp < nn_outch; pp++)

  516.         {

  517.             int p = pp * 2;

  518. 

  519.             float* output0_tm = top_blob_tm.channel(p);

  520.             float* output1_tm = top_blob_tm.channel(p + 1);

  521. 

  522.             const Mat kernel01_tm = kernel_tm.channel(pp);

  523. 

  524.             for (int r = 0; r < 64; r++)

  525.             {

  526.                 const Mat bb2 = bottom_blob_tm2.channel(r);

  527. 

  528.                 int i = 0;

  529.                 for (; i + 11 < tiles; i += 12)

  530.                 {

  531.                     const float* r0 = bb2.row(i / 12);

  532. 

  533.                     const float* k01 = kernel01_tm.row(r);

  534. 

  535.                     int nn = inch; // inch always > 0

  536. 

  537.                     asm volatile(

  538.                         "eor    v8.16b, v8.16b, v8.16b      \n"

  539.                         "eor    v9.16b, v9.16b, v9.16b      \n"

  540.                         "eor    v10.16b, v10.16b, v10.16b   \n"

  541.                         "eor    v11.16b, v11.16b, v11.16b   \n"

  542.                         "eor    v12.16b, v12.16b, v12.16b   \n"

  543.                         "eor    v13.16b, v13.16b, v13.16b   \n"

  544.                         "eor    v14.16b, v14.16b, v14.16b   \n"

  545.                         "eor    v15.16b, v15.16b, v15.16b   \n"

  546.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  547.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  548.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  549.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  550.                         "eor    v20.16b, v20.16b, v20.16b   \n"

  551.                         "eor    v21.16b, v21.16b, v21.16b   \n"

  552.                         "eor    v22.16b, v22.16b, v22.16b   \n"

  553.                         "eor    v23.16b, v23.16b, v23.16b   \n"

  554.                         "eor    v24.16b, v24.16b, v24.16b   \n"

  555.                         "eor    v25.16b, v25.16b, v25.16b   \n"

  556.                         "eor    v26.16b, v26.16b, v26.16b   \n"

  557.                         "eor    v27.16b, v27.16b, v27.16b   \n"

  558.                         "eor    v28.16b, v28.16b, v28.16b   \n"

  559.                         "eor    v29.16b, v29.16b, v29.16b   \n"

  560.                         "eor    v30.16b, v30.16b, v30.16b   \n"

  561.                         "eor    v31.16b, v31.16b, v31.16b   \n"

  562. 

  563.                         "0:                                 \n"

  564. 

  565.                         "prfm   pldl1keep, [%3, #512]       \n"

  566.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n"

  567. 

  568.                         "prfm   pldl1keep, [%4, #512]       \n"

  569.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64   \n" // w0011_01

  570. 

  571.                         "fmla   v8.4s, v4.4s, v0.s[0]       \n"

  572.                         "fmla   v9.4s, v4.4s, v0.s[1]       \n"

  573.                         "fmla   v10.4s, v4.4s, v0.s[2]      \n"

  574.                         "fmla   v11.4s, v4.4s, v0.s[3]      \n"

  575.                         "fmla   v12.4s, v4.4s, v1.s[0]      \n"

  576.                         "fmla   v13.4s, v4.4s, v1.s[1]      \n"

  577.                         "fmla   v14.4s, v4.4s, v1.s[2]      \n"

  578.                         "fmla   v15.4s, v4.4s, v1.s[3]      \n"

  579.                         "fmla   v16.4s, v4.4s, v2.s[0]      \n"

  580.                         "fmla   v17.4s, v4.4s, v2.s[1]      \n"

  581.                         "fmla   v18.4s, v4.4s, v2.s[2]      \n"

  582.                         "fmla   v19.4s, v4.4s, v2.s[3]      \n"

  583. 

  584.                         "fmla   v20.4s, v5.4s, v0.s[0]      \n"

  585.                         "fmla   v21.4s, v5.4s, v0.s[1]      \n"

  586.                         "fmla   v22.4s, v5.4s, v0.s[2]      \n"

  587.                         "fmla   v23.4s, v5.4s, v0.s[3]      \n"

  588.                         "fmla   v24.4s, v5.4s, v1.s[0]      \n"

  589.                         "fmla   v25.4s, v5.4s, v1.s[1]      \n"

  590.                         "fmla   v26.4s, v5.4s, v1.s[2]      \n"

  591.                         "fmla   v27.4s, v5.4s, v1.s[3]      \n"

  592.                         "fmla   v28.4s, v5.4s, v2.s[0]      \n"

  593.                         "fmla   v29.4s, v5.4s, v2.s[1]      \n"

  594.                         "fmla   v30.4s, v5.4s, v2.s[2]      \n"

  595.                         "fmla   v31.4s, v5.4s, v2.s[3]      \n"

  596. 

  597.                         "fmla   v8.4s, v6.4s, v3.s[0]       \n"

  598.                         "fmla   v9.4s, v6.4s, v3.s[1]       \n"

  599.                         "fmla   v10.4s, v6.4s, v3.s[2]      \n"

  600.                         "fmla   v11.4s, v6.4s, v3.s[3]      \n"

  601. 

  602.                         "fmla   v20.4s, v7.4s, v3.s[0]      \n"

  603.                         "fmla   v21.4s, v7.4s, v3.s[1]      \n"

  604.                         "fmla   v22.4s, v7.4s, v3.s[2]      \n"

  605.                         "fmla   v23.4s, v7.4s, v3.s[3]      \n"

  606. 

  607.                         "prfm   pldl1keep, [%3, #512]       \n"

  608.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n"

  609. 

  610.                         "fmla   v12.4s, v6.4s, v0.s[0]      \n"

  611.                         "fmla   v13.4s, v6.4s, v0.s[1]      \n"

  612.                         "fmla   v14.4s, v6.4s, v0.s[2]      \n"

  613.                         "fmla   v15.4s, v6.4s, v0.s[3]      \n"

  614.                         "fmla   v16.4s, v6.4s, v1.s[0]      \n"

  615.                         "fmla   v17.4s, v6.4s, v1.s[1]      \n"

  616.                         "fmla   v18.4s, v6.4s, v1.s[2]      \n"

  617.                         "fmla   v19.4s, v6.4s, v1.s[3]      \n"

  618. 

  619.                         "fmla   v24.4s, v7.4s, v0.s[0]      \n"

  620.                         "fmla   v25.4s, v7.4s, v0.s[1]      \n"

  621.                         "fmla   v26.4s, v7.4s, v0.s[2]      \n"

  622.                         "fmla   v27.4s, v7.4s, v0.s[3]      \n"

  623.                         "fmla   v28.4s, v7.4s, v1.s[0]      \n"

  624.                         "fmla   v29.4s, v7.4s, v1.s[1]      \n"

  625.                         "fmla   v30.4s, v7.4s, v1.s[2]      \n"

  626.                         "fmla   v31.4s, v7.4s, v1.s[3]      \n"

  627. 

  628.                         "prfm   pldl1keep, [%4, #512]       \n"

  629.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64   \n" // w2233_01

  630. 

  631.                         "fmla   v8.4s, v4.4s, v2.s[0]       \n"

  632.                         "fmla   v9.4s, v4.4s, v2.s[1]       \n"

  633.                         "fmla   v10.4s, v4.4s, v2.s[2]      \n"

  634.                         "fmla   v11.4s, v4.4s, v2.s[3]      \n"

  635.                         "fmla   v12.4s, v4.4s, v3.s[0]      \n"

  636.                         "fmla   v13.4s, v4.4s, v3.s[1]      \n"

  637.                         "fmla   v14.4s, v4.4s, v3.s[2]      \n"

  638.                         "fmla   v15.4s, v4.4s, v3.s[3]      \n"

  639. 

  640.                         "fmla   v20.4s, v5.4s, v2.s[0]      \n"

  641.                         "fmla   v21.4s, v5.4s, v2.s[1]      \n"

  642.                         "fmla   v22.4s, v5.4s, v2.s[2]      \n"

  643.                         "fmla   v23.4s, v5.4s, v2.s[3]      \n"

  644.                         "fmla   v24.4s, v5.4s, v3.s[0]      \n"

  645.                         "fmla   v25.4s, v5.4s, v3.s[1]      \n"

  646.                         "fmla   v26.4s, v5.4s, v3.s[2]      \n"

  647.                         "fmla   v27.4s, v5.4s, v3.s[3]      \n"

  648. 

  649.                         "prfm   pldl1keep, [%3, #512]       \n"

  650.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n"

  651. 

  652.                         "fmla   v16.4s, v4.4s, v0.s[0]      \n"

  653.                         "fmla   v17.4s, v4.4s, v0.s[1]      \n"

  654.                         "fmla   v18.4s, v4.4s, v0.s[2]      \n"

  655.                         "fmla   v19.4s, v4.4s, v0.s[3]      \n"

  656. 

  657.                         "fmla   v28.4s, v5.4s, v0.s[0]      \n"

  658.                         "fmla   v29.4s, v5.4s, v0.s[1]      \n"

  659.                         "fmla   v30.4s, v5.4s, v0.s[2]      \n"

  660.                         "fmla   v31.4s, v5.4s, v0.s[3]      \n"

  661. 

  662.                         "fmla   v8.4s, v6.4s, v1.s[0]       \n"

  663.                         "fmla   v9.4s, v6.4s, v1.s[1]       \n"

  664.                         "fmla   v10.4s, v6.4s, v1.s[2]      \n"

  665.                         "fmla   v11.4s, v6.4s, v1.s[3]      \n"

  666.                         "fmla   v12.4s, v6.4s, v2.s[0]      \n"

  667.                         "fmla   v13.4s, v6.4s, v2.s[1]      \n"

  668.                         "fmla   v14.4s, v6.4s, v2.s[2]      \n"

  669.                         "fmla   v15.4s, v6.4s, v2.s[3]      \n"

  670.                         "fmla   v16.4s, v6.4s, v3.s[0]      \n"

  671.                         "fmla   v17.4s, v6.4s, v3.s[1]      \n"

  672.                         "fmla   v18.4s, v6.4s, v3.s[2]      \n"

  673.                         "fmla   v19.4s, v6.4s, v3.s[3]      \n"

  674. 

  675.                         "subs   %w0, %w0, #1                \n"

  676. 

  677.                         "fmla   v20.4s, v7.4s, v1.s[0]      \n"

  678.                         "fmla   v21.4s, v7.4s, v1.s[1]      \n"

  679.                         "fmla   v22.4s, v7.4s, v1.s[2]      \n"

  680.                         "fmla   v23.4s, v7.4s, v1.s[3]      \n"

  681.                         "fmla   v24.4s, v7.4s, v2.s[0]      \n"

  682.                         "fmla   v25.4s, v7.4s, v2.s[1]      \n"

  683.                         "fmla   v26.4s, v7.4s, v2.s[2]      \n"

  684.                         "fmla   v27.4s, v7.4s, v2.s[3]      \n"

  685.                         "fmla   v28.4s, v7.4s, v3.s[0]      \n"

  686.                         "fmla   v29.4s, v7.4s, v3.s[1]      \n"

  687.                         "fmla   v30.4s, v7.4s, v3.s[2]      \n"

  688.                         "fmla   v31.4s, v7.4s, v3.s[3]      \n"

  689. 

  690.                         "bne    0b                          \n"

  691. 

  692.                         "st1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n"

  693.                         "st1    {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n"

  694.                         "st1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n"

  695.                         "st1    {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n"

  696.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  697.                         "st1    {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n"

  698. 

  699.                         : "=r"(nn),         // %0

  700.                         "=r"(output0_tm), // %1

  701.                         "=r"(output1_tm), // %2

  702.                         "=r"(r0),         // %3

  703.                         "=r"(k01)         // %4

  704.                         : "0"(nn),

  705.                         "1"(output0_tm),

  706.                         "2"(output1_tm),

  707.                         "3"(r0),

  708.                         "4"(k01)

  709.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");

  710.                 }

  711.                 for (; i + 7 < tiles; i += 8)

  712.                 {

  713.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8);

  714. 

  715.                     const float* k01 = kernel01_tm.row(r);

  716. 

  717.                     int nn = inch; // inch always > 0

  718. 

  719.                     asm volatile(

  720.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  721.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  722.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  723.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  724.                         "eor    v20.16b, v20.16b, v20.16b   \n"

  725.                         "eor    v21.16b, v21.16b, v21.16b   \n"

  726.                         "eor    v22.16b, v22.16b, v22.16b   \n"

  727.                         "eor    v23.16b, v23.16b, v23.16b   \n"

  728.                         "eor    v24.16b, v24.16b, v24.16b   \n"

  729.                         "eor    v25.16b, v25.16b, v25.16b   \n"

  730.                         "eor    v26.16b, v26.16b, v26.16b   \n"

  731.                         "eor    v27.16b, v27.16b, v27.16b   \n"

  732.                         "eor    v28.16b, v28.16b, v28.16b   \n"

  733.                         "eor    v29.16b, v29.16b, v29.16b   \n"

  734.                         "eor    v30.16b, v30.16b, v30.16b   \n"

  735.                         "eor    v31.16b, v31.16b, v31.16b   \n"

  736. 

  737.                         "0:                                 \n"

  738. 

  739.                         "prfm   pldl1keep, [%3, #512]       \n"

  740.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3

  741. 

  742.                         "prfm   pldl1keep, [%4, #512]       \n"

  743.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01

  744. 

  745.                         "prfm   pldl1keep, [%3, #512]       \n"

  746.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // r4 r5 r6 r7

  747. 

  748.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  749.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  750.                         "fmla   v18.4s, v8.4s, v2.s[0]      \n"

  751.                         "fmla   v19.4s, v8.4s, v3.s[0]      \n"

  752.                         "fmla   v20.4s, v8.4s, v4.s[0]      \n"

  753.                         "fmla   v21.4s, v8.4s, v5.s[0]      \n"

  754.                         "fmla   v22.4s, v8.4s, v6.s[0]      \n"

  755.                         "fmla   v23.4s, v8.4s, v7.s[0]      \n"

  756. 

  757.                         "fmla   v24.4s, v9.4s, v0.s[0]      \n"

  758.                         "fmla   v25.4s, v9.4s, v1.s[0]      \n"

  759.                         "fmla   v26.4s, v9.4s, v2.s[0]      \n"

  760.                         "fmla   v27.4s, v9.4s, v3.s[0]      \n"

  761.                         "fmla   v28.4s, v9.4s, v4.s[0]      \n"

  762.                         "fmla   v29.4s, v9.4s, v5.s[0]      \n"

  763.                         "fmla   v30.4s, v9.4s, v6.s[0]      \n"

  764.                         "fmla   v31.4s, v9.4s, v7.s[0]      \n"

  765. 

  766.                         "prfm   pldl1keep, [%4, #512]       \n"

  767.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01

  768. 

  769.                         "fmla   v16.4s, v10.4s, v0.s[1]     \n"

  770.                         "fmla   v17.4s, v10.4s, v1.s[1]     \n"

  771.                         "fmla   v18.4s, v10.4s, v2.s[1]     \n"

  772.                         "fmla   v19.4s, v10.4s, v3.s[1]     \n"

  773.                         "fmla   v20.4s, v10.4s, v4.s[1]     \n"

  774.                         "fmla   v21.4s, v10.4s, v5.s[1]     \n"

  775.                         "fmla   v22.4s, v10.4s, v6.s[1]     \n"

  776.                         "fmla   v23.4s, v10.4s, v7.s[1]     \n"

  777. 

  778.                         "fmla   v24.4s, v11.4s, v0.s[1]     \n"

  779.                         "fmla   v25.4s, v11.4s, v1.s[1]     \n"

  780.                         "fmla   v26.4s, v11.4s, v2.s[1]     \n"

  781.                         "fmla   v27.4s, v11.4s, v3.s[1]     \n"

  782.                         "fmla   v28.4s, v11.4s, v4.s[1]     \n"

  783.                         "fmla   v29.4s, v11.4s, v5.s[1]     \n"

  784.                         "fmla   v30.4s, v11.4s, v6.s[1]     \n"

  785.                         "fmla   v31.4s, v11.4s, v7.s[1]     \n"

  786. 

  787.                         "fmla   v16.4s, v12.4s, v0.s[2]     \n"

  788.                         "fmla   v17.4s, v12.4s, v1.s[2]     \n"

  789.                         "fmla   v18.4s, v12.4s, v2.s[2]     \n"

  790.                         "fmla   v19.4s, v12.4s, v3.s[2]     \n"

  791.                         "fmla   v20.4s, v12.4s, v4.s[2]     \n"

  792.                         "fmla   v21.4s, v12.4s, v5.s[2]     \n"

  793.                         "fmla   v22.4s, v12.4s, v6.s[2]     \n"

  794.                         "fmla   v23.4s, v12.4s, v7.s[2]     \n"

  795. 

  796.                         "fmla   v24.4s, v13.4s, v0.s[2]     \n"

  797.                         "fmla   v25.4s, v13.4s, v1.s[2]     \n"

  798.                         "fmla   v26.4s, v13.4s, v2.s[2]     \n"

  799.                         "fmla   v27.4s, v13.4s, v3.s[2]     \n"

  800.                         "fmla   v28.4s, v13.4s, v4.s[2]     \n"

  801.                         "fmla   v29.4s, v13.4s, v5.s[2]     \n"

  802.                         "fmla   v30.4s, v13.4s, v6.s[2]     \n"

  803.                         "fmla   v31.4s, v13.4s, v7.s[2]     \n"

  804. 

  805.                         "fmla   v16.4s, v14.4s, v0.s[3]     \n"

  806.                         "fmla   v17.4s, v14.4s, v1.s[3]     \n"

  807.                         "fmla   v18.4s, v14.4s, v2.s[3]     \n"

  808.                         "fmla   v19.4s, v14.4s, v3.s[3]     \n"

  809.                         "fmla   v20.4s, v14.4s, v4.s[3]     \n"

  810.                         "fmla   v21.4s, v14.4s, v5.s[3]     \n"

  811.                         "fmla   v22.4s, v14.4s, v6.s[3]     \n"

  812.                         "fmla   v23.4s, v14.4s, v7.s[3]     \n"

  813. 

  814.                         "subs   %w0, %w0, #1                \n"

  815. 

  816.                         "fmla   v24.4s, v15.4s, v0.s[3]     \n"

  817.                         "fmla   v25.4s, v15.4s, v1.s[3]     \n"

  818.                         "fmla   v26.4s, v15.4s, v2.s[3]     \n"

  819.                         "fmla   v27.4s, v15.4s, v3.s[3]     \n"

  820.                         "fmla   v28.4s, v15.4s, v4.s[3]     \n"

  821.                         "fmla   v29.4s, v15.4s, v5.s[3]     \n"

  822.                         "fmla   v30.4s, v15.4s, v6.s[3]     \n"

  823.                         "fmla   v31.4s, v15.4s, v7.s[3]     \n"

  824. 

  825.                         "bne    0b                          \n"

  826. 

  827.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  828.                         "st1    {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n"

  829.                         "st1    {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n"

  830.                         "st1    {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n"

  831. 

  832.                         : "=r"(nn),         // %0

  833.                         "=r"(output0_tm), // %1

  834.                         "=r"(output1_tm), // %2

  835.                         "=r"(r0),         // %3

  836.                         "=r"(k01)         // %4

  837.                         : "0"(nn),

  838.                         "1"(output0_tm),

  839.                         "2"(output1_tm),

  840.                         "3"(r0),

  841.                         "4"(k01)

  842.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");

  843.                 }

  844.                 for (; i + 3 < tiles; i += 4)

  845.                 {

  846.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);

  847. 

  848.                     const float* k01 = kernel01_tm.row(r);

  849. 

  850.                     int nn = inch; // inch always > 0

  851. 

  852.                     asm volatile(

  853.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  854.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  855.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  856.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  857.                         "eor    v20.16b, v20.16b, v20.16b   \n"

  858.                         "eor    v21.16b, v21.16b, v21.16b   \n"

  859.                         "eor    v22.16b, v22.16b, v22.16b   \n"

  860.                         "eor    v23.16b, v23.16b, v23.16b   \n"

  861. 

  862.                         "0:                                 \n"

  863. 

  864.                         "prfm   pldl1keep, [%3, #512]       \n"

  865.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3

  866. 

  867.                         "prfm   pldl1keep, [%4, #512]       \n"

  868.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01

  869. 

  870.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  871.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  872.                         "fmla   v18.4s, v8.4s, v2.s[0]      \n"

  873.                         "fmla   v19.4s, v8.4s, v3.s[0]      \n"

  874. 

  875.                         "fmla   v20.4s, v9.4s, v0.s[0]     \n"

  876.                         "fmla   v21.4s, v9.4s, v1.s[0]     \n"

  877.                         "fmla   v22.4s, v9.4s, v2.s[0]     \n"

  878.                         "fmla   v23.4s, v9.4s, v3.s[0]     \n"

  879. 

  880.                         "prfm   pldl1keep, [%4, #512]       \n"

  881.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01

  882. 

  883.                         "fmla   v16.4s, v10.4s, v0.s[1]      \n"

  884.                         "fmla   v17.4s, v10.4s, v1.s[1]      \n"

  885.                         "fmla   v18.4s, v10.4s, v2.s[1]      \n"

  886.                         "fmla   v19.4s, v10.4s, v3.s[1]      \n"

  887. 

  888.                         "fmla   v20.4s, v11.4s, v0.s[1]     \n"

  889.                         "fmla   v21.4s, v11.4s, v1.s[1]     \n"

  890.                         "fmla   v22.4s, v11.4s, v2.s[1]     \n"

  891.                         "fmla   v23.4s, v11.4s, v3.s[1]     \n"

  892. 

  893.                         "fmla   v16.4s, v12.4s, v0.s[2]     \n"

  894.                         "fmla   v17.4s, v12.4s, v1.s[2]     \n"

  895.                         "fmla   v18.4s, v12.4s, v2.s[2]     \n"

  896.                         "fmla   v19.4s, v12.4s, v3.s[2]     \n"

  897. 

  898.                         "fmla   v20.4s, v13.4s, v0.s[2]     \n"

  899.                         "fmla   v21.4s, v13.4s, v1.s[2]     \n"

  900.                         "fmla   v22.4s, v13.4s, v2.s[2]     \n"

  901.                         "fmla   v23.4s, v13.4s, v3.s[2]     \n"

  902. 

  903.                         "subs   %w0, %w0, #1                \n"

  904. 

  905.                         "fmla   v16.4s, v14.4s, v0.s[3]     \n"

  906.                         "fmla   v17.4s, v14.4s, v1.s[3]     \n"

  907.                         "fmla   v18.4s, v14.4s, v2.s[3]     \n"

  908.                         "fmla   v19.4s, v14.4s, v3.s[3]     \n"

  909. 

  910.                         "fmla   v20.4s, v15.4s, v0.s[3]     \n"

  911.                         "fmla   v21.4s, v15.4s, v1.s[3]     \n"

  912.                         "fmla   v22.4s, v15.4s, v2.s[3]     \n"

  913.                         "fmla   v23.4s, v15.4s, v3.s[3]     \n"

  914. 

  915.                         "bne    0b                          \n"

  916. 

  917.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  918.                         "st1    {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n"

  919. 

  920.                         : "=r"(nn),         // %0

  921.                         "=r"(output0_tm), // %1

  922.                         "=r"(output1_tm), // %2

  923.                         "=r"(r0),         // %3

  924.                         "=r"(k01)         // %4

  925.                         : "0"(nn),

  926.                         "1"(output0_tm),

  927.                         "2"(output1_tm),

  928.                         "3"(r0),

  929.                         "4"(k01)

  930.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");

  931.                 }

  932.                 for (; i + 1 < tiles; i += 2)

  933.                 {

  934.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);

  935. 

  936.                     const float* k01 = kernel01_tm.row(r);

  937. 

  938.                     int nn = inch; // inch always > 0

  939. 

  940.                     asm volatile(

  941.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  942.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  943.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  944.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  945. 

  946.                         "0:                                 \n"

  947. 

  948.                         "prfm   pldl1keep, [%3, #256]       \n"

  949.                         "ld1    {v0.4s, v1.4s}, [%3], #32   \n" // r0 r1

  950. 

  951.                         "prfm   pldl1keep, [%4, #512]       \n"

  952.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01

  953. 

  954.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  955.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  956.                         "fmla   v18.4s, v9.4s, v0.s[0]      \n"

  957.                         "fmla   v19.4s, v9.4s, v1.s[0]      \n"

  958. 

  959.                         "prfm   pldl1keep, [%4, #512]       \n"

  960.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01

  961. 

  962.                         "fmla   v16.4s, v10.4s, v0.s[1]     \n"

  963.                         "fmla   v17.4s, v10.4s, v1.s[1]     \n"

  964.                         "fmla   v18.4s, v11.4s, v0.s[1]     \n"

  965.                         "fmla   v19.4s, v11.4s, v1.s[1]     \n"

  966. 

  967.                         "fmla   v16.4s, v12.4s, v0.s[2]     \n"

  968.                         "fmla   v17.4s, v12.4s, v1.s[2]     \n"

  969.                         "fmla   v18.4s, v13.4s, v0.s[2]     \n"

  970.                         "fmla   v19.4s, v13.4s, v1.s[2]     \n"

  971. 

  972.                         "subs   %w0, %w0, #1                \n"

  973. 

  974.                         "fmla   v16.4s, v14.4s, v0.s[3]     \n"

  975.                         "fmla   v17.4s, v14.4s, v1.s[3]     \n"

  976.                         "fmla   v18.4s, v15.4s, v0.s[3]     \n"

  977.                         "fmla   v19.4s, v15.4s, v1.s[3]     \n"

  978. 

  979.                         "bne    0b                          \n"

  980. 

  981.                         "st1    {v16.4s, v17.4s}, [%1], #32 \n"

  982.                         "st1    {v18.4s, v19.4s}, [%2], #32 \n"

  983. 

  984.                         : "=r"(nn),         // %0

  985.                         "=r"(output0_tm), // %1

  986.                         "=r"(output1_tm), // %2

  987.                         "=r"(r0),         // %3

  988.                         "=r"(k01)         // %4

  989.                         : "0"(nn),

  990.                         "1"(output0_tm),

  991.                         "2"(output1_tm),

  992.                         "3"(r0),

  993.                         "4"(k01)

  994.                         : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19");

  995.                 }

  996.                 for (; i < tiles; i++)

  997.                 {

  998.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);

  999. 

  1000.                     const float* k01 = kernel01_tm.row(r);

  1001. 

  1002.                     int nn = inch; // inch always > 0

  1003. 

  1004.                     asm volatile(

  1005.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1006.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  1007. 

  1008.                         "0:                                 \n"

  1009. 

  1010.                         "prfm   pldl1keep, [%3, #128]       \n"

  1011.                         "ld1    {v0.4s}, [%3], #16          \n" // r0

  1012. 

  1013.                         "prfm   pldl1keep, [%4, #512]       \n"

  1014.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01

  1015. 

  1016.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  1017.                         "fmla   v17.4s, v9.4s, v0.s[0]      \n"

  1018. 

  1019.                         "prfm   pldl1keep, [%4, #512]       \n"

  1020.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01

  1021. 

  1022.                         "fmla   v16.4s, v10.4s, v0.s[1]     \n"

  1023.                         "fmla   v17.4s, v11.4s, v0.s[1]     \n"

  1024. 

  1025.                         "fmla   v16.4s, v12.4s, v0.s[2]     \n"

  1026.                         "fmla   v17.4s, v13.4s, v0.s[2]     \n"

  1027. 

  1028.                         "subs   %w0, %w0, #1                \n"

  1029. 

  1030.                         "fmla   v16.4s, v14.4s, v0.s[3]     \n"

  1031.                         "fmla   v17.4s, v15.4s, v0.s[3]     \n"

  1032. 

  1033.                         "bne    0b                          \n"

  1034. 

  1035.                         "st1    {v16.4s}, [%1], #16         \n"

  1036.                         "st1    {v17.4s}, [%2], #16         \n"

  1037. 

  1038.                         : "=r"(nn),         // %0

  1039.                         "=r"(output0_tm), // %1

  1040.                         "=r"(output1_tm), // %2

  1041.                         "=r"(r0),         // %3

  1042.                         "=r"(k01)         // %4

  1043.                         : "0"(nn),

  1044.                         "1"(output0_tm),

  1045.                         "2"(output1_tm),

  1046.                         "3"(r0),

  1047.                         "4"(k01)

  1048.                         : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17");

  1049.                 }

  1050.             }

  1051.         }

  1052. #endif // __ARM_NEON && __aarch64__

  1053. 

  1054.         #pragma omp parallel for num_threads(opt.num_threads)

  1055.         for (int p = remain_outch_start; p < outch; p++)

  1056.         {

  1057.             float* output0_tm = top_blob_tm.channel(p);

  1058. 

  1059. #if __aarch64__

  1060.             const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2);

  1061. #else

  1062.             const Mat kernel0_tm = kernel_tm.channel(p);

  1063. #endif

  1064. 

  1065.             for (int r = 0; r < 64; r++)

  1066.             {

  1067.                 const Mat bb2 = bottom_blob_tm2.channel(r);

  1068. 

  1069.                 int i = 0;

  1070. #if __aarch64__

  1071.                 for (; i + 11 < tiles; i += 12)

  1072.                 {

  1073.                     const float* r0 = bb2.row(i / 12);

  1074. 

  1075.                     const float* k0 = kernel0_tm.row(r);

  1076. 

  1077.                     int nn = inch; // inch always > 0

  1078. 

  1079.                     asm volatile(

  1080.                         "eor    v8.16b, v8.16b, v8.16b      \n"

  1081.                         "eor    v9.16b, v9.16b, v9.16b      \n"

  1082.                         "eor    v10.16b, v10.16b, v10.16b   \n"

  1083.                         "eor    v11.16b, v11.16b, v11.16b   \n"

  1084.                         "eor    v12.16b, v12.16b, v12.16b   \n"

  1085.                         "eor    v13.16b, v13.16b, v13.16b   \n"

  1086.                         "eor    v14.16b, v14.16b, v14.16b   \n"

  1087.                         "eor    v15.16b, v15.16b, v15.16b   \n"

  1088.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1089.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  1090.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  1091.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  1092. 

  1093.                         "0:                                 \n"

  1094. 

  1095.                         "prfm   pldl1keep, [%2, #512]       \n"

  1096.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n"

  1097. 

  1098.                         "prfm   pldl1keep, [%3, #512]       \n"

  1099.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // w0123_0

  1100. 

  1101.                         "fmla   v8.4s, v4.4s, v0.s[0]       \n"

  1102.                         "fmla   v9.4s, v4.4s, v0.s[1]       \n"

  1103.                         "fmla   v10.4s, v4.4s, v0.s[2]      \n"

  1104.                         "fmla   v11.4s, v4.4s, v0.s[3]      \n"

  1105.                         "fmla   v12.4s, v4.4s, v1.s[0]      \n"

  1106.                         "fmla   v13.4s, v4.4s, v1.s[1]      \n"

  1107.                         "fmla   v14.4s, v4.4s, v1.s[2]      \n"

  1108.                         "fmla   v15.4s, v4.4s, v1.s[3]      \n"

  1109.                         "fmla   v16.4s, v4.4s, v2.s[0]      \n"

  1110.                         "fmla   v17.4s, v4.4s, v2.s[1]      \n"

  1111.                         "fmla   v18.4s, v4.4s, v2.s[2]      \n"

  1112.                         "fmla   v19.4s, v4.4s, v2.s[3]      \n"

  1113. 

  1114.                         "prfm   pldl1keep, [%2, #512]       \n"

  1115.                         "ld1    {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n"

  1116. 

  1117.                         "fmla   v8.4s, v5.4s, v3.s[0]       \n"

  1118.                         "fmla   v9.4s, v5.4s, v3.s[1]       \n"

  1119.                         "fmla   v10.4s, v5.4s, v3.s[2]      \n"

  1120.                         "fmla   v11.4s, v5.4s, v3.s[3]      \n"

  1121.                         "fmla   v12.4s, v5.4s, v20.s[0]     \n"

  1122.                         "fmla   v13.4s, v5.4s, v20.s[1]     \n"

  1123.                         "fmla   v14.4s, v5.4s, v20.s[2]     \n"

  1124.                         "fmla   v15.4s, v5.4s, v20.s[3]     \n"

  1125.                         "fmla   v16.4s, v5.4s, v21.s[0]     \n"

  1126.                         "fmla   v17.4s, v5.4s, v21.s[1]     \n"

  1127.                         "fmla   v18.4s, v5.4s, v21.s[2]     \n"

  1128.                         "fmla   v19.4s, v5.4s, v21.s[3]     \n"

  1129. 

  1130.                         "prfm   pldl1keep, [%2, #512]       \n"

  1131.                         "ld1    {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n"

  1132. 

  1133.                         "fmla   v8.4s, v6.4s, v22.s[0]      \n"

  1134.                         "fmla   v9.4s, v6.4s, v22.s[1]      \n"

  1135.                         "fmla   v10.4s, v6.4s, v22.s[2]     \n"

  1136.                         "fmla   v11.4s, v6.4s, v22.s[3]     \n"

  1137.                         "fmla   v12.4s, v6.4s, v23.s[0]     \n"

  1138.                         "fmla   v13.4s, v6.4s, v23.s[1]     \n"

  1139.                         "fmla   v14.4s, v6.4s, v23.s[2]     \n"

  1140.                         "fmla   v15.4s, v6.4s, v23.s[3]     \n"

  1141.                         "fmla   v16.4s, v6.4s, v24.s[0]     \n"

  1142.                         "fmla   v17.4s, v6.4s, v24.s[1]     \n"

  1143.                         "fmla   v18.4s, v6.4s, v24.s[2]     \n"

  1144.                         "fmla   v19.4s, v6.4s, v24.s[3]     \n"

  1145. 

  1146.                         "subs   %w0, %w0, #1                \n"

  1147. 

  1148.                         "fmla   v8.4s, v7.4s, v25.s[0]      \n"

  1149.                         "fmla   v9.4s, v7.4s, v25.s[1]      \n"

  1150.                         "fmla   v10.4s, v7.4s, v25.s[2]     \n"

  1151.                         "fmla   v11.4s, v7.4s, v25.s[3]     \n"

  1152.                         "fmla   v12.4s, v7.4s, v26.s[0]     \n"

  1153.                         "fmla   v13.4s, v7.4s, v26.s[1]     \n"

  1154.                         "fmla   v14.4s, v7.4s, v26.s[2]     \n"

  1155.                         "fmla   v15.4s, v7.4s, v26.s[3]     \n"

  1156.                         "fmla   v16.4s, v7.4s, v27.s[0]     \n"

  1157.                         "fmla   v17.4s, v7.4s, v27.s[1]     \n"

  1158.                         "fmla   v18.4s, v7.4s, v27.s[2]     \n"

  1159.                         "fmla   v19.4s, v7.4s, v27.s[3]     \n"

  1160. 

  1161.                         "bne    0b                          \n"

  1162. 

  1163.                         "st1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n"

  1164.                         "st1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n"

  1165.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  1166. 

  1167.                         : "=r"(nn),         // %0

  1168.                         "=r"(output0_tm), // %1

  1169.                         "=r"(r0),         // %2

  1170.                         "=r"(k0)          // %3

  1171.                         : "0"(nn),

  1172.                         "1"(output0_tm),

  1173.                         "2"(r0),

  1174.                         "3"(k0)

  1175.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27");

  1176.                 }

  1177. #endif

  1178.                 for (; i + 7 < tiles; i += 8)

  1179.                 {

  1180. #if __aarch64__

  1181.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8);

  1182. #else

  1183.                     const float* r0 = bb2.row(i / 8);

  1184. #endif

  1185. 

  1186.                     const float* k0 = kernel0_tm.row(r);

  1187. 

  1188.                     int nn = inch; // inch always > 0

  1189. 

  1190. #if __aarch64__

  1191.                     asm volatile(

  1192.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1193.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  1194.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  1195.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  1196.                         "eor    v20.16b, v20.16b, v20.16b   \n"

  1197.                         "eor    v21.16b, v21.16b, v21.16b   \n"

  1198.                         "eor    v22.16b, v22.16b, v22.16b   \n"

  1199.                         "eor    v23.16b, v23.16b, v23.16b   \n"

  1200. 

  1201.                         "0:                                 \n"

  1202. 

  1203.                         "prfm   pldl1keep, [%2, #512]       \n"

  1204.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3

  1205. 

  1206.                         "prfm   pldl1keep, [%3, #512]       \n"

  1207.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123

  1208. 

  1209.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  1210.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  1211.                         "fmla   v18.4s, v8.4s, v2.s[0]      \n"

  1212.                         "fmla   v19.4s, v8.4s, v3.s[0]      \n"

  1213. 

  1214.                         "prfm   pldl1keep, [%2, #512]       \n"

  1215.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%2], #64 \n" // r4 r5 r6 r7

  1216. 

  1217.                         "fmla   v20.4s, v8.4s, v4.s[0]      \n"

  1218.                         "fmla   v21.4s, v8.4s, v5.s[0]      \n"

  1219.                         "fmla   v22.4s, v8.4s, v6.s[0]      \n"

  1220.                         "fmla   v23.4s, v8.4s, v7.s[0]      \n"

  1221. 

  1222.                         "fmla   v16.4s, v9.4s, v0.s[1]      \n"

  1223.                         "fmla   v17.4s, v9.4s, v1.s[1]      \n"

  1224.                         "fmla   v18.4s, v9.4s, v2.s[1]      \n"

  1225.                         "fmla   v19.4s, v9.4s, v3.s[1]      \n"

  1226.                         "fmla   v20.4s, v9.4s, v4.s[1]      \n"

  1227.                         "fmla   v21.4s, v9.4s, v5.s[1]      \n"

  1228.                         "fmla   v22.4s, v9.4s, v6.s[1]      \n"

  1229.                         "fmla   v23.4s, v9.4s, v7.s[1]      \n"

  1230. 

  1231.                         "fmla   v16.4s, v10.4s, v0.s[2]     \n"

  1232.                         "fmla   v17.4s, v10.4s, v1.s[2]     \n"

  1233.                         "fmla   v18.4s, v10.4s, v2.s[2]     \n"

  1234.                         "fmla   v19.4s, v10.4s, v3.s[2]     \n"

  1235.                         "fmla   v20.4s, v10.4s, v4.s[2]     \n"

  1236.                         "fmla   v21.4s, v10.4s, v5.s[2]     \n"

  1237.                         "fmla   v22.4s, v10.4s, v6.s[2]     \n"

  1238.                         "fmla   v23.4s, v10.4s, v7.s[2]     \n"

  1239. 

  1240.                         "subs   %w0, %w0, #1                \n"

  1241. 

  1242.                         "fmla   v16.4s, v11.4s, v0.s[3]     \n"

  1243.                         "fmla   v17.4s, v11.4s, v1.s[3]     \n"

  1244.                         "fmla   v18.4s, v11.4s, v2.s[3]     \n"

  1245.                         "fmla   v19.4s, v11.4s, v3.s[3]     \n"

  1246.                         "fmla   v20.4s, v11.4s, v4.s[3]     \n"

  1247.                         "fmla   v21.4s, v11.4s, v5.s[3]     \n"

  1248.                         "fmla   v22.4s, v11.4s, v6.s[3]     \n"

  1249.                         "fmla   v23.4s, v11.4s, v7.s[3]     \n"

  1250. 

  1251.                         "bne    0b                          \n"

  1252. 

  1253.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  1254.                         "st1    {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n"

  1255. 

  1256.                         : "=r"(nn),         // %0

  1257.                         "=r"(output0_tm), // %1

  1258.                         "=r"(r0),         // %2

  1259.                         "=r"(k0)          // %3

  1260.                         : "0"(nn),

  1261.                         "1"(output0_tm),

  1262.                         "2"(r0),

  1263.                         "3"(k0)

  1264.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");

  1265. #else

  1266.                     asm volatile(

  1267.                         "veor       q8, q8          \n"

  1268.                         "veor       q9, q9          \n"

  1269.                         "veor       q10, q10        \n"

  1270.                         "veor       q11, q11        \n"

  1271.                         "veor       q12, q12        \n"

  1272.                         "veor       q13, q13        \n"

  1273.                         "veor       q14, q14        \n"

  1274.                         "veor       q15, q15        \n"

  1275. 

  1276.                         "0:                         \n"

  1277. 

  1278.                         "pld        [%2, #512]      \n"

  1279.                         "vldm       %2!, {d0-d7}    \n"

  1280. 

  1281.                         "pld        [%3, #512]      \n"

  1282.                         "vldm       %3!, {d8-d15}   \n"

  1283. 

  1284.                         "vmla.f32   q8, q4, d0[0]   \n"

  1285.                         "vmla.f32   q9, q4, d0[1]   \n"

  1286.                         "vmla.f32   q10, q4, d1[0]  \n"

  1287.                         "vmla.f32   q11, q4, d1[1]  \n"

  1288.                         "vmla.f32   q12, q4, d2[0]  \n"

  1289.                         "vmla.f32   q13, q4, d2[1]  \n"

  1290.                         "vmla.f32   q14, q4, d3[0]  \n"

  1291.                         "vmla.f32   q15, q4, d3[1]  \n"

  1292. 

  1293.                         "vmla.f32   q8, q5, d4[0]   \n"

  1294.                         "vmla.f32   q9, q5, d4[1]   \n"

  1295.                         "vmla.f32   q10, q5, d5[0]  \n"

  1296.                         "vmla.f32   q11, q5, d5[1]  \n"

  1297.                         "vmla.f32   q12, q5, d6[0]  \n"

  1298.                         "vmla.f32   q13, q5, d6[1]  \n"

  1299.                         "vmla.f32   q14, q5, d7[0]  \n"

  1300.                         "vmla.f32   q15, q5, d7[1]  \n"

  1301. 

  1302.                         "pld        [%2, #512]      \n"

  1303.                         "vldm       %2!, {d0-d7}    \n"

  1304. 

  1305.                         "vmla.f32   q8, q6, d0[0]   \n"

  1306.                         "vmla.f32   q9, q6, d0[1]   \n"

  1307.                         "vmla.f32   q10, q6, d1[0]  \n"

  1308.                         "vmla.f32   q11, q6, d1[1]  \n"

  1309.                         "vmla.f32   q12, q6, d2[0]  \n"

  1310.                         "vmla.f32   q13, q6, d2[1]  \n"

  1311.                         "vmla.f32   q14, q6, d3[0]  \n"

  1312.                         "vmla.f32   q15, q6, d3[1]  \n"

  1313. 

  1314.                         "subs       %0, %0, #1      \n"

  1315. 

  1316.                         "vmla.f32   q8, q7, d4[0]   \n"

  1317.                         "vmla.f32   q9, q7, d4[1]   \n"

  1318.                         "vmla.f32   q10, q7, d5[0]  \n"

  1319.                         "vmla.f32   q11, q7, d5[1]  \n"

  1320.                         "vmla.f32   q12, q7, d6[0]  \n"

  1321.                         "vmla.f32   q13, q7, d6[1]  \n"

  1322.                         "vmla.f32   q14, q7, d7[0]  \n"

  1323.                         "vmla.f32   q15, q7, d7[1]  \n"

  1324. 

  1325.                         "bne        0b              \n"

  1326. 

  1327.                         "vstm       %1!, {d16-d23}  \n"

  1328.                         "vstm       %1!, {d24-d31}  \n"

  1329. 

  1330.                         : "=r"(nn),         // %0

  1331.                         "=r"(output0_tm), // %1

  1332.                         "=r"(r0),         // %2

  1333.                         "=r"(k0)          // %3

  1334.                         : "0"(nn),

  1335.                         "1"(output0_tm),

  1336.                         "2"(r0),

  1337.                         "3"(k0)

  1338.                         : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  1339. #endif

  1340.                 }

  1341.                 for (; i + 3 < tiles; i += 4)

  1342.                 {

  1343. #if __aarch64__

  1344.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);

  1345. #else

  1346.                     const float* r0 = bb2.row(i / 8 + (i % 8) / 4);

  1347. #endif

  1348. 

  1349.                     const float* k0 = kernel0_tm.row(r);

  1350. 

  1351.                     int nn = inch; // inch always > 0

  1352. 

  1353. #if __aarch64__

  1354.                     asm volatile(

  1355.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1356.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  1357.                         "eor    v18.16b, v18.16b, v18.16b   \n"

  1358.                         "eor    v19.16b, v19.16b, v19.16b   \n"

  1359. 

  1360.                         "0:                                 \n"

  1361. 

  1362.                         "prfm   pldl1keep, [%2, #512]       \n"

  1363.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3

  1364. 

  1365.                         "prfm   pldl1keep, [%3, #512]       \n"

  1366.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123

  1367. 

  1368.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  1369.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  1370.                         "fmla   v18.4s, v8.4s, v2.s[0]      \n"

  1371.                         "fmla   v19.4s, v8.4s, v3.s[0]      \n"

  1372. 

  1373.                         "fmla   v16.4s, v9.4s, v0.s[1]      \n"

  1374.                         "fmla   v17.4s, v9.4s, v1.s[1]      \n"

  1375.                         "fmla   v18.4s, v9.4s, v2.s[1]      \n"

  1376.                         "fmla   v19.4s, v9.4s, v3.s[1]      \n"

  1377. 

  1378.                         "fmla   v16.4s, v10.4s, v0.s[2]     \n"

  1379.                         "fmla   v17.4s, v10.4s, v1.s[2]     \n"

  1380.                         "fmla   v18.4s, v10.4s, v2.s[2]     \n"

  1381.                         "fmla   v19.4s, v10.4s, v3.s[2]     \n"

  1382. 

  1383.                         "subs   %w0, %w0, #1                \n"

  1384. 

  1385.                         "fmla   v16.4s, v11.4s, v0.s[3]     \n"

  1386.                         "fmla   v17.4s, v11.4s, v1.s[3]     \n"

  1387.                         "fmla   v18.4s, v11.4s, v2.s[3]     \n"

  1388.                         "fmla   v19.4s, v11.4s, v3.s[3]     \n"

  1389. 

  1390.                         "bne    0b                          \n"

  1391. 

  1392.                         "st1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n"

  1393. 

  1394.                         : "=r"(nn),         // %0

  1395.                         "=r"(output0_tm), // %1

  1396.                         "=r"(r0),         // %2

  1397.                         "=r"(k0)          // %3

  1398.                         : "0"(nn),

  1399.                         "1"(output0_tm),

  1400.                         "2"(r0),

  1401.                         "3"(k0)

  1402.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19");

  1403. #else

  1404.                     asm volatile(

  1405.                         "veor       q8, q8          \n"

  1406.                         "veor       q9, q9          \n"

  1407.                         "veor       q10, q10        \n"

  1408.                         "veor       q11, q11        \n"

  1409. 

  1410.                         "0:                         \n"

  1411. 

  1412.                         "pld        [%2, #512]      \n"

  1413.                         "vldm       %2!, {d0-d7}    \n"

  1414. 

  1415.                         "pld        [%3, #512]      \n"

  1416.                         "vldm       %3!, {d8-d15}   \n"

  1417. 

  1418.                         "vmla.f32   q8, q4, d0[0]   \n"

  1419.                         "vmla.f32   q9, q4, d2[0]   \n"

  1420.                         "vmla.f32   q10, q4, d4[0]  \n"

  1421.                         "vmla.f32   q11, q4, d6[0]  \n"

  1422. 

  1423.                         "vmla.f32   q8, q5, d0[1]   \n"

  1424.                         "vmla.f32   q9, q5, d2[1]   \n"

  1425.                         "vmla.f32   q10, q5, d4[1]  \n"

  1426.                         "vmla.f32   q11, q5, d6[1]  \n"

  1427. 

  1428.                         "vmla.f32   q8, q6, d1[0]   \n"

  1429.                         "vmla.f32   q9, q6, d3[0]   \n"

  1430.                         "vmla.f32   q10, q6, d5[0]  \n"

  1431.                         "vmla.f32   q11, q6, d7[0]  \n"

  1432. 

  1433.                         "subs       %0, %0, #1      \n"

  1434. 

  1435.                         "vmla.f32   q8, q7, d1[1]   \n"

  1436.                         "vmla.f32   q9, q7, d3[1]   \n"

  1437.                         "vmla.f32   q10, q7, d5[1]  \n"

  1438.                         "vmla.f32   q11, q7, d7[1]  \n"

  1439. 

  1440.                         "bne        0b              \n"

  1441. 

  1442.                         "vstm       %1!, {d16-d23}  \n"

  1443. 

  1444.                         : "=r"(nn),         // %0

  1445.                         "=r"(output0_tm), // %1

  1446.                         "=r"(r0),         // %2

  1447.                         "=r"(k0)          // %3

  1448.                         : "0"(nn),

  1449.                         "1"(output0_tm),

  1450.                         "2"(r0),

  1451.                         "3"(k0)

  1452.                         : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11");

  1453. #endif

  1454.                 }

  1455.                 for (; i + 1 < tiles; i += 2)

  1456.                 {

  1457. #if __aarch64__

  1458.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);

  1459. #else

  1460.                     const float* r0 = bb2.row(i / 8 + (i % 8) / 4 + (i % 4) / 2);

  1461. #endif

  1462. 

  1463.                     const float* k0 = kernel0_tm.row(r);

  1464. 

  1465.                     int nn = inch; // inch always > 0

  1466. 

  1467. #if __aarch64__

  1468.                     asm volatile(

  1469.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1470.                         "eor    v17.16b, v17.16b, v17.16b   \n"

  1471. 

  1472.                         "0:                                 \n"

  1473. 

  1474.                         "prfm   pldl1keep, [%2, #256]       \n"

  1475.                         "ld1    {v0.4s, v1.4s}, [%2], #32   \n" // r0 r1

  1476. 

  1477.                         "prfm   pldl1keep, [%3, #512]       \n"

  1478.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123

  1479. 

  1480.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  1481.                         "fmla   v17.4s, v8.4s, v1.s[0]      \n"

  1482. 

  1483.                         "fmla   v16.4s, v9.4s, v0.s[1]      \n"

  1484.                         "fmla   v17.4s, v9.4s, v1.s[1]      \n"

  1485. 

  1486.                         "fmla   v16.4s, v10.4s, v0.s[2]     \n"

  1487.                         "fmla   v17.4s, v10.4s, v1.s[2]     \n"

  1488. 

  1489.                         "subs   %w0, %w0, #1                \n"

  1490. 

  1491.                         "fmla   v16.4s, v11.4s, v0.s[3]     \n"

  1492.                         "fmla   v17.4s, v11.4s, v1.s[3]     \n"

  1493. 

  1494.                         "bne    0b                          \n"

  1495. 

  1496.                         "st1    {v16.4s, v17.4s}, [%1], #32 \n"

  1497. 

  1498.                         : "=r"(nn),         // %0

  1499.                         "=r"(output0_tm), // %1

  1500.                         "=r"(r0),         // %2

  1501.                         "=r"(k0)          // %3

  1502.                         : "0"(nn),

  1503.                         "1"(output0_tm),

  1504.                         "2"(r0),

  1505.                         "3"(k0)

  1506.                         : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v16", "v17");

  1507. #else

  1508.                     asm volatile(

  1509.                         "veor       q8, q8          \n"

  1510.                         "veor       q9, q9          \n"

  1511. 

  1512.                         "0:                         \n"

  1513. 

  1514.                         "pld        [%2, #256]      \n"

  1515.                         "vld1.f32   {d0-d3}, [%2 :128]! \n"

  1516. 

  1517.                         "pld        [%3, #512]      \n"

  1518.                         "vldm       %3!, {d8-d15}   \n"

  1519. 

  1520.                         "vmla.f32   q8, q4, d0[0]   \n"

  1521.                         "vmla.f32   q9, q4, d2[0]   \n"

  1522. 

  1523.                         "vmla.f32   q8, q5, d0[1]   \n"

  1524.                         "vmla.f32   q9, q5, d2[1]   \n"

  1525. 

  1526.                         "vmla.f32   q8, q6, d1[0]   \n"

  1527.                         "vmla.f32   q9, q6, d3[0]   \n"

  1528. 

  1529.                         "subs       %0, %0, #1      \n"

  1530. 

  1531.                         "vmla.f32   q8, q7, d1[1]   \n"

  1532.                         "vmla.f32   q9, q7, d3[1]   \n"

  1533. 

  1534.                         "bne        0b              \n"

  1535. 

  1536.                         "vst1.f32   {d16-d19}, [%1 :128]! \n"

  1537. 

  1538.                         : "=r"(nn),         // %0

  1539.                         "=r"(output0_tm), // %1

  1540.                         "=r"(r0),         // %2

  1541.                         "=r"(k0)          // %3

  1542.                         : "0"(nn),

  1543.                         "1"(output0_tm),

  1544.                         "2"(r0),

  1545.                         "3"(k0)

  1546.                         : "cc", "memory", "q0", "q1", "q4", "q5", "q6", "q7", "q8", "q9");

  1547. #endif

  1548.                 }

  1549.                 for (; i < tiles; i++)

  1550.                 {

  1551. #if __aarch64__

  1552.                     const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);

  1553. #else

  1554.                     const float* r0 = bb2.row(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);

  1555. #endif

  1556. 

  1557.                     const float* k0 = kernel0_tm.row(r);

  1558. 

  1559.                     int nn = inch; // inch always > 0

  1560. 

  1561. #if __aarch64__

  1562.                     asm volatile(

  1563.                         "eor    v16.16b, v16.16b, v16.16b   \n"

  1564. 

  1565.                         "0:                                 \n"

  1566. 

  1567.                         "prfm   pldl1keep, [%2, #128]       \n"

  1568.                         "ld1    {v0.4s}, [%2], #16          \n" // r0

  1569. 

  1570.                         "prfm   pldl1keep, [%3, #512]       \n"

  1571.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123

  1572. 

  1573.                         "fmla   v16.4s, v8.4s, v0.s[0]      \n"

  1574.                         "fmla   v16.4s, v9.4s, v0.s[1]      \n"

  1575. 

  1576.                         "subs   %w0, %w0, #1                \n"

  1577. 

  1578.                         "fmla   v16.4s, v10.4s, v0.s[2]     \n"

  1579.                         "fmla   v16.4s, v11.4s, v0.s[3]     \n"

  1580. 

  1581.                         "bne    0b                          \n"

  1582. 

  1583.                         "st1    {v16.4s}, [%1], #16         \n"

  1584. 

  1585.                         : "=r"(nn),         // %0

  1586.                         "=r"(output0_tm), // %1

  1587.                         "=r"(r0),         // %2

  1588.                         "=r"(k0)          // %3

  1589.                         : "0"(nn),

  1590.                         "1"(output0_tm),

  1591.                         "2"(r0),

  1592.                         "3"(k0)

  1593.                         : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v16");

  1594. #else

  1595.                     asm volatile(

  1596.                         "veor       q8, q8          \n"

  1597. 

  1598.                         "0:                         \n"

  1599. 

  1600.                         "pld        [%2, #128]      \n"

  1601.                         "vld1.f32   {d0-d1}, [%2 :128]! \n"

  1602. 

  1603.                         "pld        [%3, #512]      \n"

  1604.                         "vldm       %3!, {d8-d15}   \n"

  1605. 

  1606.                         "vmla.f32   q8, q4, d0[0]   \n"

  1607.                         "vmla.f32   q8, q5, d0[1]   \n"

  1608. 

  1609.                         "subs       %0, %0, #1      \n"

  1610. 

  1611.                         "vmla.f32   q8, q6, d1[0]   \n"

  1612.                         "vmla.f32   q8, q7, d1[1]   \n"

  1613. 

  1614.                         "bne        0b              \n"

  1615. 

  1616.                         "vst1.f32   {d16-d17}, [%1 :128]! \n"

  1617. 

  1618.                         : "=r"(nn),         // %0

  1619.                         "=r"(output0_tm), // %1

  1620.                         "=r"(r0),         // %2

  1621.                         "=r"(k0)          // %3

  1622.                         : "0"(nn),

  1623.                         "1"(output0_tm),

  1624.                         "2"(r0),

  1625.                         "3"(k0)

  1626.                         : "cc", "memory", "q0", "q4", "q5", "q6", "q7", "q8");

  1627. #endif

  1628.                 }

  1629.             }

  1630.         }

  1631.     }

  1632.     bottom_blob_tm = Mat();

  1633.     // END dot

  1634. 

  1635.     // BEGIN transform output

  1636.     Mat top_blob_bordered;

  1637.     if (outw == top_blob.w && outh == top_blob.h)

  1638.     {

  1639.         top_blob_bordered = top_blob;

  1640.     }

  1641.     else

  1642.     {

  1643.         top_blob_bordered.create(outw, outh, outch, elemsize, elempack, opt.workspace_allocator);

  1644.     }

  1645.     {

  1646.         //         const float otm[6][8] = {

  1647.         //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  1648.         //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  1649.         //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  1650.         //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  1651.         //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  1652.         //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  1653.         //         };

  1654. 

  1655.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  1656.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  1657.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  1658.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  1659.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  1660.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  1661. 

  1662.         int w_tm = outw / 6 * 8;

  1663.         int h_tm = outh / 6 * 8;

  1664.         const int tiles = w_tm / 8 * h_tm / 8;

  1665. 

  1666.         #pragma omp parallel for num_threads(opt.num_threads)

  1667.         for (int p = 0; p < outch; p++)

  1668.         {

  1669.             const Mat out0_tm = top_blob_tm.channel(p);

  1670.             Mat out0 = top_blob_bordered.channel(p);

  1671. 

  1672.             //             const float bias0 = bias ? bias[p] : 0.f;

  1673.             float32x4_t _bias0 = bias ? vld1q_f32((const float*)bias + p * 4) : vdupq_n_f32(0.f);

  1674. 

  1675.             float tmp[6][8][4];

  1676. 

  1677.             // tile

  1678.             for (int i = 0; i < outh / 6; i++)

  1679.             {

  1680.                 for (int j = 0; j < outw / 6; j++)

  1681.                 {

  1682.                     //                     top_blob_tm.create(tiles, 64, outch, elemsize, elempack);

  1683. 

  1684.                     const float* output0_tm_0 = (const float*)out0_tm + (i * w_tm / 8 + j) * 4;

  1685.                     const float* output0_tm_1 = output0_tm_0 + tiles * 4;

  1686.                     const float* output0_tm_2 = output0_tm_0 + tiles * 8;

  1687.                     const float* output0_tm_3 = output0_tm_0 + tiles * 12;

  1688.                     const float* output0_tm_4 = output0_tm_0 + tiles * 16;

  1689.                     const float* output0_tm_5 = output0_tm_0 + tiles * 20;

  1690.                     const float* output0_tm_6 = output0_tm_0 + tiles * 24;

  1691.                     const float* output0_tm_7 = output0_tm_0 + tiles * 28;

  1692. 

  1693.                     unsigned short* output0 = out0.row<unsigned short>(i * 6) + (j * 6) * 4;

  1694. 

  1695.                     // TODO neon optimize

  1696.                     for (int m = 0; m < 8; m++)

  1697.                     {

  1698.                         float32x4_t _out0tm0 = vld1q_f32(output0_tm_0);

  1699.                         float32x4_t _out0tm1 = vld1q_f32(output0_tm_1);

  1700.                         float32x4_t _out0tm2 = vld1q_f32(output0_tm_2);

  1701.                         float32x4_t _out0tm3 = vld1q_f32(output0_tm_3);

  1702.                         float32x4_t _out0tm4 = vld1q_f32(output0_tm_4);

  1703.                         float32x4_t _out0tm5 = vld1q_f32(output0_tm_5);

  1704.                         float32x4_t _out0tm6 = vld1q_f32(output0_tm_6);

  1705.                         float32x4_t _out0tm7 = vld1q_f32(output0_tm_7);

  1706. 

  1707.                         float32x4_t _tmp024a = vaddq_f32(_out0tm1, _out0tm2);

  1708.                         float32x4_t _tmp135a = vsubq_f32(_out0tm1, _out0tm2);

  1709. 

  1710.                         //                         float tmp024a = output0_tm[1] + output0_tm[2];

  1711.                         //                         float tmp135a = output0_tm[1] - output0_tm[2];

  1712. 

  1713.                         float32x4_t _tmp024b = vaddq_f32(_out0tm3, _out0tm4);

  1714.                         float32x4_t _tmp135b = vsubq_f32(_out0tm3, _out0tm4);

  1715. 

  1716.                         //                         float tmp024b = output0_tm[3] + output0_tm[4];

  1717.                         //                         float tmp135b = output0_tm[3] - output0_tm[4];

  1718. 

  1719.                         float32x4_t _tmp024c = vaddq_f32(_out0tm5, _out0tm6);

  1720.                         float32x4_t _tmp135c = vsubq_f32(_out0tm5, _out0tm6);

  1721. 

  1722.                         //                         float tmp024c = output0_tm[5] + output0_tm[6];

  1723.                         //                         float tmp135c = output0_tm[5] - output0_tm[6];

  1724. 

  1725.                         float32x4_t _tmp0m = vaddq_f32(vaddq_f32(_out0tm0, _tmp024a), vmlaq_n_f32(_tmp024b, _tmp024c, 32.f));

  1726.                         float32x4_t _tmp2m = vmlaq_n_f32(vmlaq_n_f32(_tmp024a, _tmp024b, 4.f), _tmp024c, 8.f);

  1727.                         float32x4_t _tmp4m = vmlaq_n_f32(vmlaq_n_f32(_tmp024a, _tmp024b, 16.f), _tmp024c, 2.f);

  1728.                         vst1q_f32(tmp[0][m], _tmp0m);

  1729.                         vst1q_f32(tmp[2][m], _tmp2m);

  1730.                         vst1q_f32(tmp[4][m], _tmp4m);

  1731. 

  1732.                         //                         tmp[0][m] = output0_tm[0] + tmp024a + tmp024b + tmp024c * 32;

  1733.                         //                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  1734.                         //                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  1735. 

  1736.                         float32x4_t _tmp1m = vmlaq_n_f32(vmlaq_n_f32(_tmp135a, _tmp135b, 2.f), _tmp135c, 16.f);

  1737.                         float32x4_t _tmp3m = vmlaq_n_f32(vmlaq_n_f32(_tmp135a, _tmp135b, 8.f), _tmp135c, 4.f);

  1738.                         float32x4_t _tmp5m = vaddq_f32(vaddq_f32(_out0tm7, _tmp135a), vmlaq_n_f32(_tmp135c, _tmp135b, 32.f));

  1739.                         vst1q_f32(tmp[1][m], _tmp1m);

  1740.                         vst1q_f32(tmp[3][m], _tmp3m);

  1741.                         vst1q_f32(tmp[5][m], _tmp5m);

  1742. 

  1743.                         //                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  1744.                         //                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  1745.                         //                         tmp[5][m] = output0_tm[7] + tmp135a + tmp135b * 32 + tmp135c;

  1746. 

  1747.                         output0_tm_0 += tiles * 32;

  1748.                         output0_tm_1 += tiles * 32;

  1749.                         output0_tm_2 += tiles * 32;

  1750.                         output0_tm_3 += tiles * 32;

  1751.                         output0_tm_4 += tiles * 32;

  1752.                         output0_tm_5 += tiles * 32;

  1753.                         output0_tm_6 += tiles * 32;

  1754.                         output0_tm_7 += tiles * 32;

  1755.                     }

  1756. 

  1757.                     for (int m = 0; m < 6; m++)

  1758.                     {

  1759.                         float32x4_t _tmp00 = vld1q_f32(tmp[m][0]);

  1760.                         float32x4_t _tmp01 = vld1q_f32(tmp[m][1]);

  1761.                         float32x4_t _tmp02 = vld1q_f32(tmp[m][2]);

  1762.                         float32x4_t _tmp03 = vld1q_f32(tmp[m][3]);

  1763.                         float32x4_t _tmp04 = vld1q_f32(tmp[m][4]);

  1764.                         float32x4_t _tmp05 = vld1q_f32(tmp[m][5]);

  1765.                         float32x4_t _tmp06 = vld1q_f32(tmp[m][6]);

  1766.                         float32x4_t _tmp07 = vld1q_f32(tmp[m][7]);

  1767. 

  1768.                         float32x4_t _tmp024a = vaddq_f32(_tmp01, _tmp02);

  1769.                         float32x4_t _tmp135a = vsubq_f32(_tmp01, _tmp02);

  1770. 

  1771.                         //                         float tmp024a = tmp0[1] + tmp0[2];

  1772.                         //                         float tmp135a = tmp0[1] - tmp0[2];

  1773. 

  1774.                         float32x4_t _tmp024b = vaddq_f32(_tmp03, _tmp04);

  1775.                         float32x4_t _tmp135b = vsubq_f32(_tmp03, _tmp04);

  1776. 

  1777.                         //                         float tmp024b = tmp0[3] + tmp0[4];

  1778.                         //                         float tmp135b = tmp0[3] - tmp0[4];

  1779. 

  1780.                         float32x4_t _tmp024c = vaddq_f32(_tmp05, _tmp06);

  1781.                         float32x4_t _tmp135c = vsubq_f32(_tmp05, _tmp06);

  1782. 

  1783.                         //                         float tmp024c = tmp0[5] + tmp0[6];

  1784.                         //                         float tmp135c = tmp0[5] - tmp0[6];

  1785. 

  1786.                         float32x4_t _out00 = vaddq_f32(_bias0, vaddq_f32(vaddq_f32(_tmp00, _tmp024a), vmlaq_n_f32(_tmp024b, _tmp024c, 32.f)));

  1787.                         float32x4_t _out02 = vaddq_f32(_bias0, vmlaq_n_f32(vmlaq_n_f32(_tmp024a, _tmp024b, 4.f), _tmp024c, 8.f));

  1788.                         float32x4_t _out04 = vaddq_f32(_bias0, vmlaq_n_f32(vmlaq_n_f32(_tmp024a, _tmp024b, 16.f), _tmp024c, 2.f));

  1789.                         vst1_u16(output0, vcvt_bf16_f32(_out00));

  1790.                         vst1_u16(output0 + 8, vcvt_bf16_f32(_out02));

  1791.                         vst1_u16(output0 + 16, vcvt_bf16_f32(_out04));

  1792. 

  1793.                         //                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  1794.                         //                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  1795.                         //                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  1796. 

  1797.                         float32x4_t _out01 = vaddq_f32(_bias0, vmlaq_n_f32(vmlaq_n_f32(_tmp135a, _tmp135b, 2.f), _tmp135c, 16.f));

  1798.                         float32x4_t _out03 = vaddq_f32(_bias0, vmlaq_n_f32(vmlaq_n_f32(_tmp135a, _tmp135b, 8.f), _tmp135c, 4.f));

  1799.                         float32x4_t _out05 = vaddq_f32(_bias0, vaddq_f32(vaddq_f32(_tmp07, _tmp135a), vmlaq_n_f32(_tmp135c, _tmp135b, 32.f)));

  1800.                         vst1_u16(output0 + 4, vcvt_bf16_f32(_out01));

  1801.                         vst1_u16(output0 + 12, vcvt_bf16_f32(_out03));

  1802.                         vst1_u16(output0 + 20, vcvt_bf16_f32(_out05));

  1803. 

  1804.                         //                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  1805.                         //                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  1806.                         //                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  1807. 

  1808.                         output0 += outw * 4;

  1809.                     }

  1810.                 }

  1811.             }

  1812.         }

  1813.     }

  1814.     // END transform output

  1815. 

  1816.     // cut result pad

  1817.     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt);

  1818. }

  1819. 

  1820. static void conv3x3s2_pack4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)

  1821. {

  1822.     int w = bottom_blob.w;

  1823.     int inch = bottom_blob.c;

  1824.     int outw = top_blob.w;

  1825.     int outh = top_blob.h;

  1826.     int outch = top_blob.c;

  1827. 

  1828.     Mat top_blob_fp32(outw, outh, opt.num_threads, (size_t)4u * 4, 4, opt.workspace_allocator);

  1829. 

  1830.     const int tailstep = (w - 2 * outw + w) * 4;

  1831. 

  1832.     const float* bias = _bias;

  1833. 

  1834.     #pragma omp parallel for num_threads(opt.num_threads)

  1835.     for (int p = 0; p < outch; p++)

  1836.     {

  1837.         Mat out0 = top_blob_fp32.channel(get_omp_thread_num());

  1838. 

  1839.         float32x4_t _bias0 = bias ? vld1q_f32((const float*)bias + p * 4) : vdupq_n_f32(0.f);

  1840.         out0.fill(_bias0);

  1841. 

  1842.         int q = 0;

  1843.         for (; q < inch - 1; q++)

  1844.         {

  1845.             float* outptr0 = out0.row(0);

  1846. 

  1847.             const Mat img0 = bottom_blob.channel(q);

  1848. 

  1849.             const unsigned short* r0 = img0.row<const unsigned short>(0);

  1850.             const unsigned short* r1 = img0.row<const unsigned short>(1);

  1851.             const unsigned short* r2 = img0.row<const unsigned short>(2);

  1852. 

  1853.             const unsigned short* kptr = (const unsigned short*)kernel.channel(p).row<const unsigned short>(q);

  1854. 

  1855. #if __aarch64__

  1856.             // 16 * 9

  1857.             uint16x8_t _k00_01 = vld1q_u16(kptr);

  1858.             uint16x8_t _k00_23 = vld1q_u16(kptr + 8);

  1859.             uint16x8_t _k01_01 = vld1q_u16(kptr + 16);

  1860.             uint16x8_t _k01_23 = vld1q_u16(kptr + 24);

  1861.             uint16x8_t _k02_01 = vld1q_u16(kptr + 32);

  1862.             uint16x8_t _k02_23 = vld1q_u16(kptr + 40);

  1863.             uint16x8_t _k10_01 = vld1q_u16(kptr + 48);

  1864.             uint16x8_t _k10_23 = vld1q_u16(kptr + 56);

  1865.             uint16x8_t _k11_01 = vld1q_u16(kptr + 64);

  1866.             uint16x8_t _k11_23 = vld1q_u16(kptr + 72);

  1867.             uint16x8_t _k12_01 = vld1q_u16(kptr + 80);

  1868.             uint16x8_t _k12_23 = vld1q_u16(kptr + 88);

  1869.             uint16x8_t _k20_01 = vld1q_u16(kptr + 96);

  1870.             uint16x8_t _k20_23 = vld1q_u16(kptr + 104);

  1871.             uint16x8_t _k21_01 = vld1q_u16(kptr + 112);

  1872.             uint16x8_t _k21_23 = vld1q_u16(kptr + 120);

  1873.             uint16x8_t _k22_01 = vld1q_u16(kptr + 128);

  1874.             uint16x8_t _k22_23 = vld1q_u16(kptr + 136);

  1875. #endif // __aarch64__

  1876. 

  1877.             int i = 0;

  1878.             for (; i < outh; i++)

  1879.             {

  1880.                 int j = 0;

  1881.                 for (; j + 3 < outw; j += 4)

  1882.                 {

  1883. #if __aarch64__

  1884.                     asm volatile(

  1885.                         "prfm   pldl1keep, [%0, #512]       \n"

  1886.                         "ld1    {v10.4s, v11.4s, v12.4s, v13.4s}, [%0] \n" // sum0 sum1 sum2 sum3

  1887. 

  1888.                         "prfm   pldl1keep, [%1, #512]       \n"

  1889.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%1], #64 \n" // r00 r01 r02 r03

  1890. 

  1891.                         "shll   v0.4s, v4.4h, #16           \n"

  1892.                         "shll2  v1.4s, v4.8h, #16           \n"

  1893.                         "shll   v2.4s, v5.4h, #16           \n"

  1894.                         "shll2  v3.4s, v5.8h, #16           \n"

  1895. 

  1896.                         "shll   v4.4s, v6.4h, #16           \n"

  1897.                         "shll2  v5.4s, v6.8h, #16           \n"

  1898.                         "shll   v6.4s, v7.4h, #16           \n"

  1899.                         "shll2  v7.4s, v7.8h, #16           \n"

  1900. 

  1901.                         "shll   v8.4s, %8.4h, #16           \n"

  1902.                         "shll2  v9.4s, %8.8h, #16           \n"

  1903. 

  1904.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  1905.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  1906.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  1907.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  1908.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  1909.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  1910.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  1911.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  1912. 

  1913.                         "shll   v8.4s, %9.4h, #16           \n"

  1914.                         "shll2  v9.4s, %9.8h, #16           \n"

  1915. 

  1916.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  1917.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  1918.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  1919.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  1920.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  1921.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  1922.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  1923.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  1924. 

  1925.                         "shll   v8.4s, %10.4h, #16          \n"

  1926.                         "shll2  v9.4s, %10.8h, #16          \n"

  1927. 

  1928.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  1929.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  1930.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  1931.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  1932.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  1933.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  1934.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  1935.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  1936. 

  1937.                         "shll   v8.4s, %11.4h, #16          \n"

  1938.                         "shll2  v9.4s, %11.8h, #16          \n"

  1939. 

  1940.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  1941.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  1942.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  1943.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  1944.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  1945.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  1946.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  1947.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  1948. 

  1949.                         "prfm   pldl1keep, [%1, #64]        \n"

  1950.                         "ld1    {v0.4h}, [%1]               \n" // r08

  1951. 

  1952.                         "shll   v0.4s, v0.4h, #16           \n"

  1953. 

  1954.                         "shll   v8.4s, %12.4h, #16          \n"

  1955.                         "shll2  v9.4s, %12.8h, #16          \n"

  1956. 

  1957.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  1958.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  1959.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  1960.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  1961.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  1962.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  1963.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  1964.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  1965. 

  1966.                         "shll   v8.4s, %13.4h, #16          \n"

  1967.                         "shll2  v9.4s, %13.8h, #16          \n"

  1968. 

  1969.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  1970.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  1971.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  1972.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  1973.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  1974.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  1975.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  1976.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  1977. 

  1978.                         "prfm   pldl1keep, [%2, #512]       \n"

  1979.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r10 r11 r12 r13

  1980. 

  1981.                         "shll   v0.4s, v4.4h, #16           \n"

  1982.                         "shll2  v1.4s, v4.8h, #16           \n"

  1983.                         "shll   v2.4s, v5.4h, #16           \n"

  1984.                         "shll2  v3.4s, v5.8h, #16           \n"

  1985. 

  1986.                         "shll   v4.4s, v6.4h, #16           \n"

  1987.                         "shll2  v5.4s, v6.8h, #16           \n"

  1988.                         "shll   v6.4s, v7.4h, #16           \n"

  1989.                         "shll2  v7.4s, v7.8h, #16           \n"

  1990. 

  1991.                         "shll   v8.4s, %14.4h, #16          \n"

  1992.                         "shll2  v9.4s, %14.8h, #16          \n"

  1993. 

  1994.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  1995.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  1996.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  1997.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  1998.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  1999.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  2000.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  2001.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  2002. 

  2003.                         "shll   v8.4s, %15.4h, #16          \n"

  2004.                         "shll2  v9.4s, %15.8h, #16          \n"

  2005. 

  2006.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  2007.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  2008.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  2009.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  2010.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  2011.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  2012.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  2013.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  2014. 

  2015.                         "shll   v8.4s, %16.4h, #16          \n"

  2016.                         "shll2  v9.4s, %16.8h, #16          \n"

  2017. 

  2018.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  2019.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  2020.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  2021.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  2022.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  2023.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  2024.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  2025.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  2026. 

  2027.                         "shll   v8.4s, %17.4h, #16          \n"

  2028.                         "shll2  v9.4s, %17.8h, #16          \n"

  2029. 

  2030.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  2031.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  2032.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  2033.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  2034.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  2035.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  2036.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  2037.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  2038. 

  2039.                         "prfm   pldl1keep, [%2, #64]        \n"

  2040.                         "ld1    {v0.4h}, [%2]               \n" // r18

  2041. 

  2042.                         "shll   v0.4s, v0.4h, #16           \n"

  2043. 

  2044.                         "shll   v8.4s, %18.4h, #16          \n"

  2045.                         "shll2  v9.4s, %18.8h, #16          \n"

  2046. 

  2047.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  2048.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  2049.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  2050.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  2051.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  2052.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  2053.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  2054.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  2055. 

  2056.                         "shll   v8.4s, %19.4h, #16          \n"

  2057.                         "shll2  v9.4s, %19.8h, #16          \n"

  2058. 

  2059.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  2060.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  2061.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  2062.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  2063.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  2064.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  2065.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  2066.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  2067. 

  2068.                         "prfm   pldl1keep, [%3, #512]       \n"

  2069.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%3], #64 \n" // r20 r21 r22 r23

  2070. 

  2071.                         "shll   v0.4s, v4.4h, #16           \n"

  2072.                         "shll2  v1.4s, v4.8h, #16           \n"

  2073.                         "shll   v2.4s, v5.4h, #16           \n"

  2074.                         "shll2  v3.4s, v5.8h, #16           \n"

  2075. 

  2076.                         "shll   v4.4s, v6.4h, #16           \n"

  2077.                         "shll2  v5.4s, v6.8h, #16           \n"

  2078.                         "shll   v6.4s, v7.4h, #16           \n"

  2079.                         "shll2  v7.4s, v7.8h, #16           \n"

  2080. 

  2081.                         "shll   v8.4s, %20.4h, #16          \n"

  2082.                         "shll2  v9.4s, %20.8h, #16          \n"

  2083. 

  2084.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  2085.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  2086.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  2087.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  2088.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  2089.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  2090.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  2091.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  2092. 

  2093.                         "shll   v8.4s, %21.4h, #16          \n"

  2094.                         "shll2  v9.4s, %21.8h, #16          \n"

  2095. 

  2096.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  2097.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  2098.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  2099.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  2100.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  2101.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  2102.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  2103.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  2104. 

  2105.                         "shll   v8.4s, %22.4h, #16          \n"

  2106.                         "shll2  v9.4s, %22.8h, #16          \n"

  2107. 

  2108.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  2109.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  2110.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  2111.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  2112.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  2113.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  2114.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  2115.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  2116. 

  2117.                         "shll   v8.4s, %23.4h, #16          \n"

  2118.                         "shll2  v9.4s, %23.8h, #16          \n"

  2119. 

  2120.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  2121.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  2122.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  2123.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  2124.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  2125.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  2126.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  2127.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  2128. 

  2129.                         "prfm   pldl1keep, [%3, #64]        \n"

  2130.                         "ld1    {v0.4h}, [%3]               \n" // r28

  2131. 

  2132.                         "shll   v0.4s, v0.4h, #16           \n"

  2133. 

  2134.                         "shll   v8.4s, %24.4h, #16          \n"

  2135.                         "shll2  v9.4s, %24.8h, #16          \n"

  2136. 

  2137.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  2138.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  2139.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  2140.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  2141.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  2142.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  2143.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  2144.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  2145. 

  2146.                         "shll   v8.4s, %25.4h, #16          \n"

  2147.                         "shll2  v9.4s, %25.8h, #16          \n"

  2148. 

  2149.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  2150.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  2151.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  2152.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  2153.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  2154.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  2155.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  2156.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  2157. 

  2158.                         "st1    {v10.4s, v11.4s, v12.4s, v13.4s}, [%0], #64 \n"

  2159. 

  2160.                         : "=r"(outptr0), // %0

  2161.                         "=r"(r0),      // %1

  2162.                         "=r"(r1),      // %2

  2163.                         "=r"(r2)       // %3

  2164.                         : "0"(outptr0),

  2165.                         "1"(r0),

  2166.                         "2"(r1),

  2167.                         "3"(r2),

  2168.                         "w"(_k00_01), // %8

  2169.                         "w"(_k00_23), // %9

  2170.                         "w"(_k01_01), // %10

  2171.                         "w"(_k01_23), // %11

  2172.                         "w"(_k02_01), // %12

  2173.                         "w"(_k02_23), // %13

  2174.                         "w"(_k10_01), // %14

  2175.                         "w"(_k10_23), // %15

  2176.                         "w"(_k11_01), // %16

  2177.                         "w"(_k11_23), // %17

  2178.                         "w"(_k12_01), // %18

  2179.                         "w"(_k12_23), // %19

  2180.                         "w"(_k20_01), // %20

  2181.                         "w"(_k20_23), // %21

  2182.                         "w"(_k21_01), // %22

  2183.                         "w"(_k21_23), // %23

  2184.                         "w"(_k22_01), // %24

  2185.                         "w"(_k22_23)  // %25

  2186.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  2187. #else  // __aarch64__

  2188.                     asm volatile(

  2189.                         "pld        [%0, #512]          \n"

  2190.                         "vldm       %0, {d24-d31}       \n" // sum0 sum1 sum2 sum3

  2191. 

  2192.                         "pld        [%1, #512]          \n"

  2193.                         "vldm       %1!, {d8-d15}       \n" // r00 r01 r02 r03 r04 r05 r06 r07

  2194. 

  2195.                         "vshll.u16  q0, d8, #16         \n"

  2196.                         "vshll.u16  q1, d9, #16         \n"

  2197.                         "vshll.u16  q2, d10, #16        \n"

  2198.                         "vshll.u16  q3, d11, #16        \n"

  2199. 

  2200.                         "vshll.u16  q4, d12, #16        \n"

  2201.                         "vshll.u16  q5, d13, #16        \n"

  2202.                         "vshll.u16  q6, d14, #16        \n"

  2203.                         "vshll.u16  q7, d15, #16        \n"

  2204. 

  2205.                         "pld        [%4, #256]          \n"

  2206.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2207. 

  2208.                         "vshll.u16  q8, d20, #16        \n"

  2209.                         "vshll.u16  q9, d21, #16        \n"

  2210.                         "vshll.u16  q10, d22, #16       \n"

  2211.                         "vshll.u16  q11, d23, #16       \n"

  2212. 

  2213.                         "vmla.f32   q12, q8, d0[0]      \n"

  2214.                         "vmla.f32   q13, q8, d4[0]      \n"

  2215.                         "vmla.f32   q14, q8, d8[0]      \n"

  2216.                         "vmla.f32   q15, q8, d12[0]     \n"

  2217.                         "vmla.f32   q12, q9, d0[1]      \n"

  2218.                         "vmla.f32   q13, q9, d4[1]      \n"

  2219.                         "vmla.f32   q14, q9, d8[1]      \n"

  2220.                         "vmla.f32   q15, q9, d12[1]     \n"

  2221. 

  2222.                         "vmla.f32   q12, q10, d1[0]     \n"

  2223.                         "vmla.f32   q13, q10, d5[0]     \n"

  2224.                         "vmla.f32   q14, q10, d9[0]     \n"

  2225.                         "vmla.f32   q15, q10, d13[0]    \n"

  2226.                         "vmla.f32   q12, q11, d1[1]     \n"

  2227.                         "vmla.f32   q13, q11, d5[1]     \n"

  2228.                         "vmla.f32   q14, q11, d9[1]     \n"

  2229.                         "vmla.f32   q15, q11, d13[1]    \n"

  2230. 

  2231.                         "pld        [%4, #256]          \n"

  2232.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2233. 

  2234.                         "vshll.u16  q8, d20, #16        \n"

  2235.                         "vshll.u16  q9, d21, #16        \n"

  2236.                         "vshll.u16  q10, d22, #16       \n"

  2237.                         "vshll.u16  q11, d23, #16       \n"

  2238. 

  2239.                         "pld        [%1, #64]           \n"

  2240.                         "vld1.f32   {d1}, [%1 :64]      \n" // r08

  2241. 

  2242.                         "vshll.u16  q0, d1, #16         \n"

  2243. 

  2244.                         "vmla.f32   q12, q8, d2[0]      \n"

  2245.                         "vmla.f32   q13, q8, d6[0]      \n"

  2246.                         "vmla.f32   q14, q8, d10[0]     \n"

  2247.                         "vmla.f32   q15, q8, d14[0]     \n"

  2248.                         "vmla.f32   q12, q9, d2[1]      \n"

  2249.                         "vmla.f32   q13, q9, d6[1]      \n"

  2250.                         "vmla.f32   q14, q9, d10[1]     \n"

  2251.                         "vmla.f32   q15, q9, d14[1]     \n"

  2252. 

  2253.                         "vmla.f32   q12, q10, d3[0]     \n"

  2254.                         "vmla.f32   q13, q10, d7[0]     \n"

  2255.                         "vmla.f32   q14, q10, d11[0]    \n"

  2256.                         "vmla.f32   q15, q10, d15[0]    \n"

  2257.                         "vmla.f32   q12, q11, d3[1]     \n"

  2258.                         "vmla.f32   q13, q11, d7[1]     \n"

  2259.                         "vmla.f32   q14, q11, d11[1]    \n"

  2260.                         "vmla.f32   q15, q11, d15[1]    \n"

  2261. 

  2262.                         "pld        [%4, #256]          \n"

  2263.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2264. 

  2265.                         "vshll.u16  q8, d20, #16        \n"

  2266.                         "vshll.u16  q9, d21, #16        \n"

  2267.                         "vshll.u16  q10, d22, #16       \n"

  2268.                         "vshll.u16  q11, d23, #16       \n"

  2269. 

  2270.                         "vmla.f32   q12, q8, d4[0]      \n"

  2271.                         "vmla.f32   q13, q8, d8[0]      \n"

  2272.                         "vmla.f32   q14, q8, d12[0]     \n"

  2273.                         "vmla.f32   q15, q8, d0[0]      \n"

  2274.                         "vmla.f32   q12, q9, d4[1]      \n"

  2275.                         "vmla.f32   q13, q9, d8[1]      \n"

  2276.                         "vmla.f32   q14, q9, d12[1]     \n"

  2277.                         "vmla.f32   q15, q9, d0[1]      \n"

  2278. 

  2279.                         "vmla.f32   q12, q10, d5[0]     \n"

  2280.                         "vmla.f32   q13, q10, d9[0]     \n"

  2281.                         "vmla.f32   q14, q10, d13[0]    \n"

  2282.                         "vmla.f32   q15, q10, d1[0]     \n"

  2283.                         "vmla.f32   q12, q11, d5[1]     \n"

  2284.                         "vmla.f32   q13, q11, d9[1]     \n"

  2285.                         "vmla.f32   q14, q11, d13[1]    \n"

  2286.                         "vmla.f32   q15, q11, d1[1]     \n"

  2287. 

  2288.                         "pld        [%2, #512]          \n"

  2289.                         "vldm       %2!, {d8-d15}       \n" // r10 r11 r12 r13 r14 r15 r16 r17

  2290. 

  2291.                         "vshll.u16  q0, d8, #16         \n"

  2292.                         "vshll.u16  q1, d9, #16         \n"

  2293.                         "vshll.u16  q2, d10, #16        \n"

  2294.                         "vshll.u16  q3, d11, #16        \n"

  2295. 

  2296.                         "vshll.u16  q4, d12, #16        \n"

  2297.                         "vshll.u16  q5, d13, #16        \n"

  2298.                         "vshll.u16  q6, d14, #16        \n"

  2299.                         "vshll.u16  q7, d15, #16        \n"

  2300. 

  2301.                         "pld        [%4, #256]          \n"

  2302.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2303. 

  2304.                         "vshll.u16  q8, d20, #16        \n"

  2305.                         "vshll.u16  q9, d21, #16        \n"

  2306.                         "vshll.u16  q10, d22, #16       \n"

  2307.                         "vshll.u16  q11, d23, #16       \n"

  2308. 

  2309.                         "vmla.f32   q12, q8, d0[0]      \n"

  2310.                         "vmla.f32   q13, q8, d4[0]      \n"

  2311.                         "vmla.f32   q14, q8, d8[0]      \n"

  2312.                         "vmla.f32   q15, q8, d12[0]     \n"

  2313.                         "vmla.f32   q12, q9, d0[1]      \n"

  2314.                         "vmla.f32   q13, q9, d4[1]      \n"

  2315.                         "vmla.f32   q14, q9, d8[1]      \n"

  2316.                         "vmla.f32   q15, q9, d12[1]     \n"

  2317. 

  2318.                         "vmla.f32   q12, q10, d1[0]     \n"

  2319.                         "vmla.f32   q13, q10, d5[0]     \n"

  2320.                         "vmla.f32   q14, q10, d9[0]     \n"

  2321.                         "vmla.f32   q15, q10, d13[0]    \n"

  2322.                         "vmla.f32   q12, q11, d1[1]     \n"

  2323.                         "vmla.f32   q13, q11, d5[1]     \n"

  2324.                         "vmla.f32   q14, q11, d9[1]     \n"

  2325.                         "vmla.f32   q15, q11, d13[1]    \n"

  2326. 

  2327.                         "pld        [%4, #256]          \n"

  2328.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2329. 

  2330.                         "vshll.u16  q8, d20, #16        \n"

  2331.                         "vshll.u16  q9, d21, #16        \n"

  2332.                         "vshll.u16  q10, d22, #16       \n"

  2333.                         "vshll.u16  q11, d23, #16       \n"

  2334. 

  2335.                         "pld        [%2, #64]           \n"

  2336.                         "vld1.f32   {d1}, [%2 :64]      \n" // r18

  2337. 

  2338.                         "vshll.u16  q0, d1, #16         \n"

  2339. 

  2340.                         "vmla.f32   q12, q8, d2[0]      \n"

  2341.                         "vmla.f32   q13, q8, d6[0]      \n"

  2342.                         "vmla.f32   q14, q8, d10[0]     \n"

  2343.                         "vmla.f32   q15, q8, d14[0]     \n"

  2344.                         "vmla.f32   q12, q9, d2[1]      \n"

  2345.                         "vmla.f32   q13, q9, d6[1]      \n"

  2346.                         "vmla.f32   q14, q9, d10[1]     \n"

  2347.                         "vmla.f32   q15, q9, d14[1]     \n"

  2348. 

  2349.                         "vmla.f32   q12, q10, d3[0]     \n"

  2350.                         "vmla.f32   q13, q10, d7[0]     \n"

  2351.                         "vmla.f32   q14, q10, d11[0]    \n"

  2352.                         "vmla.f32   q15, q10, d15[0]    \n"

  2353.                         "vmla.f32   q12, q11, d3[1]     \n"

  2354.                         "vmla.f32   q13, q11, d7[1]     \n"

  2355.                         "vmla.f32   q14, q11, d11[1]    \n"

  2356.                         "vmla.f32   q15, q11, d15[1]    \n"

  2357. 

  2358.                         "pld        [%4, #256]          \n"

  2359.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2360. 

  2361.                         "vshll.u16  q8, d20, #16        \n"

  2362.                         "vshll.u16  q9, d21, #16        \n"

  2363.                         "vshll.u16  q10, d22, #16       \n"

  2364.                         "vshll.u16  q11, d23, #16       \n"

  2365. 

  2366.                         "vmla.f32   q12, q8, d4[0]      \n"

  2367.                         "vmla.f32   q13, q8, d8[0]      \n"

  2368.                         "vmla.f32   q14, q8, d12[0]     \n"

  2369.                         "vmla.f32   q15, q8, d0[0]      \n"

  2370.                         "vmla.f32   q12, q9, d4[1]      \n"

  2371.                         "vmla.f32   q13, q9, d8[1]      \n"

  2372.                         "vmla.f32   q14, q9, d12[1]     \n"

  2373.                         "vmla.f32   q15, q9, d0[1]      \n"

  2374. 

  2375.                         "vmla.f32   q12, q10, d5[0]     \n"

  2376.                         "vmla.f32   q13, q10, d9[0]     \n"

  2377.                         "vmla.f32   q14, q10, d13[0]    \n"

  2378.                         "vmla.f32   q15, q10, d1[0]     \n"

  2379.                         "vmla.f32   q12, q11, d5[1]     \n"

  2380.                         "vmla.f32   q13, q11, d9[1]     \n"

  2381.                         "vmla.f32   q14, q11, d13[1]    \n"

  2382.                         "vmla.f32   q15, q11, d1[1]     \n"

  2383. 

  2384.                         "pld        [%3, #256]          \n"

  2385.                         "vldm       %3!, {d8-d15}       \n" // r20 r21 r22 r23 r24 r25 r26 r27

  2386. 

  2387.                         "vshll.u16  q0, d8, #16         \n"

  2388.                         "vshll.u16  q1, d9, #16         \n"

  2389.                         "vshll.u16  q2, d10, #16        \n"

  2390.                         "vshll.u16  q3, d11, #16        \n"

  2391. 

  2392.                         "vshll.u16  q4, d12, #16        \n"

  2393.                         "vshll.u16  q5, d13, #16        \n"

  2394.                         "vshll.u16  q6, d14, #16        \n"

  2395.                         "vshll.u16  q7, d15, #16        \n"

  2396. 

  2397.                         "pld        [%4, #256]          \n"

  2398.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2399. 

  2400.                         "vshll.u16  q8, d20, #16        \n"

  2401.                         "vshll.u16  q9, d21, #16        \n"

  2402.                         "vshll.u16  q10, d22, #16       \n"

  2403.                         "vshll.u16  q11, d23, #16       \n"

  2404. 

  2405.                         "vmla.f32   q12, q8, d0[0]      \n"

  2406.                         "vmla.f32   q13, q8, d4[0]      \n"

  2407.                         "vmla.f32   q14, q8, d8[0]      \n"

  2408.                         "vmla.f32   q15, q8, d12[0]     \n"

  2409.                         "vmla.f32   q12, q9, d0[1]      \n"

  2410.                         "vmla.f32   q13, q9, d4[1]      \n"

  2411.                         "vmla.f32   q14, q9, d8[1]      \n"

  2412.                         "vmla.f32   q15, q9, d12[1]     \n"

  2413. 

  2414.                         "vmla.f32   q12, q10, d1[0]     \n"

  2415.                         "vmla.f32   q13, q10, d5[0]     \n"

  2416.                         "vmla.f32   q14, q10, d9[0]     \n"

  2417.                         "vmla.f32   q15, q10, d13[0]    \n"

  2418.                         "vmla.f32   q12, q11, d1[1]     \n"

  2419.                         "vmla.f32   q13, q11, d5[1]     \n"

  2420.                         "vmla.f32   q14, q11, d9[1]     \n"

  2421.                         "vmla.f32   q15, q11, d13[1]    \n"

  2422. 

  2423.                         "pld        [%4, #256]          \n"

  2424.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2425. 

  2426.                         "vshll.u16  q8, d20, #16        \n"

  2427.                         "vshll.u16  q9, d21, #16        \n"

  2428.                         "vshll.u16  q10, d22, #16       \n"

  2429.                         "vshll.u16  q11, d23, #16       \n"

  2430. 

  2431.                         "pld        [%3, #64]           \n"

  2432.                         "vld1.f32   {d1}, [%3 :64]      \n" // r28

  2433. 

  2434.                         "vshll.u16  q0, d1, #16         \n"

  2435. 

  2436.                         "vmla.f32   q12, q8, d2[0]      \n"

  2437.                         "vmla.f32   q13, q8, d6[0]      \n"

  2438.                         "vmla.f32   q14, q8, d10[0]     \n"

  2439.                         "vmla.f32   q15, q8, d14[0]     \n"

  2440.                         "vmla.f32   q12, q9, d2[1]      \n"

  2441.                         "vmla.f32   q13, q9, d6[1]      \n"

  2442.                         "vmla.f32   q14, q9, d10[1]     \n"

  2443.                         "vmla.f32   q15, q9, d14[1]     \n"

  2444. 

  2445.                         "vmla.f32   q12, q10, d3[0]     \n"

  2446.                         "vmla.f32   q13, q10, d7[0]     \n"

  2447.                         "vmla.f32   q14, q10, d11[0]    \n"

  2448.                         "vmla.f32   q15, q10, d15[0]    \n"

  2449.                         "vmla.f32   q12, q11, d3[1]     \n"

  2450.                         "vmla.f32   q13, q11, d7[1]     \n"

  2451.                         "vmla.f32   q14, q11, d11[1]    \n"

  2452.                         "vmla.f32   q15, q11, d15[1]    \n"

  2453. 

  2454.                         //                         "pld        [%4, #256]          \n"

  2455.                         "vld1.u16   {d20-d23}, [%4 :128] \n"

  2456. 

  2457.                         "vshll.u16  q8, d20, #16        \n"

  2458.                         "vshll.u16  q9, d21, #16        \n"

  2459.                         "vshll.u16  q10, d22, #16       \n"

  2460.                         "vshll.u16  q11, d23, #16       \n"

  2461. 

  2462.                         "vmla.f32   q12, q8, d4[0]      \n"

  2463.                         "vmla.f32   q13, q8, d8[0]      \n"

  2464.                         "vmla.f32   q14, q8, d12[0]     \n"

  2465.                         "vmla.f32   q15, q8, d0[0]      \n"

  2466.                         "vmla.f32   q12, q9, d4[1]      \n"

  2467.                         "vmla.f32   q13, q9, d8[1]      \n"

  2468.                         "vmla.f32   q14, q9, d12[1]     \n"

  2469.                         "vmla.f32   q15, q9, d0[1]      \n"

  2470. 

  2471.                         "vmla.f32   q12, q10, d5[0]     \n"

  2472.                         "vmla.f32   q13, q10, d9[0]     \n"

  2473.                         "vmla.f32   q14, q10, d13[0]    \n"

  2474.                         "vmla.f32   q15, q10, d1[0]     \n"

  2475.                         "vmla.f32   q12, q11, d5[1]     \n"

  2476.                         "vmla.f32   q13, q11, d9[1]     \n"

  2477.                         "vmla.f32   q14, q11, d13[1]    \n"

  2478.                         "vmla.f32   q15, q11, d1[1]     \n"

  2479. 

  2480.                         "sub        %4, %4, #256        \n" // kptr -= 8 * 16;

  2481. 

  2482.                         "vstm       %0!, {d24-d31}      \n"

  2483. 

  2484.                         : "=r"(outptr0), // %0

  2485.                         "=r"(r0),      // %1

  2486.                         "=r"(r1),      // %2

  2487.                         "=r"(r2),      // %3

  2488.                         "=r"(kptr)     // %4

  2489.                         : "0"(outptr0),

  2490.                         "1"(r0),

  2491.                         "2"(r1),

  2492.                         "3"(r2),

  2493.                         "4"(kptr)

  2494.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  2495. #endif // __aarch64__

  2496.                 }

  2497.                 for (; j + 1 < outw; j += 2)

  2498.                 {

  2499. #if __aarch64__

  2500.                     asm volatile(

  2501.                         "prfm   pldl1keep, [%0, #256]       \n"

  2502.                         "ld1    {v12.4s, v13.4s}, [%0]      \n" // sum0 sum1

  2503. 

  2504.                         "prfm   pldl1keep, [%1, #256]       \n"

  2505.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%1], #32 \n" // r00 r01 r02 r03

  2506. 

  2507.                         "shll   v0.4s, v0.4h, #16           \n"

  2508.                         "shll   v1.4s, v1.4h, #16           \n"

  2509.                         "shll   v2.4s, v2.4h, #16           \n"

  2510.                         "shll   v3.4s, v3.4h, #16           \n"

  2511. 

  2512.                         "shll   v6.4s, %8.4h, #16           \n"

  2513.                         "shll2  v7.4s, %8.8h, #16           \n"

  2514.                         "shll   v8.4s, %9.4h, #16           \n"

  2515.                         "shll2  v9.4s, %9.8h, #16           \n"

  2516. 

  2517.                         "fmul   v10.4s, v6.4s, v0.s[0]      \n"

  2518.                         "fmul   v11.4s, v6.4s, v2.s[0]      \n"

  2519.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  2520.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  2521. 

  2522.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  2523.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  2524.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  2525.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  2526. 

  2527.                         "prfm   pldl1keep, [%1, #64]        \n"

  2528.                         "ld1    {v4.4h}, [%1]               \n" // r04

  2529. 

  2530.                         "shll   v4.4s, v4.4h, #16           \n"

  2531. 

  2532.                         "shll   v6.4s, %10.4h, #16          \n"

  2533.                         "shll2  v7.4s, %10.8h, #16          \n"

  2534.                         "shll   v8.4s, %11.4h, #16          \n"

  2535.                         "shll2  v9.4s, %11.8h, #16          \n"

  2536. 

  2537.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  2538.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  2539.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  2540.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  2541. 

  2542.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  2543.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  2544.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  2545.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  2546. 

  2547.                         "shll   v6.4s, %12.4h, #16          \n"

  2548.                         "shll2  v7.4s, %12.8h, #16          \n"

  2549.                         "shll   v8.4s, %13.4h, #16          \n"

  2550.                         "shll2  v9.4s, %13.8h, #16          \n"

  2551. 

  2552.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  2553.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  2554.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  2555.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  2556. 

  2557.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  2558.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  2559.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  2560.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  2561. 

  2562.                         "prfm   pldl1keep, [%2, #256]       \n"

  2563.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r10 r11 r12 r13

  2564. 

  2565.                         "shll   v0.4s, v0.4h, #16           \n"

  2566.                         "shll   v1.4s, v1.4h, #16           \n"

  2567.                         "shll   v2.4s, v2.4h, #16           \n"

  2568.                         "shll   v3.4s, v3.4h, #16           \n"

  2569. 

  2570.                         "shll   v6.4s, %14.4h, #16          \n"

  2571.                         "shll2  v7.4s, %14.8h, #16          \n"

  2572.                         "shll   v8.4s, %15.4h, #16          \n"

  2573.                         "shll2  v9.4s, %15.8h, #16          \n"

  2574. 

  2575.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  2576.                         "fmla   v11.4s, v6.4s, v2.s[0]      \n"

  2577.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  2578.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  2579. 

  2580.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  2581.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  2582.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  2583.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  2584. 

  2585.                         "prfm   pldl1keep, [%2, #64]        \n"

  2586.                         "ld1    {v4.4h}, [%2]               \n" // r14

  2587. 

  2588.                         "shll   v4.4s, v4.4h, #16           \n"

  2589. 

  2590.                         "shll   v6.4s, %16.4h, #16          \n"

  2591.                         "shll2  v7.4s, %16.8h, #16          \n"

  2592.                         "shll   v8.4s, %17.4h, #16          \n"

  2593.                         "shll2  v9.4s, %17.8h, #16          \n"

  2594. 

  2595.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  2596.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  2597.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  2598.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  2599. 

  2600.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  2601.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  2602.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  2603.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  2604. 

  2605.                         "shll   v6.4s, %18.4h, #16          \n"

  2606.                         "shll2  v7.4s, %18.8h, #16          \n"

  2607.                         "shll   v8.4s, %19.4h, #16          \n"

  2608.                         "shll2  v9.4s, %19.8h, #16          \n"

  2609. 

  2610.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  2611.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  2612.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  2613.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  2614. 

  2615.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  2616.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  2617.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  2618.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  2619. 

  2620.                         "prfm   pldl1keep, [%3, #256]       \n"

  2621.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r20 r21 r22 r23

  2622. 

  2623.                         "shll   v0.4s, v0.4h, #16           \n"

  2624.                         "shll   v1.4s, v1.4h, #16           \n"

  2625.                         "shll   v2.4s, v2.4h, #16           \n"

  2626.                         "shll   v3.4s, v3.4h, #16           \n"

  2627. 

  2628.                         "shll   v6.4s, %20.4h, #16          \n"

  2629.                         "shll2  v7.4s, %20.8h, #16          \n"

  2630.                         "shll   v8.4s, %21.4h, #16          \n"

  2631.                         "shll2  v9.4s, %21.8h, #16          \n"

  2632. 

  2633.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  2634.                         "fmla   v11.4s, v6.4s, v2.s[0]      \n"

  2635.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  2636.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  2637. 

  2638.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  2639.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  2640.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  2641.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  2642. 

  2643.                         "prfm   pldl1keep, [%3, #64]        \n"

  2644.                         "ld1    {v4.4h}, [%3]               \n" // r24

  2645. 

  2646.                         "shll   v4.4s, v4.4h, #16           \n"

  2647. 

  2648.                         "shll   v6.4s, %22.4h, #16          \n"

  2649.                         "shll2  v7.4s, %22.8h, #16          \n"

  2650.                         "shll   v8.4s, %23.4h, #16          \n"

  2651.                         "shll2  v9.4s, %23.8h, #16          \n"

  2652. 

  2653.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  2654.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  2655.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  2656.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  2657. 

  2658.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  2659.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  2660.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  2661.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  2662. 

  2663.                         "shll   v6.4s, %24.4h, #16          \n"

  2664.                         "shll2  v7.4s, %24.8h, #16          \n"

  2665.                         "shll   v8.4s, %25.4h, #16          \n"

  2666.                         "shll2  v9.4s, %25.8h, #16          \n"

  2667. 

  2668.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  2669.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  2670.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  2671.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  2672. 

  2673.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  2674.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  2675.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  2676.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  2677. 

  2678.                         "fadd   v12.4s, v10.4s, v12.4s      \n"

  2679.                         "fadd   v13.4s, v11.4s, v13.4s      \n"

  2680. 

  2681.                         "st1    {v12.4s, v13.4s}, [%0], #32 \n"

  2682. 

  2683.                         : "=r"(outptr0), // %0

  2684.                         "=r"(r0),      // %1

  2685.                         "=r"(r1),      // %2

  2686.                         "=r"(r2)       // %3

  2687.                         : "0"(outptr0),

  2688.                         "1"(r0),

  2689.                         "2"(r1),

  2690.                         "3"(r2),

  2691.                         "w"(_k00_01), // %8

  2692.                         "w"(_k00_23), // %9

  2693.                         "w"(_k01_01), // %10

  2694.                         "w"(_k01_23), // %11

  2695.                         "w"(_k02_01), // %12

  2696.                         "w"(_k02_23), // %13

  2697.                         "w"(_k10_01), // %14

  2698.                         "w"(_k10_23), // %15

  2699.                         "w"(_k11_01), // %16

  2700.                         "w"(_k11_23), // %17

  2701.                         "w"(_k12_01), // %18

  2702.                         "w"(_k12_23), // %19

  2703.                         "w"(_k20_01), // %20

  2704.                         "w"(_k20_23), // %21

  2705.                         "w"(_k21_01), // %22

  2706.                         "w"(_k21_23), // %23

  2707.                         "w"(_k22_01), // %24

  2708.                         "w"(_k22_23)  // %25

  2709.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  2710. #else  // __aarch64__

  2711.                     asm volatile(

  2712.                         "pld        [%0, #256]          \n"

  2713.                         "vld1.f32   {d28-d31}, [%0 :128] \n" // sum0 sum1

  2714. 

  2715.                         "pld        [%1, #256]          \n"

  2716.                         "vld1.u16   {d4-d7}, [%1 :64]!  \n" // r00 r01 r02 r03

  2717. 

  2718.                         "vshll.u16  q0, d4, #16         \n"

  2719.                         "vshll.u16  q1, d5, #16         \n"

  2720.                         "vshll.u16  q2, d6, #16         \n"

  2721.                         "vshll.u16  q3, d7, #16         \n"

  2722. 

  2723.                         "pld        [%4, #256]          \n"

  2724.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2725. 

  2726.                         "vshll.u16  q8, d20, #16        \n"

  2727.                         "vshll.u16  q9, d21, #16        \n"

  2728.                         "vshll.u16  q10, d22, #16       \n"

  2729.                         "vshll.u16  q11, d23, #16       \n"

  2730. 

  2731.                         "vmul.f32   q12, q8, d0[0]      \n"

  2732.                         "vmul.f32   q13, q8, d4[0]      \n"

  2733.                         "vmla.f32   q14, q9, d0[1]      \n"

  2734.                         "vmla.f32   q15, q9, d4[1]      \n"

  2735. 

  2736.                         "vmla.f32   q12, q10, d1[0]     \n"

  2737.                         "vmla.f32   q13, q10, d5[0]     \n"

  2738.                         "vmla.f32   q14, q11, d1[1]     \n"

  2739.                         "vmla.f32   q15, q11, d5[1]     \n"

  2740. 

  2741.                         "pld        [%4, #256]          \n"

  2742.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2743. 

  2744.                         "vshll.u16  q8, d20, #16        \n"

  2745.                         "vshll.u16  q9, d21, #16        \n"

  2746.                         "vshll.u16  q10, d22, #16       \n"

  2747.                         "vshll.u16  q11, d23, #16       \n"

  2748. 

  2749.                         "pld        [%1, #64]           \n"

  2750.                         "vld1.f32   {d9}, [%1 :64]      \n" // r04

  2751. 

  2752.                         "vshll.u16  q4, d9, #16         \n"

  2753. 

  2754.                         "vmla.f32   q12, q8, d2[0]      \n"

  2755.                         "vmla.f32   q13, q8, d6[0]      \n"

  2756.                         "vmla.f32   q14, q9, d2[1]      \n"

  2757.                         "vmla.f32   q15, q9, d6[1]      \n"

  2758. 

  2759.                         "vmla.f32   q12, q10, d3[0]     \n"

  2760.                         "vmla.f32   q13, q10, d7[0]     \n"

  2761.                         "vmla.f32   q14, q11, d3[1]     \n"

  2762.                         "vmla.f32   q15, q11, d7[1]     \n"

  2763. 

  2764.                         "pld        [%4, #256]          \n"

  2765.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2766. 

  2767.                         "vshll.u16  q8, d20, #16        \n"

  2768.                         "vshll.u16  q9, d21, #16        \n"

  2769.                         "vshll.u16  q10, d22, #16       \n"

  2770.                         "vshll.u16  q11, d23, #16       \n"

  2771. 

  2772.                         "vmla.f32   q12, q8, d4[0]      \n"

  2773.                         "vmla.f32   q13, q8, d8[0]      \n"

  2774.                         "vmla.f32   q14, q9, d4[1]      \n"

  2775.                         "vmla.f32   q15, q9, d8[1]      \n"

  2776. 

  2777.                         "vmla.f32   q12, q10, d5[0]     \n"

  2778.                         "vmla.f32   q13, q10, d9[0]     \n"

  2779.                         "vmla.f32   q14, q11, d5[1]     \n"

  2780.                         "vmla.f32   q15, q11, d9[1]     \n"

  2781. 

  2782.                         "pld        [%2, #256]          \n"

  2783.                         "vld1.u16   {d4-d7}, [%2 :64]!  \n" // r10 r11 r12 r13

  2784. 

  2785.                         "vshll.u16  q0, d4, #16         \n"

  2786.                         "vshll.u16  q1, d5, #16         \n"

  2787.                         "vshll.u16  q2, d6, #16         \n"

  2788.                         "vshll.u16  q3, d7, #16         \n"

  2789. 

  2790.                         "pld        [%4, #256]          \n"

  2791.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2792. 

  2793.                         "vshll.u16  q8, d20, #16        \n"

  2794.                         "vshll.u16  q9, d21, #16        \n"

  2795.                         "vshll.u16  q10, d22, #16       \n"

  2796.                         "vshll.u16  q11, d23, #16       \n"

  2797. 

  2798.                         "vmla.f32   q12, q8, d0[0]      \n"

  2799.                         "vmla.f32   q13, q8, d4[0]      \n"

  2800.                         "vmla.f32   q14, q9, d0[1]      \n"

  2801.                         "vmla.f32   q15, q9, d4[1]      \n"

  2802. 

  2803.                         "vmla.f32   q12, q10, d1[0]     \n"

  2804.                         "vmla.f32   q13, q10, d5[0]     \n"

  2805.                         "vmla.f32   q14, q11, d1[1]     \n"

  2806.                         "vmla.f32   q15, q11, d5[1]     \n"

  2807. 

  2808.                         "pld        [%4, #256]          \n"

  2809.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2810. 

  2811.                         "vshll.u16  q8, d20, #16        \n"

  2812.                         "vshll.u16  q9, d21, #16        \n"

  2813.                         "vshll.u16  q10, d22, #16       \n"

  2814.                         "vshll.u16  q11, d23, #16       \n"

  2815. 

  2816.                         "pld        [%2, #64]           \n"

  2817.                         "vld1.f32   {d9}, [%2 :64]      \n" // r14

  2818. 

  2819.                         "vshll.u16  q4, d9, #16         \n"

  2820. 

  2821.                         "vmla.f32   q12, q8, d2[0]      \n"

  2822.                         "vmla.f32   q13, q8, d6[0]      \n"

  2823.                         "vmla.f32   q14, q9, d2[1]      \n"

  2824.                         "vmla.f32   q15, q9, d6[1]      \n"

  2825. 

  2826.                         "vmla.f32   q12, q10, d3[0]     \n"

  2827.                         "vmla.f32   q13, q10, d7[0]     \n"

  2828.                         "vmla.f32   q14, q11, d3[1]     \n"

  2829.                         "vmla.f32   q15, q11, d7[1]     \n"

  2830. 

  2831.                         "pld        [%4, #256]          \n"

  2832.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2833. 

  2834.                         "vshll.u16  q8, d20, #16        \n"

  2835.                         "vshll.u16  q9, d21, #16        \n"

  2836.                         "vshll.u16  q10, d22, #16       \n"

  2837.                         "vshll.u16  q11, d23, #16       \n"

  2838. 

  2839.                         "vmla.f32   q12, q8, d4[0]      \n"

  2840.                         "vmla.f32   q13, q8, d8[0]      \n"

  2841.                         "vmla.f32   q14, q9, d4[1]      \n"

  2842.                         "vmla.f32   q15, q9, d8[1]      \n"

  2843. 

  2844.                         "vmla.f32   q12, q10, d5[0]     \n"

  2845.                         "vmla.f32   q13, q10, d9[0]     \n"

  2846.                         "vmla.f32   q14, q11, d5[1]     \n"

  2847.                         "vmla.f32   q15, q11, d9[1]     \n"

  2848. 

  2849.                         "pld        [%3, #256]          \n"

  2850.                         "vld1.u16   {d4-d7}, [%3 :64]!  \n" // r20 r21 r22 r23

  2851. 

  2852.                         "vshll.u16  q0, d4, #16         \n"

  2853.                         "vshll.u16  q1, d5, #16         \n"

  2854.                         "vshll.u16  q2, d6, #16         \n"

  2855.                         "vshll.u16  q3, d7, #16         \n"

  2856. 

  2857.                         "pld        [%4, #256]          \n"

  2858.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2859. 

  2860.                         "vshll.u16  q8, d20, #16        \n"

  2861.                         "vshll.u16  q9, d21, #16        \n"

  2862.                         "vshll.u16  q10, d22, #16       \n"

  2863.                         "vshll.u16  q11, d23, #16       \n"

  2864. 

  2865.                         "vmla.f32   q12, q8, d0[0]      \n"

  2866.                         "vmla.f32   q13, q8, d4[0]      \n"

  2867.                         "vmla.f32   q14, q9, d0[1]      \n"

  2868.                         "vmla.f32   q15, q9, d4[1]      \n"

  2869. 

  2870.                         "vmla.f32   q12, q10, d1[0]     \n"

  2871.                         "vmla.f32   q13, q10, d5[0]     \n"

  2872.                         "vmla.f32   q14, q11, d1[1]     \n"

  2873.                         "vmla.f32   q15, q11, d5[1]     \n"

  2874. 

  2875.                         "pld        [%4, #256]          \n"

  2876.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  2877. 

  2878.                         "vshll.u16  q8, d20, #16        \n"

  2879.                         "vshll.u16  q9, d21, #16        \n"

  2880.                         "vshll.u16  q10, d22, #16       \n"

  2881.                         "vshll.u16  q11, d23, #16       \n"

  2882. 

  2883.                         "pld        [%3, #64]           \n"

  2884.                         "vld1.f32   {d9}, [%3 :64]      \n" // r24

  2885. 

  2886.                         "vshll.u16  q4, d9, #16         \n"

  2887. 

  2888.                         "vmla.f32   q12, q8, d2[0]      \n"

  2889.                         "vmla.f32   q13, q8, d6[0]      \n"

  2890.                         "vmla.f32   q14, q9, d2[1]      \n"

  2891.                         "vmla.f32   q15, q9, d6[1]      \n"

  2892. 

  2893.                         "vmla.f32   q12, q10, d3[0]     \n"

  2894.                         "vmla.f32   q13, q10, d7[0]     \n"

  2895.                         "vmla.f32   q14, q11, d3[1]     \n"

  2896.                         "vmla.f32   q15, q11, d7[1]     \n"

  2897. 

  2898.                         //                         "pld        [%4, #256]          \n"

  2899.                         "vld1.u16   {d20-d23}, [%4 :128] \n"

  2900. 

  2901.                         "vshll.u16  q8, d20, #16        \n"

  2902.                         "vshll.u16  q9, d21, #16        \n"

  2903.                         "vshll.u16  q10, d22, #16       \n"

  2904.                         "vshll.u16  q11, d23, #16       \n"

  2905. 

  2906.                         "vmla.f32   q12, q8, d4[0]      \n"

  2907.                         "vmla.f32   q13, q8, d8[0]      \n"

  2908.                         "vmla.f32   q14, q9, d4[1]      \n"

  2909.                         "vmla.f32   q15, q9, d8[1]      \n"

  2910. 

  2911.                         "vmla.f32   q12, q10, d5[0]     \n"

  2912.                         "vmla.f32   q13, q10, d9[0]     \n"

  2913.                         "vmla.f32   q14, q11, d5[1]     \n"

  2914.                         "vmla.f32   q15, q11, d9[1]     \n"

  2915. 

  2916.                         "vadd.f32   q14, q12, q14       \n"

  2917.                         "vadd.f32   q15, q13, q15       \n"

  2918. 

  2919.                         "sub        %4, %4, #256        \n" // kptr -= 8 * 16;

  2920. 

  2921.                         "vst1.f32   {d28-d31}, [%0 :128]! \n"

  2922. 

  2923.                         : "=r"(outptr0), // %0

  2924.                         "=r"(r0),      // %1

  2925.                         "=r"(r1),      // %2

  2926.                         "=r"(r2),      // %3

  2927.                         "=r"(kptr)     // %4

  2928.                         : "0"(outptr0),

  2929.                         "1"(r0),

  2930.                         "2"(r1),

  2931.                         "3"(r2),

  2932.                         "4"(kptr)

  2933.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  2934. #endif // __aarch64__

  2935.                 }

  2936.                 for (; j < outw; j++)

  2937.                 {

  2938. #if __aarch64__

  2939.                     asm volatile(

  2940.                         "prfm   pldl1keep, [%0, #128]       \n"

  2941.                         "ld1    {v13.4s}, [%0]              \n" // sum0

  2942. 

  2943.                         "prfm   pldl1keep, [%1, #192]       \n"

  2944.                         "ld1    {v0.4h, v1.4h, v2.4h}, [%1] \n" // r00 r01 r02

  2945. 

  2946.                         "shll   v0.4s, v0.4h, #16           \n"

  2947.                         "shll   v1.4s, v1.4h, #16           \n"

  2948.                         "shll   v2.4s, v2.4h, #16           \n"

  2949. 

  2950.                         "shll   v6.4s, %8.4h, #16           \n"

  2951.                         "shll2  v7.4s, %8.8h, #16           \n"

  2952. 

  2953.                         "fmul   v10.4s, v6.4s, v0.s[0]      \n"

  2954.                         "fmul   v11.4s, v7.4s, v0.s[1]      \n"

  2955. 

  2956.                         "shll   v8.4s, %9.4h, #16           \n"

  2957.                         "shll2  v9.4s, %9.8h, #16           \n"

  2958. 

  2959.                         "fmul   v12.4s, v8.4s, v0.s[2]      \n"

  2960.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  2961. 

  2962.                         "shll   v6.4s, %10.4h, #16          \n"

  2963.                         "shll2  v7.4s, %10.8h, #16          \n"

  2964. 

  2965.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  2966.                         "fmla   v11.4s, v7.4s, v1.s[1]      \n"

  2967. 

  2968.                         "shll   v8.4s, %11.4h, #16          \n"

  2969.                         "shll2  v9.4s, %11.8h, #16          \n"

  2970. 

  2971.                         "fmla   v12.4s, v8.4s, v1.s[2]      \n"

  2972.                         "fmla   v13.4s, v9.4s, v1.s[3]      \n"

  2973. 

  2974.                         "shll   v6.4s, %12.4h, #16          \n"

  2975.                         "shll2  v7.4s, %12.8h, #16          \n"

  2976. 

  2977.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  2978.                         "fmla   v11.4s, v7.4s, v2.s[1]      \n"

  2979. 

  2980.                         "shll   v8.4s, %13.4h, #16          \n"

  2981.                         "shll2  v9.4s, %13.8h, #16          \n"

  2982. 

  2983.                         "fmla   v12.4s, v8.4s, v2.s[2]      \n"

  2984.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  2985. 

  2986.                         "prfm   pldl1keep, [%2, #192]       \n"

  2987.                         "ld1    {v3.4h, v4.4h, v5.4h}, [%2] \n" // r10 r11 r12

  2988. 

  2989.                         "shll   v3.4s, v3.4h, #16           \n"

  2990.                         "shll   v4.4s, v4.4h, #16           \n"

  2991.                         "shll   v5.4s, v5.4h, #16           \n"

  2992. 

  2993.                         "shll   v6.4s, %14.4h, #16          \n"

  2994.                         "shll2  v7.4s, %14.8h, #16          \n"

  2995. 

  2996.                         "fmla   v10.4s, v6.4s, v3.s[0]      \n"

  2997.                         "fmla   v11.4s, v7.4s, v3.s[1]      \n"

  2998. 

  2999.                         "shll   v8.4s, %15.4h, #16          \n"

  3000.                         "shll2  v9.4s, %15.8h, #16          \n"

  3001. 

  3002.                         "fmla   v12.4s, v8.4s, v3.s[2]      \n"

  3003.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  3004. 

  3005.                         "shll   v6.4s, %16.4h, #16          \n"

  3006.                         "shll2  v7.4s, %16.8h, #16          \n"

  3007. 

  3008.                         "fmla   v10.4s, v6.4s, v4.s[0]      \n"

  3009.                         "fmla   v11.4s, v7.4s, v4.s[1]      \n"

  3010. 

  3011.                         "shll   v8.4s, %17.4h, #16          \n"

  3012.                         "shll2  v9.4s, %17.8h, #16          \n"

  3013. 

  3014.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  3015.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  3016. 

  3017.                         "shll   v6.4s, %18.4h, #16          \n"

  3018.                         "shll2  v7.4s, %18.8h, #16          \n"

  3019. 

  3020.                         "fmla   v10.4s, v6.4s, v5.s[0]      \n"

  3021.                         "fmla   v11.4s, v7.4s, v5.s[1]      \n"

  3022. 

  3023.                         "shll   v8.4s, %19.4h, #16          \n"

  3024.                         "shll2  v9.4s, %19.8h, #16          \n"

  3025. 

  3026.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  3027.                         "fmla   v13.4s, v9.4s, v5.s[3]      \n"

  3028. 

  3029.                         "prfm   pldl1keep, [%3, #192]       \n"

  3030.                         "ld1    {v0.4h, v1.4h, v2.4h}, [%3] \n" // r20 r21 r22

  3031. 

  3032.                         "shll   v0.4s, v0.4h, #16           \n"

  3033.                         "shll   v1.4s, v1.4h, #16           \n"

  3034.                         "shll   v2.4s, v2.4h, #16           \n"

  3035. 

  3036.                         "shll   v6.4s, %20.4h, #16          \n"

  3037.                         "shll2  v7.4s, %20.8h, #16          \n"

  3038. 

  3039.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  3040.                         "fmla   v11.4s, v7.4s, v0.s[1]      \n"

  3041. 

  3042.                         "shll   v8.4s, %21.4h, #16          \n"

  3043.                         "shll2  v9.4s, %21.8h, #16          \n"

  3044. 

  3045.                         "fmla   v12.4s, v8.4s, v0.s[2]      \n"

  3046.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  3047. 

  3048.                         "shll   v6.4s, %22.4h, #16          \n"

  3049.                         "shll2  v7.4s, %22.8h, #16          \n"

  3050. 

  3051.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  3052.                         "fmla   v11.4s, v7.4s, v1.s[1]      \n"

  3053. 

  3054.                         "shll   v8.4s, %23.4h, #16          \n"

  3055.                         "shll2  v9.4s, %23.8h, #16          \n"

  3056. 

  3057.                         "fmla   v12.4s, v8.4s, v1.s[2]      \n"

  3058.                         "fmla   v13.4s, v9.4s, v1.s[3]      \n"

  3059. 

  3060.                         "shll   v6.4s, %24.4h, #16          \n"

  3061.                         "shll2  v7.4s, %24.8h, #16          \n"

  3062. 

  3063.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  3064.                         "fmla   v11.4s, v7.4s, v2.s[1]      \n"

  3065. 

  3066.                         "shll   v8.4s, %25.4h, #16          \n"

  3067.                         "shll2  v9.4s, %25.8h, #16          \n"

  3068. 

  3069.                         "fmla   v12.4s, v8.4s, v2.s[2]      \n"

  3070.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  3071. 

  3072.                         "fadd   v11.4s, v10.4s, v11.4s      \n"

  3073. 

  3074.                         "add    %1, %1, #16                 \n"

  3075.                         "fadd   v13.4s, v12.4s, v13.4s      \n"

  3076. 

  3077.                         "add    %2, %2, #16                 \n"

  3078.                         "fadd   v13.4s, v11.4s, v13.4s      \n"

  3079. 

  3080.                         "add    %3, %3, #16                 \n"

  3081. 

  3082.                         "st1    {v13.4s}, [%0], #16         \n"

  3083. 

  3084.                         : "=r"(outptr0), // %0

  3085.                         "=r"(r0),      // %1

  3086.                         "=r"(r1),      // %2

  3087.                         "=r"(r2)       // %3

  3088.                         : "0"(outptr0),

  3089.                         "1"(r0),

  3090.                         "2"(r1),

  3091.                         "3"(r2),

  3092.                         "w"(_k00_01), // %8

  3093.                         "w"(_k00_23), // %9

  3094.                         "w"(_k01_01), // %10

  3095.                         "w"(_k01_23), // %11

  3096.                         "w"(_k02_01), // %12

  3097.                         "w"(_k02_23), // %13

  3098.                         "w"(_k10_01), // %14

  3099.                         "w"(_k10_23), // %15

  3100.                         "w"(_k11_01), // %16

  3101.                         "w"(_k11_23), // %17

  3102.                         "w"(_k12_01), // %18

  3103.                         "w"(_k12_23), // %19

  3104.                         "w"(_k20_01), // %20

  3105.                         "w"(_k20_23), // %21

  3106.                         "w"(_k21_01), // %22

  3107.                         "w"(_k21_23), // %23

  3108.                         "w"(_k22_01), // %24

  3109.                         "w"(_k22_23)  // %25

  3110.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  3111. #else  // __aarch64__

  3112.                     asm volatile(

  3113.                         "pld        [%0, #128]          \n"

  3114.                         "vld1.f32   {d30-d31}, [%0 :128] \n" // sum0

  3115. 

  3116.                         "pld        [%1, #192]          \n"

  3117.                         "vld1.u16   {d2-d4}, [%1 :64]   \n" // r00 r01 r02

  3118. 

  3119.                         "vshll.u16  q0, d2, #16         \n"

  3120.                         "vshll.u16  q1, d3, #16         \n"

  3121.                         "vshll.u16  q2, d4, #16         \n"

  3122. 

  3123.                         "pld        [%4, #256]          \n"

  3124.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3125. 

  3126.                         "vshll.u16  q8, d20, #16        \n"

  3127.                         "vshll.u16  q9, d21, #16        \n"

  3128. 

  3129.                         "vmul.f32   q12, q8, d0[0]      \n"

  3130.                         "vmul.f32   q13, q9, d0[1]      \n"

  3131. 

  3132.                         "vshll.u16  q10, d22, #16       \n"

  3133.                         "vshll.u16  q11, d23, #16       \n"

  3134. 

  3135.                         "vmul.f32   q14, q10, d1[0]     \n"

  3136.                         "vmla.f32   q15, q11, d1[1]     \n"

  3137. 

  3138.                         "pld        [%4, #256]          \n"

  3139.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3140. 

  3141.                         "vshll.u16  q8, d20, #16        \n"

  3142.                         "vshll.u16  q9, d21, #16        \n"

  3143. 

  3144.                         "vmla.f32   q12, q8, d2[0]      \n"

  3145.                         "vmla.f32   q13, q9, d2[1]      \n"

  3146. 

  3147.                         "vshll.u16  q10, d22, #16       \n"

  3148.                         "vshll.u16  q11, d23, #16       \n"

  3149. 

  3150.                         "vmla.f32   q14, q10, d3[0]     \n"

  3151.                         "vmla.f32   q15, q11, d3[1]     \n"

  3152. 

  3153.                         "pld        [%4, #256]          \n"

  3154.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3155. 

  3156.                         "vshll.u16  q8, d20, #16        \n"

  3157.                         "vshll.u16  q9, d21, #16        \n"

  3158. 

  3159.                         "vmla.f32   q12, q8, d4[0]      \n"

  3160.                         "vmla.f32   q13, q9, d4[1]      \n"

  3161. 

  3162.                         "vshll.u16  q10, d22, #16       \n"

  3163.                         "vshll.u16  q11, d23, #16       \n"

  3164. 

  3165.                         "vmla.f32   q14, q10, d5[0]     \n"

  3166.                         "vmla.f32   q15, q11, d5[1]     \n"

  3167. 

  3168.                         "pld        [%2, #192]          \n"

  3169.                         "vld1.u16   {d2-d4}, [%2 :64]   \n" // r10 r11 r12

  3170. 

  3171.                         "vshll.u16  q0, d2, #16         \n"

  3172.                         "vshll.u16  q1, d3, #16         \n"

  3173.                         "vshll.u16  q2, d4, #16         \n"

  3174. 

  3175.                         "pld        [%4, #256]          \n"

  3176.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3177. 

  3178.                         "vshll.u16  q8, d20, #16        \n"

  3179.                         "vshll.u16  q9, d21, #16        \n"

  3180. 

  3181.                         "vmla.f32   q12, q8, d0[0]      \n"

  3182.                         "vmla.f32   q13, q9, d0[1]      \n"

  3183. 

  3184.                         "vshll.u16  q10, d22, #16       \n"

  3185.                         "vshll.u16  q11, d23, #16       \n"

  3186. 

  3187.                         "vmla.f32   q14, q10, d1[0]     \n"

  3188.                         "vmla.f32   q15, q11, d1[1]     \n"

  3189. 

  3190.                         "pld        [%4, #256]          \n"

  3191.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3192. 

  3193.                         "vshll.u16  q8, d20, #16        \n"

  3194.                         "vshll.u16  q9, d21, #16        \n"

  3195. 

  3196.                         "vmla.f32   q12, q8, d2[0]      \n"

  3197.                         "vmla.f32   q13, q9, d2[1]      \n"

  3198. 

  3199.                         "vshll.u16  q10, d22, #16       \n"

  3200.                         "vshll.u16  q11, d23, #16       \n"

  3201. 

  3202.                         "vmla.f32   q14, q10, d3[0]     \n"

  3203.                         "vmla.f32   q15, q11, d3[1]     \n"

  3204. 

  3205.                         "pld        [%4, #256]          \n"

  3206.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3207. 

  3208.                         "vshll.u16  q8, d20, #16        \n"

  3209.                         "vshll.u16  q9, d21, #16        \n"

  3210. 

  3211.                         "vmla.f32   q12, q8, d4[0]      \n"

  3212.                         "vmla.f32   q13, q9, d4[1]      \n"

  3213. 

  3214.                         "vshll.u16  q10, d22, #16       \n"

  3215.                         "vshll.u16  q11, d23, #16       \n"

  3216. 

  3217.                         "vmla.f32   q14, q10, d5[0]     \n"

  3218.                         "vmla.f32   q15, q11, d5[1]     \n"

  3219. 

  3220.                         "pld        [%3, #192]          \n"

  3221.                         "vld1.u16   {d2-d4}, [%3 :64]   \n" // r20 r21 r22

  3222. 

  3223.                         "vshll.u16  q0, d2, #16         \n"

  3224.                         "vshll.u16  q1, d3, #16         \n"

  3225.                         "vshll.u16  q2, d4, #16         \n"

  3226. 

  3227.                         "pld        [%4, #256]          \n"

  3228.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3229. 

  3230.                         "vshll.u16  q8, d20, #16        \n"

  3231.                         "vshll.u16  q9, d21, #16        \n"

  3232. 

  3233.                         "vmla.f32   q12, q8, d0[0]      \n"

  3234.                         "vmla.f32   q13, q9, d0[1]      \n"

  3235. 

  3236.                         "vshll.u16  q10, d22, #16       \n"

  3237.                         "vshll.u16  q11, d23, #16       \n"

  3238. 

  3239.                         "vmla.f32   q14, q10, d1[0]     \n"

  3240.                         "vmla.f32   q15, q11, d1[1]     \n"

  3241. 

  3242.                         "pld        [%4, #256]          \n"

  3243.                         "vld1.u16   {d20-d23}, [%4 :128]! \n"

  3244. 

  3245.                         "vshll.u16  q8, d20, #16        \n"

  3246.                         "vshll.u16  q9, d21, #16        \n"

  3247. 

  3248.                         "vmla.f32   q12, q8, d2[0]      \n"

  3249.                         "vmla.f32   q13, q9, d2[1]      \n"

  3250. 

  3251.                         "vshll.u16  q10, d22, #16       \n"

  3252.                         "vshll.u16  q11, d23, #16       \n"

  3253. 

  3254.                         "vmla.f32   q14, q10, d3[0]     \n"

  3255.                         "vmla.f32   q15, q11, d3[1]     \n"

  3256. 

  3257.                         //                         "pld        [%4, #256]          \n"

  3258.                         "vld1.u16   {d20-d23}, [%4 :128] \n"

  3259. 

  3260.                         "vshll.u16  q8, d20, #16        \n"

  3261.                         "vshll.u16  q9, d21, #16        \n"

  3262. 

  3263.                         "vmla.f32   q12, q8, d4[0]      \n"

  3264.                         "vmla.f32   q13, q9, d4[1]      \n"

  3265. 

  3266.                         "vshll.u16  q10, d22, #16       \n"

  3267.                         "vshll.u16  q11, d23, #16       \n"

  3268. 

  3269.                         "vmla.f32   q14, q10, d5[0]     \n"

  3270.                         "vmla.f32   q15, q11, d5[1]     \n"

  3271. 

  3272.                         "add        %1, %1, #16         \n"

  3273.                         "vadd.f32   q13, q12, q13       \n"

  3274. 

  3275.                         "add        %2, %2, #16         \n"

  3276.                         "vadd.f32   q15, q14, q15       \n"

  3277. 

  3278.                         "add        %3, %3, #16         \n"

  3279.                         "vadd.f32   q15, q13, q15       \n"

  3280. 

  3281.                         "sub        %4, %4, #256        \n" // kptr -= 8 * 16 * 2;

  3282. 

  3283.                         "vst1.f32   {d30-d31}, [%0 :128]! \n"

  3284. 

  3285.                         : "=r"(outptr0), // %0

  3286.                         "=r"(r0),      // %1

  3287.                         "=r"(r1),      // %2

  3288.                         "=r"(r2),      // %3

  3289.                         "=r"(kptr)     // %4

  3290.                         : "0"(outptr0),

  3291.                         "1"(r0),

  3292.                         "2"(r1),

  3293.                         "3"(r2),

  3294.                         "4"(kptr)

  3295.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  3296. #endif // __aarch64__

  3297.                 }

  3298. 

  3299.                 r0 += tailstep;

  3300.                 r1 += tailstep;

  3301.                 r2 += tailstep;

  3302.             }

  3303.         }

  3304.         for (; q < inch; q++)

  3305.         {

  3306.             unsigned short* outptr0_bf16 = top_blob.channel(p);

  3307. 

  3308.             const float* outptr0 = out0.row(0);

  3309. 

  3310.             const Mat img0 = bottom_blob.channel(q);

  3311. 

  3312.             const unsigned short* r0 = img0.row<const unsigned short>(0);

  3313.             const unsigned short* r1 = img0.row<const unsigned short>(1);

  3314.             const unsigned short* r2 = img0.row<const unsigned short>(2);

  3315. 

  3316.             const unsigned short* kptr = (const unsigned short*)kernel.channel(p).row<const unsigned short>(q);

  3317. 

  3318. #if __aarch64__

  3319.             // 16 * 9

  3320.             uint16x8_t _k00_01 = vld1q_u16(kptr);

  3321.             uint16x8_t _k00_23 = vld1q_u16(kptr + 8);

  3322.             uint16x8_t _k01_01 = vld1q_u16(kptr + 16);

  3323.             uint16x8_t _k01_23 = vld1q_u16(kptr + 24);

  3324.             uint16x8_t _k02_01 = vld1q_u16(kptr + 32);

  3325.             uint16x8_t _k02_23 = vld1q_u16(kptr + 40);

  3326.             uint16x8_t _k10_01 = vld1q_u16(kptr + 48);

  3327.             uint16x8_t _k10_23 = vld1q_u16(kptr + 56);

  3328.             uint16x8_t _k11_01 = vld1q_u16(kptr + 64);

  3329.             uint16x8_t _k11_23 = vld1q_u16(kptr + 72);

  3330.             uint16x8_t _k12_01 = vld1q_u16(kptr + 80);

  3331.             uint16x8_t _k12_23 = vld1q_u16(kptr + 88);

  3332.             uint16x8_t _k20_01 = vld1q_u16(kptr + 96);

  3333.             uint16x8_t _k20_23 = vld1q_u16(kptr + 104);

  3334.             uint16x8_t _k21_01 = vld1q_u16(kptr + 112);

  3335.             uint16x8_t _k21_23 = vld1q_u16(kptr + 120);

  3336.             uint16x8_t _k22_01 = vld1q_u16(kptr + 128);

  3337.             uint16x8_t _k22_23 = vld1q_u16(kptr + 136);

  3338. #endif // __aarch64__

  3339. 

  3340.             int i = 0;

  3341.             for (; i < outh; i++)

  3342.             {

  3343.                 int j = 0;

  3344.                 for (; j + 3 < outw; j += 4)

  3345.                 {

  3346. #if __aarch64__

  3347.                     asm volatile(

  3348.                         "prfm   pldl1keep, [%1, #512]       \n"

  3349.                         "ld1    {v10.4s, v11.4s, v12.4s, v13.4s}, [%1], #64 \n" // sum0 sum1 sum2 sum3

  3350. 

  3351.                         "prfm   pldl1keep, [%2, #512]       \n"

  3352.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r00 r01 r02 r03

  3353. 

  3354.                         "shll   v0.4s, v4.4h, #16           \n"

  3355.                         "shll2  v1.4s, v4.8h, #16           \n"

  3356.                         "shll   v2.4s, v5.4h, #16           \n"

  3357.                         "shll2  v3.4s, v5.8h, #16           \n"

  3358. 

  3359.                         "shll   v4.4s, v6.4h, #16           \n"

  3360.                         "shll2  v5.4s, v6.8h, #16           \n"

  3361.                         "shll   v6.4s, v7.4h, #16           \n"

  3362.                         "shll2  v7.4s, v7.8h, #16           \n"

  3363. 

  3364.                         "shll   v8.4s, %10.4h, #16          \n"

  3365.                         "shll2  v9.4s, %10.8h, #16          \n"

  3366. 

  3367.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  3368.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  3369.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  3370.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  3371.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  3372.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  3373.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  3374.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  3375. 

  3376.                         "shll   v8.4s, %11.4h, #16          \n"

  3377.                         "shll2  v9.4s, %11.8h, #16          \n"

  3378. 

  3379.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  3380.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  3381.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  3382.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  3383.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  3384.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  3385.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  3386.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  3387. 

  3388.                         "shll   v8.4s, %12.4h, #16          \n"

  3389.                         "shll2  v9.4s, %12.8h, #16          \n"

  3390. 

  3391.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  3392.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  3393.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  3394.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  3395.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  3396.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  3397.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  3398.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  3399. 

  3400.                         "shll   v8.4s, %13.4h, #16          \n"

  3401.                         "shll2  v9.4s, %13.8h, #16          \n"

  3402. 

  3403.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  3404.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  3405.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  3406.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  3407.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  3408.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  3409.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  3410.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  3411. 

  3412.                         "prfm   pldl1keep, [%2, #64]        \n"

  3413.                         "ld1    {v0.4h}, [%2]               \n" // r08

  3414. 

  3415.                         "shll   v0.4s, v0.4h, #16           \n"

  3416. 

  3417.                         "shll   v8.4s, %14.4h, #16          \n"

  3418.                         "shll2  v9.4s, %14.8h, #16          \n"

  3419. 

  3420.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  3421.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  3422.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  3423.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  3424.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  3425.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  3426.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  3427.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  3428. 

  3429.                         "shll   v8.4s, %15.4h, #16          \n"

  3430.                         "shll2  v9.4s, %15.8h, #16          \n"

  3431. 

  3432.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  3433.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  3434.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  3435.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  3436.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  3437.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  3438.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  3439.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  3440. 

  3441.                         "prfm   pldl1keep, [%3, #512]       \n"

  3442.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%3], #64 \n" // r10 r11 r12 r13

  3443. 

  3444.                         "shll   v0.4s, v4.4h, #16           \n"

  3445.                         "shll2  v1.4s, v4.8h, #16           \n"

  3446.                         "shll   v2.4s, v5.4h, #16           \n"

  3447.                         "shll2  v3.4s, v5.8h, #16           \n"

  3448. 

  3449.                         "shll   v4.4s, v6.4h, #16           \n"

  3450.                         "shll2  v5.4s, v6.8h, #16           \n"

  3451.                         "shll   v6.4s, v7.4h, #16           \n"

  3452.                         "shll2  v7.4s, v7.8h, #16           \n"

  3453. 

  3454.                         "shll   v8.4s, %16.4h, #16          \n"

  3455.                         "shll2  v9.4s, %16.8h, #16          \n"

  3456. 

  3457.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  3458.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  3459.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  3460.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  3461.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  3462.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  3463.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  3464.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  3465. 

  3466.                         "shll   v8.4s, %17.4h, #16          \n"

  3467.                         "shll2  v9.4s, %17.8h, #16          \n"

  3468. 

  3469.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  3470.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  3471.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  3472.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  3473.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  3474.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  3475.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  3476.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  3477. 

  3478.                         "shll   v8.4s, %18.4h, #16          \n"

  3479.                         "shll2  v9.4s, %18.8h, #16          \n"

  3480. 

  3481.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  3482.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  3483.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  3484.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  3485.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  3486.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  3487.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  3488.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  3489. 

  3490.                         "shll   v8.4s, %19.4h, #16          \n"

  3491.                         "shll2  v9.4s, %19.8h, #16          \n"

  3492. 

  3493.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  3494.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  3495.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  3496.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  3497.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  3498.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  3499.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  3500.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  3501. 

  3502.                         "prfm   pldl1keep, [%3, #64]        \n"

  3503.                         "ld1    {v0.4h}, [%3]               \n" // r18

  3504. 

  3505.                         "shll   v0.4s, v0.4h, #16           \n"

  3506. 

  3507.                         "shll   v8.4s, %20.4h, #16          \n"

  3508.                         "shll2  v9.4s, %20.8h, #16          \n"

  3509. 

  3510.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  3511.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  3512.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  3513.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  3514.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  3515.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  3516.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  3517.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  3518. 

  3519.                         "shll   v8.4s, %21.4h, #16          \n"

  3520.                         "shll2  v9.4s, %21.8h, #16          \n"

  3521. 

  3522.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  3523.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  3524.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  3525.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  3526.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  3527.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  3528.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  3529.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  3530. 

  3531.                         "prfm   pldl1keep, [%4, #512]       \n"

  3532.                         "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%4], #64 \n" // r20 r21 r22 r23

  3533. 

  3534.                         "shll   v0.4s, v4.4h, #16           \n"

  3535.                         "shll2  v1.4s, v4.8h, #16           \n"

  3536.                         "shll   v2.4s, v5.4h, #16           \n"

  3537.                         "shll2  v3.4s, v5.8h, #16           \n"

  3538. 

  3539.                         "shll   v4.4s, v6.4h, #16           \n"

  3540.                         "shll2  v5.4s, v6.8h, #16           \n"

  3541.                         "shll   v6.4s, v7.4h, #16           \n"

  3542.                         "shll2  v7.4s, v7.8h, #16           \n"

  3543. 

  3544.                         "shll   v8.4s, %22.4h, #16          \n"

  3545.                         "shll2  v9.4s, %22.8h, #16          \n"

  3546. 

  3547.                         "fmla   v10.4s, v8.4s, v0.s[0]      \n"

  3548.                         "fmla   v11.4s, v8.4s, v2.s[0]      \n"

  3549.                         "fmla   v12.4s, v8.4s, v4.s[0]      \n"

  3550.                         "fmla   v13.4s, v8.4s, v6.s[0]      \n"

  3551.                         "fmla   v10.4s, v9.4s, v0.s[1]      \n"

  3552.                         "fmla   v11.4s, v9.4s, v2.s[1]      \n"

  3553.                         "fmla   v12.4s, v9.4s, v4.s[1]      \n"

  3554.                         "fmla   v13.4s, v9.4s, v6.s[1]      \n"

  3555. 

  3556.                         "shll   v8.4s, %23.4h, #16          \n"

  3557.                         "shll2  v9.4s, %23.8h, #16          \n"

  3558. 

  3559.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  3560.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  3561.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  3562.                         "fmla   v13.4s, v8.4s, v6.s[2]      \n"

  3563.                         "fmla   v10.4s, v9.4s, v0.s[3]      \n"

  3564.                         "fmla   v11.4s, v9.4s, v2.s[3]      \n"

  3565.                         "fmla   v12.4s, v9.4s, v4.s[3]      \n"

  3566.                         "fmla   v13.4s, v9.4s, v6.s[3]      \n"

  3567. 

  3568.                         "shll   v8.4s, %24.4h, #16          \n"

  3569.                         "shll2  v9.4s, %24.8h, #16          \n"

  3570. 

  3571.                         "fmla   v10.4s, v8.4s, v1.s[0]      \n"

  3572.                         "fmla   v11.4s, v8.4s, v3.s[0]      \n"

  3573.                         "fmla   v12.4s, v8.4s, v5.s[0]      \n"

  3574.                         "fmla   v13.4s, v8.4s, v7.s[0]      \n"

  3575.                         "fmla   v10.4s, v9.4s, v1.s[1]      \n"

  3576.                         "fmla   v11.4s, v9.4s, v3.s[1]      \n"

  3577.                         "fmla   v12.4s, v9.4s, v5.s[1]      \n"

  3578.                         "fmla   v13.4s, v9.4s, v7.s[1]      \n"

  3579. 

  3580.                         "shll   v8.4s, %25.4h, #16          \n"

  3581.                         "shll2  v9.4s, %25.8h, #16          \n"

  3582. 

  3583.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  3584.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  3585.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  3586.                         "fmla   v13.4s, v8.4s, v7.s[2]      \n"

  3587.                         "fmla   v10.4s, v9.4s, v1.s[3]      \n"

  3588.                         "fmla   v11.4s, v9.4s, v3.s[3]      \n"

  3589.                         "fmla   v12.4s, v9.4s, v5.s[3]      \n"

  3590.                         "fmla   v13.4s, v9.4s, v7.s[3]      \n"

  3591. 

  3592.                         "prfm   pldl1keep, [%4, #64]        \n"

  3593.                         "ld1    {v0.4h}, [%4]               \n" // r28

  3594. 

  3595.                         "shll   v0.4s, v0.4h, #16           \n"

  3596. 

  3597.                         "shll   v8.4s, %26.4h, #16          \n"

  3598.                         "shll2  v9.4s, %26.8h, #16          \n"

  3599. 

  3600.                         "fmla   v10.4s, v8.4s, v2.s[0]      \n"

  3601.                         "fmla   v11.4s, v8.4s, v4.s[0]      \n"

  3602.                         "fmla   v12.4s, v8.4s, v6.s[0]      \n"

  3603.                         "fmla   v13.4s, v8.4s, v0.s[0]      \n"

  3604.                         "fmla   v10.4s, v9.4s, v2.s[1]      \n"

  3605.                         "fmla   v11.4s, v9.4s, v4.s[1]      \n"

  3606.                         "fmla   v12.4s, v9.4s, v6.s[1]      \n"

  3607.                         "fmla   v13.4s, v9.4s, v0.s[1]      \n"

  3608. 

  3609.                         "shll   v8.4s, %27.4h, #16          \n"

  3610.                         "shll2  v9.4s, %27.8h, #16          \n"

  3611. 

  3612.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  3613.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  3614.                         "fmla   v12.4s, v8.4s, v6.s[2]      \n"

  3615.                         "fmla   v13.4s, v8.4s, v0.s[2]      \n"

  3616.                         "fmla   v10.4s, v9.4s, v2.s[3]      \n"

  3617.                         "fmla   v11.4s, v9.4s, v4.s[3]      \n"

  3618.                         "fmla   v12.4s, v9.4s, v6.s[3]      \n"

  3619.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  3620. 

  3621.                         "shrn   v10.4h, v10.4s, #16         \n"

  3622.                         "shrn   v11.4h, v11.4s, #16         \n"

  3623.                         "shrn   v12.4h, v12.4s, #16         \n"

  3624.                         "shrn   v13.4h, v13.4s, #16         \n"

  3625. 

  3626.                         "st1    {v10.4h, v11.4h, v12.4h, v13.4h}, [%0], #32 \n"

  3627. 

  3628.                         : "=r"(outptr0_bf16), // %0

  3629.                         "=r"(outptr0),      // %1

  3630.                         "=r"(r0),           // %2

  3631.                         "=r"(r1),           // %3

  3632.                         "=r"(r2)            // %4

  3633.                         : "0"(outptr0_bf16),

  3634.                         "1"(outptr0),

  3635.                         "2"(r0),

  3636.                         "3"(r1),

  3637.                         "4"(r2),

  3638.                         "w"(_k00_01), // %10

  3639.                         "w"(_k00_23), // %11

  3640.                         "w"(_k01_01), // %12

  3641.                         "w"(_k01_23), // %13

  3642.                         "w"(_k02_01), // %14

  3643.                         "w"(_k02_23), // %15

  3644.                         "w"(_k10_01), // %16

  3645.                         "w"(_k10_23), // %17

  3646.                         "w"(_k11_01), // %18

  3647.                         "w"(_k11_23), // %19

  3648.                         "w"(_k12_01), // %20

  3649.                         "w"(_k12_23), // %21

  3650.                         "w"(_k20_01), // %22

  3651.                         "w"(_k20_23), // %23

  3652.                         "w"(_k21_01), // %24

  3653.                         "w"(_k21_23), // %25

  3654.                         "w"(_k22_01), // %26

  3655.                         "w"(_k22_23)  // %27

  3656.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  3657. #else  // __aarch64__

  3658.                     asm volatile(

  3659.                         "pld        [%1, #512]          \n"

  3660.                         "vldm       %1!, {d24-d31}      \n" // sum0 sum1 sum2 sum3

  3661. 

  3662.                         "pld        [%2, #512]          \n"

  3663.                         "vldm       %2!, {d8-d15}       \n" // r00 r01 r02 r03 r04 r05 r06 r07

  3664. 

  3665.                         "vshll.u16  q0, d8, #16         \n"

  3666.                         "vshll.u16  q1, d9, #16         \n"

  3667.                         "vshll.u16  q2, d10, #16        \n"

  3668.                         "vshll.u16  q3, d11, #16        \n"

  3669. 

  3670.                         "vshll.u16  q4, d12, #16        \n"

  3671.                         "vshll.u16  q5, d13, #16        \n"

  3672.                         "vshll.u16  q6, d14, #16        \n"

  3673.                         "vshll.u16  q7, d15, #16        \n"

  3674. 

  3675.                         "pld        [%5, #256]          \n"

  3676.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3677. 

  3678.                         "vshll.u16  q8, d20, #16        \n"

  3679.                         "vshll.u16  q9, d21, #16        \n"

  3680.                         "vshll.u16  q10, d22, #16       \n"

  3681.                         "vshll.u16  q11, d23, #16       \n"

  3682. 

  3683.                         "vmla.f32   q12, q8, d0[0]      \n"

  3684.                         "vmla.f32   q13, q8, d4[0]      \n"

  3685.                         "vmla.f32   q14, q8, d8[0]      \n"

  3686.                         "vmla.f32   q15, q8, d12[0]     \n"

  3687.                         "vmla.f32   q12, q9, d0[1]      \n"

  3688.                         "vmla.f32   q13, q9, d4[1]      \n"

  3689.                         "vmla.f32   q14, q9, d8[1]      \n"

  3690.                         "vmla.f32   q15, q9, d12[1]     \n"

  3691. 

  3692.                         "vmla.f32   q12, q10, d1[0]     \n"

  3693.                         "vmla.f32   q13, q10, d5[0]     \n"

  3694.                         "vmla.f32   q14, q10, d9[0]     \n"

  3695.                         "vmla.f32   q15, q10, d13[0]    \n"

  3696.                         "vmla.f32   q12, q11, d1[1]     \n"

  3697.                         "vmla.f32   q13, q11, d5[1]     \n"

  3698.                         "vmla.f32   q14, q11, d9[1]     \n"

  3699.                         "vmla.f32   q15, q11, d13[1]    \n"

  3700. 

  3701.                         "pld        [%5, #256]          \n"

  3702.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3703. 

  3704.                         "vshll.u16  q8, d20, #16        \n"

  3705.                         "vshll.u16  q9, d21, #16        \n"

  3706.                         "vshll.u16  q10, d22, #16       \n"

  3707.                         "vshll.u16  q11, d23, #16       \n"

  3708. 

  3709.                         "pld        [%2, #64]           \n"

  3710.                         "vld1.f32   {d1}, [%2 :64]      \n" // r08

  3711. 

  3712.                         "vshll.u16  q0, d1, #16         \n"

  3713. 

  3714.                         "vmla.f32   q12, q8, d2[0]      \n"

  3715.                         "vmla.f32   q13, q8, d6[0]      \n"

  3716.                         "vmla.f32   q14, q8, d10[0]     \n"

  3717.                         "vmla.f32   q15, q8, d14[0]     \n"

  3718.                         "vmla.f32   q12, q9, d2[1]      \n"

  3719.                         "vmla.f32   q13, q9, d6[1]      \n"

  3720.                         "vmla.f32   q14, q9, d10[1]     \n"

  3721.                         "vmla.f32   q15, q9, d14[1]     \n"

  3722. 

  3723.                         "vmla.f32   q12, q10, d3[0]     \n"

  3724.                         "vmla.f32   q13, q10, d7[0]     \n"

  3725.                         "vmla.f32   q14, q10, d11[0]    \n"

  3726.                         "vmla.f32   q15, q10, d15[0]    \n"

  3727.                         "vmla.f32   q12, q11, d3[1]     \n"

  3728.                         "vmla.f32   q13, q11, d7[1]     \n"

  3729.                         "vmla.f32   q14, q11, d11[1]    \n"

  3730.                         "vmla.f32   q15, q11, d15[1]    \n"

  3731. 

  3732.                         "pld        [%5, #256]          \n"

  3733.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3734. 

  3735.                         "vshll.u16  q8, d20, #16        \n"

  3736.                         "vshll.u16  q9, d21, #16        \n"

  3737.                         "vshll.u16  q10, d22, #16       \n"

  3738.                         "vshll.u16  q11, d23, #16       \n"

  3739. 

  3740.                         "vmla.f32   q12, q8, d4[0]      \n"

  3741.                         "vmla.f32   q13, q8, d8[0]      \n"

  3742.                         "vmla.f32   q14, q8, d12[0]     \n"

  3743.                         "vmla.f32   q15, q8, d0[0]      \n"

  3744.                         "vmla.f32   q12, q9, d4[1]      \n"

  3745.                         "vmla.f32   q13, q9, d8[1]      \n"

  3746.                         "vmla.f32   q14, q9, d12[1]     \n"

  3747.                         "vmla.f32   q15, q9, d0[1]      \n"

  3748. 

  3749.                         "vmla.f32   q12, q10, d5[0]     \n"

  3750.                         "vmla.f32   q13, q10, d9[0]     \n"

  3751.                         "vmla.f32   q14, q10, d13[0]    \n"

  3752.                         "vmla.f32   q15, q10, d1[0]     \n"

  3753.                         "vmla.f32   q12, q11, d5[1]     \n"

  3754.                         "vmla.f32   q13, q11, d9[1]     \n"

  3755.                         "vmla.f32   q14, q11, d13[1]    \n"

  3756.                         "vmla.f32   q15, q11, d1[1]     \n"

  3757. 

  3758.                         "pld        [%3, #512]          \n"

  3759.                         "vldm       %3!, {d8-d15}       \n" // r10 r11 r12 r13 r14 r15 r16 r17

  3760. 

  3761.                         "vshll.u16  q0, d8, #16         \n"

  3762.                         "vshll.u16  q1, d9, #16         \n"

  3763.                         "vshll.u16  q2, d10, #16        \n"

  3764.                         "vshll.u16  q3, d11, #16        \n"

  3765. 

  3766.                         "vshll.u16  q4, d12, #16        \n"

  3767.                         "vshll.u16  q5, d13, #16        \n"

  3768.                         "vshll.u16  q6, d14, #16        \n"

  3769.                         "vshll.u16  q7, d15, #16        \n"

  3770. 

  3771.                         "pld        [%5, #256]          \n"

  3772.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3773. 

  3774.                         "vshll.u16  q8, d20, #16        \n"

  3775.                         "vshll.u16  q9, d21, #16        \n"

  3776.                         "vshll.u16  q10, d22, #16       \n"

  3777.                         "vshll.u16  q11, d23, #16       \n"

  3778. 

  3779.                         "vmla.f32   q12, q8, d0[0]      \n"

  3780.                         "vmla.f32   q13, q8, d4[0]      \n"

  3781.                         "vmla.f32   q14, q8, d8[0]      \n"

  3782.                         "vmla.f32   q15, q8, d12[0]     \n"

  3783.                         "vmla.f32   q12, q9, d0[1]      \n"

  3784.                         "vmla.f32   q13, q9, d4[1]      \n"

  3785.                         "vmla.f32   q14, q9, d8[1]      \n"

  3786.                         "vmla.f32   q15, q9, d12[1]     \n"

  3787. 

  3788.                         "vmla.f32   q12, q10, d1[0]     \n"

  3789.                         "vmla.f32   q13, q10, d5[0]     \n"

  3790.                         "vmla.f32   q14, q10, d9[0]     \n"

  3791.                         "vmla.f32   q15, q10, d13[0]    \n"

  3792.                         "vmla.f32   q12, q11, d1[1]     \n"

  3793.                         "vmla.f32   q13, q11, d5[1]     \n"

  3794.                         "vmla.f32   q14, q11, d9[1]     \n"

  3795.                         "vmla.f32   q15, q11, d13[1]    \n"

  3796. 

  3797.                         "pld        [%5, #256]          \n"

  3798.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3799. 

  3800.                         "vshll.u16  q8, d20, #16        \n"

  3801.                         "vshll.u16  q9, d21, #16        \n"

  3802.                         "vshll.u16  q10, d22, #16       \n"

  3803.                         "vshll.u16  q11, d23, #16       \n"

  3804. 

  3805.                         "pld        [%3, #64]           \n"

  3806.                         "vld1.f32   {d1}, [%3 :64]      \n" // r18

  3807. 

  3808.                         "vshll.u16  q0, d1, #16         \n"

  3809. 

  3810.                         "vmla.f32   q12, q8, d2[0]      \n"

  3811.                         "vmla.f32   q13, q8, d6[0]      \n"

  3812.                         "vmla.f32   q14, q8, d10[0]     \n"

  3813.                         "vmla.f32   q15, q8, d14[0]     \n"

  3814.                         "vmla.f32   q12, q9, d2[1]      \n"

  3815.                         "vmla.f32   q13, q9, d6[1]      \n"

  3816.                         "vmla.f32   q14, q9, d10[1]     \n"

  3817.                         "vmla.f32   q15, q9, d14[1]     \n"

  3818. 

  3819.                         "vmla.f32   q12, q10, d3[0]     \n"

  3820.                         "vmla.f32   q13, q10, d7[0]     \n"

  3821.                         "vmla.f32   q14, q10, d11[0]    \n"

  3822.                         "vmla.f32   q15, q10, d15[0]    \n"

  3823.                         "vmla.f32   q12, q11, d3[1]     \n"

  3824.                         "vmla.f32   q13, q11, d7[1]     \n"

  3825.                         "vmla.f32   q14, q11, d11[1]    \n"

  3826.                         "vmla.f32   q15, q11, d15[1]    \n"

  3827. 

  3828.                         "pld        [%5, #256]          \n"

  3829.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3830. 

  3831.                         "vshll.u16  q8, d20, #16        \n"

  3832.                         "vshll.u16  q9, d21, #16        \n"

  3833.                         "vshll.u16  q10, d22, #16       \n"

  3834.                         "vshll.u16  q11, d23, #16       \n"

  3835. 

  3836.                         "vmla.f32   q12, q8, d4[0]      \n"

  3837.                         "vmla.f32   q13, q8, d8[0]      \n"

  3838.                         "vmla.f32   q14, q8, d12[0]     \n"

  3839.                         "vmla.f32   q15, q8, d0[0]      \n"

  3840.                         "vmla.f32   q12, q9, d4[1]      \n"

  3841.                         "vmla.f32   q13, q9, d8[1]      \n"

  3842.                         "vmla.f32   q14, q9, d12[1]     \n"

  3843.                         "vmla.f32   q15, q9, d0[1]      \n"

  3844. 

  3845.                         "vmla.f32   q12, q10, d5[0]     \n"

  3846.                         "vmla.f32   q13, q10, d9[0]     \n"

  3847.                         "vmla.f32   q14, q10, d13[0]    \n"

  3848.                         "vmla.f32   q15, q10, d1[0]     \n"

  3849.                         "vmla.f32   q12, q11, d5[1]     \n"

  3850.                         "vmla.f32   q13, q11, d9[1]     \n"

  3851.                         "vmla.f32   q14, q11, d13[1]    \n"

  3852.                         "vmla.f32   q15, q11, d1[1]     \n"

  3853. 

  3854.                         "pld        [%4, #256]          \n"

  3855.                         "vldm       %4!, {d8-d15}       \n" // r20 r21 r22 r23 r24 r25 r26 r27

  3856. 

  3857.                         "vshll.u16  q0, d8, #16         \n"

  3858.                         "vshll.u16  q1, d9, #16         \n"

  3859.                         "vshll.u16  q2, d10, #16        \n"

  3860.                         "vshll.u16  q3, d11, #16        \n"

  3861. 

  3862.                         "vshll.u16  q4, d12, #16        \n"

  3863.                         "vshll.u16  q5, d13, #16        \n"

  3864.                         "vshll.u16  q6, d14, #16        \n"

  3865.                         "vshll.u16  q7, d15, #16        \n"

  3866. 

  3867.                         "pld        [%5, #256]          \n"

  3868.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3869. 

  3870.                         "vshll.u16  q8, d20, #16        \n"

  3871.                         "vshll.u16  q9, d21, #16        \n"

  3872.                         "vshll.u16  q10, d22, #16       \n"

  3873.                         "vshll.u16  q11, d23, #16       \n"

  3874. 

  3875.                         "vmla.f32   q12, q8, d0[0]      \n"

  3876.                         "vmla.f32   q13, q8, d4[0]      \n"

  3877.                         "vmla.f32   q14, q8, d8[0]      \n"

  3878.                         "vmla.f32   q15, q8, d12[0]     \n"

  3879.                         "vmla.f32   q12, q9, d0[1]      \n"

  3880.                         "vmla.f32   q13, q9, d4[1]      \n"

  3881.                         "vmla.f32   q14, q9, d8[1]      \n"

  3882.                         "vmla.f32   q15, q9, d12[1]     \n"

  3883. 

  3884.                         "vmla.f32   q12, q10, d1[0]     \n"

  3885.                         "vmla.f32   q13, q10, d5[0]     \n"

  3886.                         "vmla.f32   q14, q10, d9[0]     \n"

  3887.                         "vmla.f32   q15, q10, d13[0]    \n"

  3888.                         "vmla.f32   q12, q11, d1[1]     \n"

  3889.                         "vmla.f32   q13, q11, d5[1]     \n"

  3890.                         "vmla.f32   q14, q11, d9[1]     \n"

  3891.                         "vmla.f32   q15, q11, d13[1]    \n"

  3892. 

  3893.                         "pld        [%5, #256]          \n"

  3894.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  3895. 

  3896.                         "vshll.u16  q8, d20, #16        \n"

  3897.                         "vshll.u16  q9, d21, #16        \n"

  3898.                         "vshll.u16  q10, d22, #16       \n"

  3899.                         "vshll.u16  q11, d23, #16       \n"

  3900. 

  3901.                         "pld        [%4, #64]           \n"

  3902.                         "vld1.f32   {d1}, [%4 :64]      \n" // r28

  3903. 

  3904.                         "vshll.u16  q0, d1, #16         \n"

  3905. 

  3906.                         "vmla.f32   q12, q8, d2[0]      \n"

  3907.                         "vmla.f32   q13, q8, d6[0]      \n"

  3908.                         "vmla.f32   q14, q8, d10[0]     \n"

  3909.                         "vmla.f32   q15, q8, d14[0]     \n"

  3910.                         "vmla.f32   q12, q9, d2[1]      \n"

  3911.                         "vmla.f32   q13, q9, d6[1]      \n"

  3912.                         "vmla.f32   q14, q9, d10[1]     \n"

  3913.                         "vmla.f32   q15, q9, d14[1]     \n"

  3914. 

  3915.                         "vmla.f32   q12, q10, d3[0]     \n"

  3916.                         "vmla.f32   q13, q10, d7[0]     \n"

  3917.                         "vmla.f32   q14, q10, d11[0]    \n"

  3918.                         "vmla.f32   q15, q10, d15[0]    \n"

  3919.                         "vmla.f32   q12, q11, d3[1]     \n"

  3920.                         "vmla.f32   q13, q11, d7[1]     \n"

  3921.                         "vmla.f32   q14, q11, d11[1]    \n"

  3922.                         "vmla.f32   q15, q11, d15[1]    \n"

  3923. 

  3924.                         //                         "pld        [%5, #256]          \n"

  3925.                         "vld1.u16   {d20-d23}, [%5 :128] \n"

  3926. 

  3927.                         "vshll.u16  q8, d20, #16        \n"

  3928.                         "vshll.u16  q9, d21, #16        \n"

  3929.                         "vshll.u16  q10, d22, #16       \n"

  3930.                         "vshll.u16  q11, d23, #16       \n"

  3931. 

  3932.                         "vmla.f32   q12, q8, d4[0]      \n"

  3933.                         "vmla.f32   q13, q8, d8[0]      \n"

  3934.                         "vmla.f32   q14, q8, d12[0]     \n"

  3935.                         "vmla.f32   q15, q8, d0[0]      \n"

  3936.                         "vmla.f32   q12, q9, d4[1]      \n"

  3937.                         "vmla.f32   q13, q9, d8[1]      \n"

  3938.                         "vmla.f32   q14, q9, d12[1]     \n"

  3939.                         "vmla.f32   q15, q9, d0[1]      \n"

  3940. 

  3941.                         "vmla.f32   q12, q10, d5[0]     \n"

  3942.                         "vmla.f32   q13, q10, d9[0]     \n"

  3943.                         "vmla.f32   q14, q10, d13[0]    \n"

  3944.                         "vmla.f32   q15, q10, d1[0]     \n"

  3945.                         "vmla.f32   q12, q11, d5[1]     \n"

  3946.                         "vmla.f32   q13, q11, d9[1]     \n"

  3947.                         "vmla.f32   q14, q11, d13[1]    \n"

  3948.                         "vmla.f32   q15, q11, d1[1]     \n"

  3949. 

  3950.                         "sub        %5, %5, #256        \n" // kptr -= 8 * 16;

  3951. 

  3952.                         "vshrn.u32  d24, q12, #16       \n"

  3953.                         "vshrn.u32  d25, q13, #16       \n"

  3954.                         "vshrn.u32  d26, q14, #16       \n"

  3955.                         "vshrn.u32  d27, q15, #16       \n"

  3956. 

  3957.                         "vst1.f32   {d24-d27}, [%0 :64]! \n"

  3958. 

  3959.                         : "=r"(outptr0_bf16), // %0

  3960.                         "=r"(outptr0),      // %1

  3961.                         "=r"(r0),           // %2

  3962.                         "=r"(r1),           // %3

  3963.                         "=r"(r2),           // %4

  3964.                         "=r"(kptr)          // %5

  3965.                         : "0"(outptr0_bf16),

  3966.                         "1"(outptr0),

  3967.                         "2"(r0),

  3968.                         "3"(r1),

  3969.                         "4"(r2),

  3970.                         "5"(kptr)

  3971.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  3972. #endif // __aarch64__

  3973.                 }

  3974.                 for (; j + 1 < outw; j += 2)

  3975.                 {

  3976. #if __aarch64__

  3977.                     asm volatile(

  3978.                         "prfm   pldl1keep, [%1, #256]       \n"

  3979.                         "ld1    {v12.4s, v13.4s}, [%1], #32 \n" // sum0 sum1

  3980. 

  3981.                         "prfm   pldl1keep, [%2, #256]       \n"

  3982.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r00 r01 r02 r03

  3983. 

  3984.                         "shll   v0.4s, v0.4h, #16           \n"

  3985.                         "shll   v1.4s, v1.4h, #16           \n"

  3986.                         "shll   v2.4s, v2.4h, #16           \n"

  3987.                         "shll   v3.4s, v3.4h, #16           \n"

  3988. 

  3989.                         "shll   v6.4s, %10.4h, #16          \n"

  3990.                         "shll2  v7.4s, %10.8h, #16          \n"

  3991. 

  3992.                         "fmul   v10.4s, v6.4s, v0.s[0]      \n"

  3993.                         "fmul   v11.4s, v6.4s, v2.s[0]      \n"

  3994.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  3995.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  3996. 

  3997.                         "shll   v8.4s, %11.4h, #16          \n"

  3998.                         "shll2  v9.4s, %11.8h, #16          \n"

  3999. 

  4000.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  4001.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  4002.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  4003.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  4004. 

  4005.                         "prfm   pldl1keep, [%2, #64]        \n"

  4006.                         "ld1    {v4.4h}, [%2]               \n" // r04

  4007. 

  4008.                         "shll   v4.4s, v4.4h, #16           \n"

  4009. 

  4010.                         "shll   v6.4s, %12.4h, #16          \n"

  4011.                         "shll2  v7.4s, %12.8h, #16          \n"

  4012. 

  4013.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  4014.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  4015.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  4016.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  4017. 

  4018.                         "shll   v8.4s, %13.4h, #16          \n"

  4019.                         "shll2  v9.4s, %13.8h, #16          \n"

  4020. 

  4021.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  4022.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  4023.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  4024.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  4025. 

  4026.                         "shll   v6.4s, %14.4h, #16          \n"

  4027.                         "shll2  v7.4s, %14.8h, #16          \n"

  4028. 

  4029.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  4030.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  4031.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  4032.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  4033. 

  4034.                         "shll   v8.4s, %15.4h, #16          \n"

  4035.                         "shll2  v9.4s, %15.8h, #16          \n"

  4036. 

  4037.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  4038.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  4039.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  4040.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  4041. 

  4042.                         "prfm   pldl1keep, [%3, #256]       \n"

  4043.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r10 r11 r12 r13

  4044. 

  4045.                         "shll   v0.4s, v0.4h, #16           \n"

  4046.                         "shll   v1.4s, v1.4h, #16           \n"

  4047.                         "shll   v2.4s, v2.4h, #16           \n"

  4048.                         "shll   v3.4s, v3.4h, #16           \n"

  4049. 

  4050.                         "shll   v6.4s, %16.4h, #16          \n"

  4051.                         "shll2  v7.4s, %16.8h, #16          \n"

  4052. 

  4053.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  4054.                         "fmla   v11.4s, v6.4s, v2.s[0]      \n"

  4055.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  4056.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  4057. 

  4058.                         "shll   v8.4s, %17.4h, #16          \n"

  4059.                         "shll2  v9.4s, %17.8h, #16          \n"

  4060. 

  4061.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  4062.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  4063.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  4064.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  4065. 

  4066.                         "prfm   pldl1keep, [%3, #64]        \n"

  4067.                         "ld1    {v4.4h}, [%3]               \n" // r14

  4068. 

  4069.                         "shll   v4.4s, v4.4h, #16           \n"

  4070. 

  4071.                         "shll   v6.4s, %18.4h, #16          \n"

  4072.                         "shll2  v7.4s, %18.8h, #16          \n"

  4073. 

  4074.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  4075.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  4076.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  4077.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  4078. 

  4079.                         "shll   v8.4s, %19.4h, #16          \n"

  4080.                         "shll2  v9.4s, %19.8h, #16          \n"

  4081. 

  4082.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  4083.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  4084.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  4085.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  4086. 

  4087.                         "shll   v6.4s, %20.4h, #16          \n"

  4088.                         "shll2  v7.4s, %20.8h, #16          \n"

  4089. 

  4090.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  4091.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  4092.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  4093.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  4094. 

  4095.                         "shll   v8.4s, %21.4h, #16          \n"

  4096.                         "shll2  v9.4s, %21.8h, #16          \n"

  4097. 

  4098.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  4099.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  4100.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  4101.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  4102. 

  4103.                         "prfm   pldl1keep, [%4, #256]       \n"

  4104.                         "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%4], #32 \n" // r20 r21 r22 r23

  4105. 

  4106.                         "shll   v0.4s, v0.4h, #16           \n"

  4107.                         "shll   v1.4s, v1.4h, #16           \n"

  4108.                         "shll   v2.4s, v2.4h, #16           \n"

  4109.                         "shll   v3.4s, v3.4h, #16           \n"

  4110. 

  4111.                         "shll   v6.4s, %22.4h, #16          \n"

  4112.                         "shll2  v7.4s, %22.8h, #16          \n"

  4113. 

  4114.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  4115.                         "fmla   v11.4s, v6.4s, v2.s[0]      \n"

  4116.                         "fmla   v12.4s, v7.4s, v0.s[1]      \n"

  4117.                         "fmla   v13.4s, v7.4s, v2.s[1]      \n"

  4118. 

  4119.                         "shll   v8.4s, %23.4h, #16          \n"

  4120.                         "shll2  v9.4s, %23.8h, #16          \n"

  4121. 

  4122.                         "fmla   v10.4s, v8.4s, v0.s[2]      \n"

  4123.                         "fmla   v11.4s, v8.4s, v2.s[2]      \n"

  4124.                         "fmla   v12.4s, v9.4s, v0.s[3]      \n"

  4125.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  4126. 

  4127.                         "prfm   pldl1keep, [%4, #64]        \n"

  4128.                         "ld1    {v4.4h}, [%4]               \n" // r24

  4129. 

  4130.                         "shll   v4.4s, v4.4h, #16           \n"

  4131. 

  4132.                         "shll   v6.4s, %24.4h, #16          \n"

  4133.                         "shll2  v7.4s, %24.8h, #16          \n"

  4134. 

  4135.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  4136.                         "fmla   v11.4s, v6.4s, v3.s[0]      \n"

  4137.                         "fmla   v12.4s, v7.4s, v1.s[1]      \n"

  4138.                         "fmla   v13.4s, v7.4s, v3.s[1]      \n"

  4139. 

  4140.                         "shll   v8.4s, %25.4h, #16          \n"

  4141.                         "shll2  v9.4s, %25.8h, #16          \n"

  4142. 

  4143.                         "fmla   v10.4s, v8.4s, v1.s[2]      \n"

  4144.                         "fmla   v11.4s, v8.4s, v3.s[2]      \n"

  4145.                         "fmla   v12.4s, v9.4s, v1.s[3]      \n"

  4146.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  4147. 

  4148.                         "shll   v6.4s, %26.4h, #16          \n"

  4149.                         "shll2  v7.4s, %26.8h, #16          \n"

  4150. 

  4151.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  4152.                         "fmla   v11.4s, v6.4s, v4.s[0]      \n"

  4153.                         "fmla   v12.4s, v7.4s, v2.s[1]      \n"

  4154.                         "fmla   v13.4s, v7.4s, v4.s[1]      \n"

  4155. 

  4156.                         "shll   v8.4s, %27.4h, #16          \n"

  4157.                         "shll2  v9.4s, %27.8h, #16          \n"

  4158. 

  4159.                         "fmla   v10.4s, v8.4s, v2.s[2]      \n"

  4160.                         "fmla   v11.4s, v8.4s, v4.s[2]      \n"

  4161.                         "fmla   v12.4s, v9.4s, v2.s[3]      \n"

  4162.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  4163. 

  4164.                         "fadd   v12.4s, v10.4s, v12.4s      \n"

  4165.                         "fadd   v13.4s, v11.4s, v13.4s      \n"

  4166. 

  4167.                         "shrn   v12.4h, v12.4s, #16         \n"

  4168.                         "shrn   v13.4h, v13.4s, #16         \n"

  4169. 

  4170.                         "st1    {v12.4h, v13.4h}, [%0], #16 \n"

  4171. 

  4172.                         : "=r"(outptr0_bf16), // %0

  4173.                         "=r"(outptr0),      // %1

  4174.                         "=r"(r0),           // %2

  4175.                         "=r"(r1),           // %3

  4176.                         "=r"(r2)            // %4

  4177.                         : "0"(outptr0_bf16),

  4178.                         "1"(outptr0),

  4179.                         "2"(r0),

  4180.                         "3"(r1),

  4181.                         "4"(r2),

  4182.                         "w"(_k00_01), // %10

  4183.                         "w"(_k00_23), // %11

  4184.                         "w"(_k01_01), // %12

  4185.                         "w"(_k01_23), // %13

  4186.                         "w"(_k02_01), // %14

  4187.                         "w"(_k02_23), // %15

  4188.                         "w"(_k10_01), // %16

  4189.                         "w"(_k10_23), // %17

  4190.                         "w"(_k11_01), // %18

  4191.                         "w"(_k11_23), // %19

  4192.                         "w"(_k12_01), // %20

  4193.                         "w"(_k12_23), // %21

  4194.                         "w"(_k20_01), // %22

  4195.                         "w"(_k20_23), // %23

  4196.                         "w"(_k21_01), // %24

  4197.                         "w"(_k21_23), // %25

  4198.                         "w"(_k22_01), // %26

  4199.                         "w"(_k22_23)  // %27

  4200.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  4201. #else  // __aarch64__

  4202.                     asm volatile(

  4203.                         "pld        [%1, #256]          \n"

  4204.                         "vld1.f32   {d28-d31}, [%1 :128]! \n" // sum0 sum1

  4205. 

  4206.                         "pld        [%2, #256]          \n"

  4207.                         "vld1.u16   {d4-d7}, [%2 :64]!  \n" // r00 r01 r02 r03

  4208. 

  4209.                         "vshll.u16  q0, d4, #16         \n"

  4210.                         "vshll.u16  q1, d5, #16         \n"

  4211.                         "vshll.u16  q2, d6, #16         \n"

  4212.                         "vshll.u16  q3, d7, #16         \n"

  4213. 

  4214.                         "pld        [%5, #256]          \n"

  4215.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4216. 

  4217.                         "vshll.u16  q8, d20, #16        \n"

  4218.                         "vshll.u16  q9, d21, #16        \n"

  4219.                         "vshll.u16  q10, d22, #16       \n"

  4220.                         "vshll.u16  q11, d23, #16       \n"

  4221. 

  4222.                         "vmul.f32   q12, q8, d0[0]      \n"

  4223.                         "vmul.f32   q13, q8, d4[0]      \n"

  4224.                         "vmla.f32   q14, q9, d0[1]      \n"

  4225.                         "vmla.f32   q15, q9, d4[1]      \n"

  4226. 

  4227.                         "vmla.f32   q12, q10, d1[0]     \n"

  4228.                         "vmla.f32   q13, q10, d5[0]     \n"

  4229.                         "vmla.f32   q14, q11, d1[1]     \n"

  4230.                         "vmla.f32   q15, q11, d5[1]     \n"

  4231. 

  4232.                         "pld        [%5, #256]          \n"

  4233.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4234. 

  4235.                         "vshll.u16  q8, d20, #16        \n"

  4236.                         "vshll.u16  q9, d21, #16        \n"

  4237.                         "vshll.u16  q10, d22, #16       \n"

  4238.                         "vshll.u16  q11, d23, #16       \n"

  4239. 

  4240.                         "pld        [%2, #64]           \n"

  4241.                         "vld1.f32   {d9}, [%2 :64]      \n" // r04

  4242. 

  4243.                         "vshll.u16  q4, d9, #16         \n"

  4244. 

  4245.                         "vmla.f32   q12, q8, d2[0]      \n"

  4246.                         "vmla.f32   q13, q8, d6[0]      \n"

  4247.                         "vmla.f32   q14, q9, d2[1]      \n"

  4248.                         "vmla.f32   q15, q9, d6[1]      \n"

  4249. 

  4250.                         "vmla.f32   q12, q10, d3[0]     \n"

  4251.                         "vmla.f32   q13, q10, d7[0]     \n"

  4252.                         "vmla.f32   q14, q11, d3[1]     \n"

  4253.                         "vmla.f32   q15, q11, d7[1]     \n"

  4254. 

  4255.                         "pld        [%5, #256]          \n"

  4256.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4257. 

  4258.                         "vshll.u16  q8, d20, #16        \n"

  4259.                         "vshll.u16  q9, d21, #16        \n"

  4260.                         "vshll.u16  q10, d22, #16       \n"

  4261.                         "vshll.u16  q11, d23, #16       \n"

  4262. 

  4263.                         "vmla.f32   q12, q8, d4[0]      \n"

  4264.                         "vmla.f32   q13, q8, d8[0]      \n"

  4265.                         "vmla.f32   q14, q9, d4[1]      \n"

  4266.                         "vmla.f32   q15, q9, d8[1]      \n"

  4267. 

  4268.                         "vmla.f32   q12, q10, d5[0]     \n"

  4269.                         "vmla.f32   q13, q10, d9[0]     \n"

  4270.                         "vmla.f32   q14, q11, d5[1]     \n"

  4271.                         "vmla.f32   q15, q11, d9[1]     \n"

  4272. 

  4273.                         "pld        [%3, #256]          \n"

  4274.                         "vld1.u16   {d4-d7}, [%3 :64]!  \n" // r10 r11 r12 r13

  4275. 

  4276.                         "vshll.u16  q0, d4, #16         \n"

  4277.                         "vshll.u16  q1, d5, #16         \n"

  4278.                         "vshll.u16  q2, d6, #16         \n"

  4279.                         "vshll.u16  q3, d7, #16         \n"

  4280. 

  4281.                         "pld        [%5, #256]          \n"

  4282.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4283. 

  4284.                         "vshll.u16  q8, d20, #16        \n"

  4285.                         "vshll.u16  q9, d21, #16        \n"

  4286.                         "vshll.u16  q10, d22, #16       \n"

  4287.                         "vshll.u16  q11, d23, #16       \n"

  4288. 

  4289.                         "vmla.f32   q12, q8, d0[0]      \n"

  4290.                         "vmla.f32   q13, q8, d4[0]      \n"

  4291.                         "vmla.f32   q14, q9, d0[1]      \n"

  4292.                         "vmla.f32   q15, q9, d4[1]      \n"

  4293. 

  4294.                         "vmla.f32   q12, q10, d1[0]     \n"

  4295.                         "vmla.f32   q13, q10, d5[0]     \n"

  4296.                         "vmla.f32   q14, q11, d1[1]     \n"

  4297.                         "vmla.f32   q15, q11, d5[1]     \n"

  4298. 

  4299.                         "pld        [%5, #256]          \n"

  4300.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4301. 

  4302.                         "vshll.u16  q8, d20, #16        \n"

  4303.                         "vshll.u16  q9, d21, #16        \n"

  4304.                         "vshll.u16  q10, d22, #16       \n"

  4305.                         "vshll.u16  q11, d23, #16       \n"

  4306. 

  4307.                         "pld        [%3, #64]           \n"

  4308.                         "vld1.f32   {d9}, [%3 :64]      \n" // r14

  4309. 

  4310.                         "vshll.u16  q4, d9, #16         \n"

  4311. 

  4312.                         "vmla.f32   q12, q8, d2[0]      \n"

  4313.                         "vmla.f32   q13, q8, d6[0]      \n"

  4314.                         "vmla.f32   q14, q9, d2[1]      \n"

  4315.                         "vmla.f32   q15, q9, d6[1]      \n"

  4316. 

  4317.                         "vmla.f32   q12, q10, d3[0]     \n"

  4318.                         "vmla.f32   q13, q10, d7[0]     \n"

  4319.                         "vmla.f32   q14, q11, d3[1]     \n"

  4320.                         "vmla.f32   q15, q11, d7[1]     \n"

  4321. 

  4322.                         "pld        [%5, #256]          \n"

  4323.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4324. 

  4325.                         "vshll.u16  q8, d20, #16        \n"

  4326.                         "vshll.u16  q9, d21, #16        \n"

  4327.                         "vshll.u16  q10, d22, #16       \n"

  4328.                         "vshll.u16  q11, d23, #16       \n"

  4329. 

  4330.                         "vmla.f32   q12, q8, d4[0]      \n"

  4331.                         "vmla.f32   q13, q8, d8[0]      \n"

  4332.                         "vmla.f32   q14, q9, d4[1]      \n"

  4333.                         "vmla.f32   q15, q9, d8[1]      \n"

  4334. 

  4335.                         "vmla.f32   q12, q10, d5[0]     \n"

  4336.                         "vmla.f32   q13, q10, d9[0]     \n"

  4337.                         "vmla.f32   q14, q11, d5[1]     \n"

  4338.                         "vmla.f32   q15, q11, d9[1]     \n"

  4339. 

  4340.                         "pld        [%4, #256]          \n"

  4341.                         "vld1.u16   {d4-d7}, [%4 :64]!  \n" // r20 r21 r22 r23

  4342. 

  4343.                         "vshll.u16  q0, d4, #16         \n"

  4344.                         "vshll.u16  q1, d5, #16         \n"

  4345.                         "vshll.u16  q2, d6, #16         \n"

  4346.                         "vshll.u16  q3, d7, #16         \n"

  4347. 

  4348.                         "pld        [%5, #256]          \n"

  4349.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4350. 

  4351.                         "vshll.u16  q8, d20, #16        \n"

  4352.                         "vshll.u16  q9, d21, #16        \n"

  4353.                         "vshll.u16  q10, d22, #16       \n"

  4354.                         "vshll.u16  q11, d23, #16       \n"

  4355. 

  4356.                         "vmla.f32   q12, q8, d0[0]      \n"

  4357.                         "vmla.f32   q13, q8, d4[0]      \n"

  4358.                         "vmla.f32   q14, q9, d0[1]      \n"

  4359.                         "vmla.f32   q15, q9, d4[1]      \n"

  4360. 

  4361.                         "vmla.f32   q12, q10, d1[0]     \n"

  4362.                         "vmla.f32   q13, q10, d5[0]     \n"

  4363.                         "vmla.f32   q14, q11, d1[1]     \n"

  4364.                         "vmla.f32   q15, q11, d5[1]     \n"

  4365. 

  4366.                         "pld        [%5, #256]          \n"

  4367.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4368. 

  4369.                         "vshll.u16  q8, d20, #16        \n"

  4370.                         "vshll.u16  q9, d21, #16        \n"

  4371.                         "vshll.u16  q10, d22, #16       \n"

  4372.                         "vshll.u16  q11, d23, #16       \n"

  4373. 

  4374.                         "pld        [%4, #64]           \n"

  4375.                         "vld1.f32   {d9}, [%4 :64]      \n" // r24

  4376. 

  4377.                         "vshll.u16  q4, d9, #16         \n"

  4378. 

  4379.                         "vmla.f32   q12, q8, d2[0]      \n"

  4380.                         "vmla.f32   q13, q8, d6[0]      \n"

  4381.                         "vmla.f32   q14, q9, d2[1]      \n"

  4382.                         "vmla.f32   q15, q9, d6[1]      \n"

  4383. 

  4384.                         "vmla.f32   q12, q10, d3[0]     \n"

  4385.                         "vmla.f32   q13, q10, d7[0]     \n"

  4386.                         "vmla.f32   q14, q11, d3[1]     \n"

  4387.                         "vmla.f32   q15, q11, d7[1]     \n"

  4388. 

  4389.                         //                         "pld        [%5, #256]          \n"

  4390.                         "vld1.u16   {d20-d23}, [%5 :128] \n"

  4391. 

  4392.                         "vshll.u16  q8, d20, #16        \n"

  4393.                         "vshll.u16  q9, d21, #16        \n"

  4394.                         "vshll.u16  q10, d22, #16       \n"

  4395.                         "vshll.u16  q11, d23, #16       \n"

  4396. 

  4397.                         "vmla.f32   q12, q8, d4[0]      \n"

  4398.                         "vmla.f32   q13, q8, d8[0]      \n"

  4399.                         "vmla.f32   q14, q9, d4[1]      \n"

  4400.                         "vmla.f32   q15, q9, d8[1]      \n"

  4401. 

  4402.                         "vmla.f32   q12, q10, d5[0]     \n"

  4403.                         "vmla.f32   q13, q10, d9[0]     \n"

  4404.                         "vmla.f32   q14, q11, d5[1]     \n"

  4405.                         "vmla.f32   q15, q11, d9[1]     \n"

  4406. 

  4407.                         "vadd.f32   q14, q12, q14       \n"

  4408.                         "vadd.f32   q15, q13, q15       \n"

  4409. 

  4410.                         "sub        %5, %5, #256        \n" // kptr -= 8 * 16;

  4411. 

  4412.                         "vshrn.u32  d28, q14, #16       \n"

  4413.                         "vshrn.u32  d29, q15, #16       \n"

  4414. 

  4415.                         "vst1.f32   {d28-d29}, [%0 :64]! \n"

  4416. 

  4417.                         : "=r"(outptr0_bf16), // %0

  4418.                         "=r"(outptr0),      // %1

  4419.                         "=r"(r0),           // %2

  4420.                         "=r"(r1),           // %3

  4421.                         "=r"(r2),           // %4

  4422.                         "=r"(kptr)          // %5

  4423.                         : "0"(outptr0_bf16),

  4424.                         "1"(outptr0),

  4425.                         "2"(r0),

  4426.                         "3"(r1),

  4427.                         "4"(r2),

  4428.                         "5"(kptr)

  4429.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  4430. #endif // __aarch64__

  4431.                 }

  4432.                 for (; j < outw; j++)

  4433.                 {

  4434. #if __aarch64__

  4435.                     asm volatile(

  4436.                         "prfm   pldl1keep, [%1, #128]       \n"

  4437.                         "ld1    {v13.4s}, [%1], #16         \n" // sum0

  4438. 

  4439.                         "prfm   pldl1keep, [%2, #192]       \n"

  4440.                         "ld1    {v0.4h, v1.4h, v2.4h}, [%2] \n" // r00 r01 r02

  4441. 

  4442.                         "shll   v0.4s, v0.4h, #16           \n"

  4443.                         "shll   v1.4s, v1.4h, #16           \n"

  4444.                         "shll   v2.4s, v2.4h, #16           \n"

  4445. 

  4446.                         "shll   v6.4s, %10.4h, #16          \n"

  4447.                         "shll2  v7.4s, %10.8h, #16          \n"

  4448. 

  4449.                         "fmul   v10.4s, v6.4s, v0.s[0]      \n"

  4450.                         "fmul   v11.4s, v7.4s, v0.s[1]      \n"

  4451. 

  4452.                         "shll   v8.4s, %11.4h, #16          \n"

  4453.                         "shll2  v9.4s, %11.8h, #16          \n"

  4454. 

  4455.                         "fmul   v12.4s, v8.4s, v0.s[2]      \n"

  4456.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  4457. 

  4458.                         "shll   v6.4s, %12.4h, #16          \n"

  4459.                         "shll2  v7.4s, %12.8h, #16          \n"

  4460. 

  4461.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  4462.                         "fmla   v11.4s, v7.4s, v1.s[1]      \n"

  4463. 

  4464.                         "shll   v8.4s, %13.4h, #16          \n"

  4465.                         "shll2  v9.4s, %13.8h, #16          \n"

  4466. 

  4467.                         "fmla   v12.4s, v8.4s, v1.s[2]      \n"

  4468.                         "fmla   v13.4s, v9.4s, v1.s[3]      \n"

  4469. 

  4470.                         "shll   v6.4s, %14.4h, #16          \n"

  4471.                         "shll2  v7.4s, %14.8h, #16          \n"

  4472. 

  4473.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  4474.                         "fmla   v11.4s, v7.4s, v2.s[1]      \n"

  4475. 

  4476.                         "shll   v8.4s, %15.4h, #16          \n"

  4477.                         "shll2  v9.4s, %15.8h, #16          \n"

  4478. 

  4479.                         "fmla   v12.4s, v8.4s, v2.s[2]      \n"

  4480.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  4481. 

  4482.                         "prfm   pldl1keep, [%3, #192]       \n"

  4483.                         "ld1    {v3.4h, v4.4h, v5.4h}, [%3] \n" // r10 r11 r12

  4484. 

  4485.                         "shll   v3.4s, v3.4h, #16           \n"

  4486.                         "shll   v4.4s, v4.4h, #16           \n"

  4487.                         "shll   v5.4s, v5.4h, #16           \n"

  4488. 

  4489.                         "shll   v6.4s, %16.4h, #16          \n"

  4490.                         "shll2  v7.4s, %16.8h, #16          \n"

  4491. 

  4492.                         "fmla   v10.4s, v6.4s, v3.s[0]      \n"

  4493.                         "fmla   v11.4s, v7.4s, v3.s[1]      \n"

  4494. 

  4495.                         "shll   v8.4s, %17.4h, #16          \n"

  4496.                         "shll2  v9.4s, %17.8h, #16          \n"

  4497. 

  4498.                         "fmla   v12.4s, v8.4s, v3.s[2]      \n"

  4499.                         "fmla   v13.4s, v9.4s, v3.s[3]      \n"

  4500. 

  4501.                         "shll   v6.4s, %18.4h, #16          \n"

  4502.                         "shll2  v7.4s, %18.8h, #16          \n"

  4503. 

  4504.                         "fmla   v10.4s, v6.4s, v4.s[0]      \n"

  4505.                         "fmla   v11.4s, v7.4s, v4.s[1]      \n"

  4506. 

  4507.                         "shll   v8.4s, %19.4h, #16          \n"

  4508.                         "shll2  v9.4s, %19.8h, #16          \n"

  4509. 

  4510.                         "fmla   v12.4s, v8.4s, v4.s[2]      \n"

  4511.                         "fmla   v13.4s, v9.4s, v4.s[3]      \n"

  4512. 

  4513.                         "shll   v6.4s, %20.4h, #16          \n"

  4514.                         "shll2  v7.4s, %20.8h, #16          \n"

  4515. 

  4516.                         "fmla   v10.4s, v6.4s, v5.s[0]      \n"

  4517.                         "fmla   v11.4s, v7.4s, v5.s[1]      \n"

  4518. 

  4519.                         "shll   v8.4s, %21.4h, #16          \n"

  4520.                         "shll2  v9.4s, %21.8h, #16          \n"

  4521. 

  4522.                         "fmla   v12.4s, v8.4s, v5.s[2]      \n"

  4523.                         "fmla   v13.4s, v9.4s, v5.s[3]      \n"

  4524. 

  4525.                         "prfm   pldl1keep, [%4, #192]       \n"

  4526.                         "ld1    {v0.4h, v1.4h, v2.4h}, [%4] \n" // r20 r21 r22

  4527. 

  4528.                         "shll   v0.4s, v0.4h, #16           \n"

  4529.                         "shll   v1.4s, v1.4h, #16           \n"

  4530.                         "shll   v2.4s, v2.4h, #16           \n"

  4531. 

  4532.                         "shll   v6.4s, %22.4h, #16          \n"

  4533.                         "shll2  v7.4s, %22.8h, #16          \n"

  4534. 

  4535.                         "fmla   v10.4s, v6.4s, v0.s[0]      \n"

  4536.                         "fmla   v11.4s, v7.4s, v0.s[1]      \n"

  4537. 

  4538.                         "shll   v8.4s, %23.4h, #16          \n"

  4539.                         "shll2  v9.4s, %23.8h, #16          \n"

  4540. 

  4541.                         "fmla   v12.4s, v8.4s, v0.s[2]      \n"

  4542.                         "fmla   v13.4s, v9.4s, v0.s[3]      \n"

  4543. 

  4544.                         "shll   v6.4s, %24.4h, #16          \n"

  4545.                         "shll2  v7.4s, %24.8h, #16          \n"

  4546. 

  4547.                         "fmla   v10.4s, v6.4s, v1.s[0]      \n"

  4548.                         "fmla   v11.4s, v7.4s, v1.s[1]      \n"

  4549. 

  4550.                         "shll   v8.4s, %25.4h, #16          \n"

  4551.                         "shll2  v9.4s, %25.8h, #16          \n"

  4552. 

  4553.                         "fmla   v12.4s, v8.4s, v1.s[2]      \n"

  4554.                         "fmla   v13.4s, v9.4s, v1.s[3]      \n"

  4555. 

  4556.                         "shll   v6.4s, %26.4h, #16          \n"

  4557.                         "shll2  v7.4s, %26.8h, #16          \n"

  4558. 

  4559.                         "fmla   v10.4s, v6.4s, v2.s[0]      \n"

  4560.                         "fmla   v11.4s, v7.4s, v2.s[1]      \n"

  4561. 

  4562.                         "shll   v8.4s, %27.4h, #16          \n"

  4563.                         "shll2  v9.4s, %27.8h, #16          \n"

  4564. 

  4565.                         "fmla   v12.4s, v8.4s, v2.s[2]      \n"

  4566.                         "fmla   v13.4s, v9.4s, v2.s[3]      \n"

  4567. 

  4568.                         "fadd   v11.4s, v10.4s, v11.4s      \n"

  4569. 

  4570.                         "add    %2, %2, #16                 \n"

  4571.                         "fadd   v13.4s, v12.4s, v13.4s      \n"

  4572. 

  4573.                         "add    %3, %3, #16                 \n"

  4574.                         "fadd   v13.4s, v11.4s, v13.4s      \n"

  4575. 

  4576.                         "add    %4, %4, #16                 \n"

  4577.                         "shrn   v13.4h, v13.4s, #16         \n"

  4578. 

  4579.                         "st1    {v13.4h}, [%0], #8          \n"

  4580. 

  4581.                         : "=r"(outptr0_bf16), // %0

  4582.                         "=r"(outptr0),      // %1

  4583.                         "=r"(r0),           // %2

  4584.                         "=r"(r1),           // %3

  4585.                         "=r"(r2)            // %4

  4586.                         : "0"(outptr0_bf16),

  4587.                         "1"(outptr0),

  4588.                         "2"(r0),

  4589.                         "3"(r1),

  4590.                         "4"(r2),

  4591.                         "w"(_k00_01), // %10

  4592.                         "w"(_k00_23), // %11

  4593.                         "w"(_k01_01), // %12

  4594.                         "w"(_k01_23), // %13

  4595.                         "w"(_k02_01), // %14

  4596.                         "w"(_k02_23), // %15

  4597.                         "w"(_k10_01), // %16

  4598.                         "w"(_k10_23), // %17

  4599.                         "w"(_k11_01), // %18

  4600.                         "w"(_k11_23), // %19

  4601.                         "w"(_k12_01), // %20

  4602.                         "w"(_k12_23), // %21

  4603.                         "w"(_k20_01), // %22

  4604.                         "w"(_k20_23), // %23

  4605.                         "w"(_k21_01), // %24

  4606.                         "w"(_k21_23), // %25

  4607.                         "w"(_k22_01), // %26

  4608.                         "w"(_k22_23)  // %27

  4609.                         : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");

  4610. #else  // __aarch64__

  4611.                     asm volatile(

  4612.                         "pld        [%1, #128]          \n"

  4613.                         "vld1.f32   {d30-d31}, [%1 :128]! \n" // sum0

  4614. 

  4615.                         "pld        [%2, #192]          \n"

  4616.                         "vld1.u16   {d2-d4}, [%2 :64]   \n" // r00 r01 r02

  4617. 

  4618.                         "vshll.u16  q0, d2, #16         \n"

  4619.                         "vshll.u16  q1, d3, #16         \n"

  4620.                         "vshll.u16  q2, d4, #16         \n"

  4621. 

  4622.                         "pld        [%5, #256]          \n"

  4623.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4624. 

  4625.                         "vshll.u16  q8, d20, #16        \n"

  4626.                         "vshll.u16  q9, d21, #16        \n"

  4627. 

  4628.                         "vmul.f32   q12, q8, d0[0]      \n"

  4629.                         "vmul.f32   q13, q9, d0[1]      \n"

  4630. 

  4631.                         "vshll.u16  q10, d22, #16       \n"

  4632.                         "vshll.u16  q11, d23, #16       \n"

  4633. 

  4634.                         "vmul.f32   q14, q10, d1[0]     \n"

  4635.                         "vmla.f32   q15, q11, d1[1]     \n"

  4636. 

  4637.                         "pld        [%5, #256]          \n"

  4638.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4639. 

  4640.                         "vshll.u16  q8, d20, #16        \n"

  4641.                         "vshll.u16  q9, d21, #16        \n"

  4642. 

  4643.                         "vmla.f32   q12, q8, d2[0]      \n"

  4644.                         "vmla.f32   q13, q9, d2[1]      \n"

  4645. 

  4646.                         "vshll.u16  q10, d22, #16       \n"

  4647.                         "vshll.u16  q11, d23, #16       \n"

  4648. 

  4649.                         "vmla.f32   q14, q10, d3[0]     \n"

  4650.                         "vmla.f32   q15, q11, d3[1]     \n"

  4651. 

  4652.                         "pld        [%5, #256]          \n"

  4653.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4654. 

  4655.                         "vshll.u16  q8, d20, #16        \n"

  4656.                         "vshll.u16  q9, d21, #16        \n"

  4657. 

  4658.                         "vmla.f32   q12, q8, d4[0]      \n"

  4659.                         "vmla.f32   q13, q9, d4[1]      \n"

  4660. 

  4661.                         "vshll.u16  q10, d22, #16       \n"

  4662.                         "vshll.u16  q11, d23, #16       \n"

  4663. 

  4664.                         "vmla.f32   q14, q10, d5[0]     \n"

  4665.                         "vmla.f32   q15, q11, d5[1]     \n"

  4666. 

  4667.                         "pld        [%3, #192]          \n"

  4668.                         "vld1.u16   {d2-d4}, [%3 :64]   \n" // r10 r11 r12

  4669. 

  4670.                         "vshll.u16  q0, d2, #16         \n"

  4671.                         "vshll.u16  q1, d3, #16         \n"

  4672.                         "vshll.u16  q2, d4, #16         \n"

  4673. 

  4674.                         "pld        [%5, #256]          \n"

  4675.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4676. 

  4677.                         "vshll.u16  q8, d20, #16        \n"

  4678.                         "vshll.u16  q9, d21, #16        \n"

  4679. 

  4680.                         "vmla.f32   q12, q8, d0[0]      \n"

  4681.                         "vmla.f32   q13, q9, d0[1]      \n"

  4682. 

  4683.                         "vshll.u16  q10, d22, #16       \n"

  4684.                         "vshll.u16  q11, d23, #16       \n"

  4685. 

  4686.                         "vmla.f32   q14, q10, d1[0]     \n"

  4687.                         "vmla.f32   q15, q11, d1[1]     \n"

  4688. 

  4689.                         "pld        [%5, #256]          \n"

  4690.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4691. 

  4692.                         "vshll.u16  q8, d20, #16        \n"

  4693.                         "vshll.u16  q9, d21, #16        \n"

  4694. 

  4695.                         "vmla.f32   q12, q8, d2[0]      \n"

  4696.                         "vmla.f32   q13, q9, d2[1]      \n"

  4697. 

  4698.                         "vshll.u16  q10, d22, #16       \n"

  4699.                         "vshll.u16  q11, d23, #16       \n"

  4700. 

  4701.                         "vmla.f32   q14, q10, d3[0]     \n"

  4702.                         "vmla.f32   q15, q11, d3[1]     \n"

  4703. 

  4704.                         "pld        [%5, #256]          \n"

  4705.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4706. 

  4707.                         "vshll.u16  q8, d20, #16        \n"

  4708.                         "vshll.u16  q9, d21, #16        \n"

  4709. 

  4710.                         "vmla.f32   q12, q8, d4[0]      \n"

  4711.                         "vmla.f32   q13, q9, d4[1]      \n"

  4712. 

  4713.                         "vshll.u16  q10, d22, #16       \n"

  4714.                         "vshll.u16  q11, d23, #16       \n"

  4715. 

  4716.                         "vmla.f32   q14, q10, d5[0]     \n"

  4717.                         "vmla.f32   q15, q11, d5[1]     \n"

  4718. 

  4719.                         "pld        [%4, #192]          \n"

  4720.                         "vld1.u16   {d2-d4}, [%4 :64]   \n" // r20 r21 r22

  4721. 

  4722.                         "vshll.u16  q0, d2, #16         \n"

  4723.                         "vshll.u16  q1, d3, #16         \n"

  4724.                         "vshll.u16  q2, d4, #16         \n"

  4725. 

  4726.                         "pld        [%5, #256]          \n"

  4727.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4728. 

  4729.                         "vshll.u16  q8, d20, #16        \n"

  4730.                         "vshll.u16  q9, d21, #16        \n"

  4731. 

  4732.                         "vmla.f32   q12, q8, d0[0]      \n"

  4733.                         "vmla.f32   q13, q9, d0[1]      \n"

  4734. 

  4735.                         "vshll.u16  q10, d22, #16       \n"

  4736.                         "vshll.u16  q11, d23, #16       \n"

  4737. 

  4738.                         "vmla.f32   q14, q10, d1[0]     \n"

  4739.                         "vmla.f32   q15, q11, d1[1]     \n"

  4740. 

  4741.                         "pld        [%5, #256]          \n"

  4742.                         "vld1.u16   {d20-d23}, [%5 :128]! \n"

  4743. 

  4744.                         "vshll.u16  q8, d20, #16        \n"

  4745.                         "vshll.u16  q9, d21, #16        \n"

  4746. 

  4747.                         "vmla.f32   q12, q8, d2[0]      \n"

  4748.                         "vmla.f32   q13, q9, d2[1]      \n"

  4749. 

  4750.                         "vshll.u16  q10, d22, #16       \n"

  4751.                         "vshll.u16  q11, d23, #16       \n"

  4752. 

  4753.                         "vmla.f32   q14, q10, d3[0]     \n"

  4754.                         "vmla.f32   q15, q11, d3[1]     \n"

  4755. 

  4756.                         //                         "pld        [%5, #256]          \n"

  4757.                         "vld1.u16   {d20-d23}, [%5 :128] \n"

  4758. 

  4759.                         "vshll.u16  q8, d20, #16        \n"

  4760.                         "vshll.u16  q9, d21, #16        \n"

  4761. 

  4762.                         "vmla.f32   q12, q8, d4[0]      \n"

  4763.                         "vmla.f32   q13, q9, d4[1]      \n"

  4764. 

  4765.                         "vshll.u16  q10, d22, #16       \n"

  4766.                         "vshll.u16  q11, d23, #16       \n"

  4767. 

  4768.                         "vmla.f32   q14, q10, d5[0]     \n"

  4769.                         "vmla.f32   q15, q11, d5[1]     \n"

  4770. 

  4771.                         "add        %2, %2, #16         \n"

  4772.                         "vadd.f32   q13, q12, q13       \n"

  4773. 

  4774.                         "add        %3, %3, #16         \n"

  4775.                         "vadd.f32   q15, q14, q15       \n"

  4776. 

  4777.                         "add        %4, %4, #16         \n"

  4778.                         "vadd.f32   q15, q13, q15       \n"

  4779. 

  4780.                         "sub        %5, %5, #256        \n" // kptr -= 8 * 16 * 2;

  4781. 

  4782.                         "vshrn.u32  d31, q15, #16       \n"

  4783. 

  4784.                         "vst1.u16   {d31}, [%0 :64]!    \n"

  4785. 

  4786.                         : "=r"(outptr0_bf16), // %0

  4787.                         "=r"(outptr0),      // %1

  4788.                         "=r"(r0),           // %2

  4789.                         "=r"(r1),           // %3

  4790.                         "=r"(r2),           // %4

  4791.                         "=r"(kptr)          // %5

  4792.                         : "0"(outptr0_bf16),

  4793.                         "1"(outptr0),

  4794.                         "2"(r0),

  4795.                         "3"(r1),

  4796.                         "4"(r2),

  4797.                         "5"(kptr)

  4798.                         : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  4799. #endif // __aarch64__

  4800.                 }

  4801. 

  4802.                 r0 += tailstep;

  4803.                 r1 += tailstep;

  4804.                 r2 += tailstep;

  4805.             }

  4806.         }

  4807.     }

  4808. }