nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: e9c890ad8a
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e9c890ad8a'
${ noResults }
ncnn/src/layer/arm/convolution_3x3.h

13445 lines
546 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  16. {

  17.     int w = bottom_blob.w;

  18.     int inch = bottom_blob.c;

  19. 

  20.     int outw = top_blob.w;

  21.     int outh = top_blob.h;

  22.     int outch = top_blob.c;

  23. 

  24.     const float* kernel = _kernel;

  25.     const float* bias = _bias;

  26. 

  27.     int nn_outch = outch >> 1;

  28.     int remain_outch_start = nn_outch << 1;

  29. 

  30.     #pragma omp parallel for num_threads(opt.num_threads)

  31.     for (int pp=0; pp<nn_outch; pp++)

  32.     {

  33.         int p = pp * 2;

  34. 

  35.         Mat out0 = top_blob.channel(p);

  36.         Mat out1 = top_blob.channel(p+1);

  37. 

  38.         const float bias0 = bias ? bias[p] : 0.f;

  39.         const float bias1 = bias ? bias[p+1] : 0.f;

  40. 

  41.         out0.fill(bias0);

  42.         out1.fill(bias1);

  43. 

  44.         const float* k0 = kernel + p*inch*9;

  45.         const float* k1 = kernel + (p+1)*inch*9;

  46. 

  47.         for (int q=0; q<inch; q++)

  48.         {

  49.             float* outptr0 = out0;

  50.             float* outptr1 = out1;

  51.             float* outptr0n = outptr0 + outw;

  52.             float* outptr1n = outptr1 + outw;

  53. 

  54.             const float* img0 = bottom_blob.channel(q);

  55. 

  56.             const float* r0 = img0;

  57.             const float* r1 = img0 + w;

  58.             const float* r2 = img0 + w*2;

  59.             const float* r3 = img0 + w*3;

  60. 

  61. #if __ARM_NEON

  62.             float32x4_t _k00 = vld1q_f32(k0);

  63.             float32x4_t _k03 = vld1q_f32(k0+3);

  64.             float32x4_t _k06 = vld1q_f32(k0+6);

  65. 

  66.             float32x4_t _k10 = vld1q_f32(k1);

  67.             float32x4_t _k13 = vld1q_f32(k1+3);

  68.             float32x4_t _k16 = vld1q_f32(k1+6);

  69. #endif // __ARM_NEON

  70. 

  71.             int i = 0;

  72. 

  73.             for (; i+1 < outh; i+=2)

  74.             {

  75. #if __ARM_NEON

  76.                 int nn = outw >> 2;

  77.                 int remain = outw & 3;

  78. #else

  79.                 int remain = outw;

  80. #endif // __ARM_NEON

  81. 

  82. #if __ARM_NEON

  83. #if __aarch64__

  84.                 if (nn > 0)

  85.                 {

  86.                 asm volatile(

  87.                     "prfm   pldl1keep, [%5, #256]       \n"

  88.                     "ld1    {v8.4s, v9.4s}, [%5]        \n"// r0

  89.                     "add    %5, %5, #16                 \n"

  90. 

  91.                     "prfm   pldl1keep, [%8, #256]       \n"

  92.                     "ld1    {v14.4s, v15.4s}, [%8]      \n"// r3

  93.                     "add    %8, %8, #16                 \n"

  94. 

  95.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  96.                     "ext    v11.16b, v14.16b, v15.16b, #8 \n"

  97. 

  98.                     "0:                                 \n"

  99. 

  100.                     "prfm   pldl1keep, [%1, #128]       \n"

  101.                     "ld1    {v6.4s}, [%1]               \n"// _sum0

  102. 

  103.                     "prfm   pldl1keep, [%2, #128]       \n"

  104.                     "ld1    {v7.4s}, [%2]               \n"// _sum1

  105. 

  106.                     "fmla   v6.4s, v8.4s, %18.s[0]      \n"

  107.                     "fmla   v7.4s, v8.4s, %21.s[0]      \n"

  108. 

  109.                     "prfm   pldl1keep, [%3, #128]       \n"

  110.                     "ld1    {v12.4s}, [%3]              \n"// _sum0n

  111. 

  112.                     "prfm   pldl1keep, [%4, #128]       \n"

  113.                     "ld1    {v13.4s}, [%4]              \n"// _sum1n

  114. 

  115.                     "fmla   v12.4s, v14.4s, %20.s[0]    \n"

  116.                     "fmla   v13.4s, v14.4s, %23.s[0]    \n"

  117. 

  118.                     "ext    v8.16b, v8.16b, v9.16b, #8  \n"

  119.                     "ext    v9.16b, v14.16b, v15.16b, #4 \n"

  120. 

  121.                     "fmla   v6.4s, v10.4s, %18.s[1]     \n"

  122.                     "fmla   v7.4s, v10.4s, %21.s[1]     \n"

  123.                     "fmla   v12.4s, v11.4s, %20.s[2]    \n"

  124.                     "fmla   v13.4s, v11.4s, %23.s[2]    \n"

  125. 

  126.                     "prfm   pldl1keep, [%6, #256]       \n"

  127.                     "ld1    {v14.4s, v15.4s}, [%6]      \n"// r1

  128.                     "add    %6, %6, #16                 \n"

  129. 

  130.                     "fmla   v6.4s, v8.4s, %18.s[2]      \n"

  131.                     "fmla   v7.4s, v8.4s, %21.s[2]      \n"

  132.                     "fmla   v12.4s, v9.4s, %20.s[1]     \n"

  133.                     "fmla   v13.4s, v9.4s, %23.s[1]     \n"

  134. 

  135.                     "ext    v10.16b, v14.16b, v15.16b, #4 \n"

  136. 

  137.                     "fmla   v6.4s, v14.4s, %19.s[0]     \n"

  138.                     "fmla   v7.4s, v14.4s, %22.s[0]     \n"

  139.                     "fmla   v12.4s, v14.4s, %18.s[0]    \n"

  140.                     "fmla   v13.4s, v14.4s, %21.s[0]    \n"

  141. 

  142.                     "ext    v11.16b, v14.16b, v15.16b, #8 \n"

  143. 

  144.                     "fmla   v6.4s, v10.4s, %19.s[1]     \n"

  145.                     "fmla   v7.4s, v10.4s, %22.s[1]     \n"

  146.                     "fmla   v12.4s, v10.4s, %18.s[1]    \n"

  147.                     "fmla   v13.4s, v10.4s, %21.s[1]    \n"

  148. 

  149.                     "prfm   pldl1keep, [%7, #256]       \n"

  150.                     "ld1    {v8.4s, v9.4s}, [%7]        \n"// r2

  151.                     "add    %7, %7, #16                 \n"

  152. 

  153.                     "fmla   v6.4s, v11.4s, %19.s[2]     \n"

  154.                     "fmla   v7.4s, v11.4s, %22.s[2]     \n"

  155.                     "fmla   v12.4s, v11.4s, %18.s[2]    \n"

  156.                     "fmla   v13.4s, v11.4s, %21.s[2]    \n"

  157. 

  158.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  159. 

  160.                     "fmla   v6.4s, v8.4s, %20.s[0]      \n"

  161.                     "fmla   v7.4s, v8.4s, %23.s[0]      \n"

  162.                     "fmla   v12.4s, v8.4s, %19.s[0]     \n"

  163.                     "fmla   v13.4s, v8.4s, %22.s[0]     \n"

  164. 

  165.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  166. 

  167.                     "fmla   v6.4s, v10.4s, %20.s[1]     \n"

  168.                     "fmla   v7.4s, v10.4s, %23.s[1]     \n"

  169.                     "fmla   v12.4s, v10.4s, %19.s[1]    \n"

  170.                     "fmla   v13.4s, v10.4s, %22.s[1]    \n"

  171. 

  172.                     "prfm   pldl1keep, [%5, #256]       \n"

  173.                     "ld1    {v8.4s, v9.4s}, [%5]        \n"// r0

  174.                     "add    %5, %5, #16                 \n"

  175. 

  176.                     "fmla   v6.4s, v11.4s, %20.s[2]     \n"

  177.                     "fmla   v7.4s, v11.4s, %23.s[2]     \n"

  178.                     "fmla   v12.4s, v11.4s, %19.s[2]    \n"

  179.                     "fmla   v13.4s, v11.4s, %22.s[2]    \n"

  180. 

  181.                     "prfm   pldl1keep, [%8, #256]       \n"

  182.                     "ld1    {v14.4s, v15.4s}, [%8]      \n"// r3

  183.                     "add    %8, %8, #16                 \n"

  184. 

  185.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  186. 

  187.                     "st1    {v6.4s}, [%1], #16          \n"

  188.                     "st1    {v7.4s}, [%2], #16          \n"

  189. 

  190.                     "ext    v11.16b, v14.16b, v15.16b, #8 \n"

  191. 

  192.                     "st1    {v12.4s}, [%3], #16         \n"

  193.                     "st1    {v13.4s}, [%4], #16         \n"

  194. 

  195.                     "subs   %w0, %w0, #1                \n"

  196.                     "bne    0b                          \n"

  197. 

  198.                     "sub    %5, %5, #16                 \n"

  199.                     "sub    %8, %8, #16                 \n"

  200.                     : "=r"(nn),         // %0

  201.                       "=r"(outptr0),    // %1

  202.                       "=r"(outptr1),    // %2

  203.                       "=r"(outptr0n),   // %3

  204.                       "=r"(outptr1n),   // %4

  205.                       "=r"(r0),         // %5

  206.                       "=r"(r1),         // %6

  207.                       "=r"(r2),         // %7

  208.                       "=r"(r3)          // %8

  209.                     : "0"(nn),

  210.                       "1"(outptr0),

  211.                       "2"(outptr1),

  212.                       "3"(outptr0n),

  213.                       "4"(outptr1n),

  214.                       "5"(r0),

  215.                       "6"(r1),

  216.                       "7"(r2),

  217.                       "8"(r3),

  218.                       "w"(_k00),      // %18

  219.                       "w"(_k03),      // %19

  220.                       "w"(_k06),      // %20

  221.                       "w"(_k10),      // %21

  222.                       "w"(_k13),      // %22

  223.                       "w"(_k16)       // %23

  224.                     : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  225.                 );

  226.                 }

  227. #else

  228.                 if (nn > 0)

  229.                 {

  230.                 asm volatile(

  231. 

  232.                     "pld        [%5, #192]          \n"

  233.                     "vld1.f32   {d16-d18}, [%5 :64] \n"// r0

  234.                     "add        %5, #16             \n"

  235. 

  236.                     "pld        [%8, #192]          \n"

  237.                     "vld1.f32   {d28-d30}, [%8]     \n"// r3

  238.                     "add        %8, #16             \n"

  239. 

  240.                     "vext.32    q10, q8, q9, #1     \n"

  241.                     "vext.32    q11, q14, q15, #2   \n"

  242. 

  243.                     "0:                             \n"

  244. 

  245.                     "pld        [%1, #128]          \n"

  246.                     "vld1.f32   {d12-d13}, [%1 :64] \n"// _sum0

  247. 

  248.                     "pld        [%2, #128]          \n"

  249.                     "vld1.f32   {d14-d15}, [%2 :64] \n"// _sum1

  250. 

  251.                     "vmla.f32   q6, q8, %e18[0]     \n"

  252.                     "vmla.f32   q7, q8, %e21[0]     \n"

  253. 

  254.                     "pld        [%3, #128]          \n"

  255.                     "vld1.f32   {d24-d25}, [%3]     \n"// _sum0n

  256. 

  257.                     "pld        [%4, #128]          \n"

  258.                     "vld1.f32   {d26-d27}, [%4]     \n"// _sum1n

  259. 

  260.                     "vmla.f32   q12, q14, %e20[0]   \n"

  261.                     "vmla.f32   q13, q14, %e23[0]   \n"

  262. 

  263.                     "vext.32    q8, q8, q9, #2      \n"

  264.                     "vext.32    q9, q14, q15, #1    \n"

  265. 

  266.                     "vmla.f32   q6, q10, %e18[1]    \n"

  267.                     "vmla.f32   q7, q10, %e21[1]    \n"

  268.                     "vmla.f32   q12, q11, %f20[0]   \n"

  269.                     "vmla.f32   q13, q11, %f23[0]   \n"

  270. 

  271.                     "pld        [%6, #192]          \n"

  272.                     "vld1.f32   {d28-d30}, [%6]     \n"// r1

  273.                     "add        %6, #16             \n"

  274. 

  275.                     "vmla.f32   q6, q8, %f18[0]     \n"

  276.                     "vmla.f32   q7, q8, %f21[0]     \n"

  277.                     "vmla.f32   q12, q9, %e20[1]    \n"

  278.                     "vmla.f32   q13, q9, %e23[1]    \n"

  279. 

  280.                     "vext.32    q10, q14, q15, #1   \n"

  281. 

  282.                     "vmla.f32   q6, q14, %e19[0]    \n"

  283.                     "vmla.f32   q7, q14, %e22[0]    \n"

  284.                     "vmla.f32   q12, q14, %e18[0]   \n"

  285.                     "vmla.f32   q13, q14, %e21[0]   \n"

  286. 

  287.                     "vext.32    q11, q14, q15, #2   \n"

  288. 

  289.                     "vmla.f32   q6, q10, %e19[1]    \n"

  290.                     "vmla.f32   q7, q10, %e22[1]    \n"

  291.                     "vmla.f32   q12, q10, %e18[1]   \n"

  292.                     "vmla.f32   q13, q10, %e21[1]   \n"

  293. 

  294.                     "pld        [%7, #192]          \n"

  295.                     "vld1.f32   {d16-d18}, [%7 :64] \n"// r2

  296.                     "add        %7, #16             \n"

  297. 

  298.                     "vmla.f32   q6, q11, %f19[0]    \n"

  299.                     "vmla.f32   q7, q11, %f22[0]    \n"

  300.                     "vmla.f32   q12, q11, %f18[0]   \n"

  301.                     "vmla.f32   q13, q11, %f21[0]   \n"

  302. 

  303.                     "vext.32    q10, q8, q9, #1     \n"

  304. 

  305.                     "vmla.f32   q6, q8, %e20[0]     \n"

  306.                     "vmla.f32   q7, q8, %e23[0]     \n"

  307.                     "vmla.f32   q12, q8, %e19[0]    \n"

  308.                     "vmla.f32   q13, q8, %e22[0]    \n"

  309. 

  310.                     "vext.32    q11, q8, q9, #2     \n"

  311. 

  312.                     "vmla.f32   q6, q10, %e20[1]    \n"

  313.                     "vmla.f32   q7, q10, %e23[1]    \n"

  314.                     "vmla.f32   q12, q10, %e19[1]   \n"

  315.                     "vmla.f32   q13, q10, %e22[1]   \n"

  316. 

  317.                     "pld        [%5, #192]          \n"

  318.                     "vld1.f32   {d16-d18}, [%5 :64] \n"// r0

  319.                     "add        %5, #16             \n"

  320. 

  321.                     "vmla.f32   q6, q11, %f20[0]    \n"

  322.                     "vmla.f32   q7, q11, %f23[0]    \n"

  323.                     "vmla.f32   q12, q11, %f19[0]   \n"

  324.                     "vmla.f32   q13, q11, %f22[0]   \n"

  325. 

  326.                     "pld        [%8, #192]          \n"

  327.                     "vld1.f32   {d28-d30}, [%8]     \n"// r3

  328.                     "add        %8, #16             \n"

  329. 

  330.                     "vext.32    q10, q8, q9, #1     \n"

  331. 

  332.                     "vst1.f32   {d12-d13}, [%1 : 64]!\n"

  333.                     "vst1.f32   {d14-d15}, [%2 : 64]!\n"

  334. 

  335.                     "vext.32    q11, q14, q15, #2   \n"

  336. 

  337.                     "vst1.f32   {d24-d25}, [%3]!    \n"

  338.                     "vst1.f32   {d26-d27}, [%4]!    \n"

  339. 

  340.                     "subs       %0, #1              \n"

  341.                     "bne        0b                  \n"

  342. 

  343.                     "sub        %5, #16             \n"

  344.                     "sub        %8, #16             \n"

  345.                     : "=r"(nn),         // %0

  346.                       "=r"(outptr0),    // %1

  347.                       "=r"(outptr1),    // %2

  348.                       "=r"(outptr0n),   // %3

  349.                       "=r"(outptr1n),   // %4

  350.                       "=r"(r0),         // %5

  351.                       "=r"(r1),         // %6

  352.                       "=r"(r2),         // %7

  353.                       "=r"(r3)          // %8

  354.                     : "0"(nn),

  355.                       "1"(outptr0),

  356.                       "2"(outptr1),

  357.                       "3"(outptr0n),

  358.                       "4"(outptr1n),

  359.                       "5"(r0),

  360.                       "6"(r1),

  361.                       "7"(r2),

  362.                       "8"(r3),

  363.                       "w"(_k00),      // %18

  364.                       "w"(_k03),      // %19

  365.                       "w"(_k06),      // %20

  366.                       "w"(_k10),      // %21

  367.                       "w"(_k13),      // %22

  368.                       "w"(_k16)       // %23

  369.                     : "cc", "memory", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  370.                 );

  371.                 }

  372. #endif // __aarch64__

  373. #endif // __ARM_NEON

  374.                 for (; remain>0; remain--)

  375.                 {

  376. #if __ARM_NEON

  377.                     float32x4_t _r00 = vld1q_f32(r0);

  378.                     float32x4_t _r10 = vld1q_f32(r1);

  379.                     float32x4_t _r20 = vld1q_f32(r2);

  380.                     float32x4_t _r30 = vld1q_f32(r3);

  381. 

  382.                     float32x4_t _sum0 = vmulq_f32(_r00, _k00);

  383.                     float32x4_t _sum1 = vmulq_f32(_r00, _k10);

  384.                     _sum0 = vmlaq_f32(_sum0, _r10, _k03);

  385.                     _sum1 = vmlaq_f32(_sum1, _r10, _k13);

  386.                     _sum0 = vmlaq_f32(_sum0, _r20, _k06);

  387.                     _sum1 = vmlaq_f32(_sum1, _r20, _k16);

  388. 

  389.                     float32x4_t _sum0n = vmulq_f32(_r10, _k00);

  390.                     float32x4_t _sum1n = vmulq_f32(_r10, _k10);

  391.                     _sum0n = vmlaq_f32(_sum0n, _r20, _k03);

  392.                     _sum1n = vmlaq_f32(_sum1n, _r20, _k13);

  393.                     _sum0n = vmlaq_f32(_sum0n, _r30, _k06);

  394.                     _sum1n = vmlaq_f32(_sum1n, _r30, _k16);

  395. 

  396.                     _sum0 = vsetq_lane_f32(*outptr0, _sum0, 3);

  397.                     _sum1 = vsetq_lane_f32(*outptr1, _sum1, 3);

  398.                     _sum0n = vsetq_lane_f32(*outptr0n, _sum0n, 3);

  399.                     _sum1n = vsetq_lane_f32(*outptr1n, _sum1n, 3);

  400. #if __aarch64__

  401.                     *outptr0 = vaddvq_f32(_sum0);

  402.                     *outptr1 = vaddvq_f32(_sum1);

  403.                     *outptr0n = vaddvq_f32(_sum0n);

  404.                     *outptr1n = vaddvq_f32(_sum1n);

  405. #else

  406.                     float32x2_t _ss0 = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));

  407.                     float32x2_t _ss1 = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1));

  408.                     float32x2_t _ss0n = vadd_f32(vget_low_f32(_sum0n), vget_high_f32(_sum0n));

  409.                     float32x2_t _ss1n = vadd_f32(vget_low_f32(_sum1n), vget_high_f32(_sum1n));

  410. 

  411.                     float32x2_t _ss01 = vpadd_f32(_ss0, _ss1);

  412.                     float32x2_t _ss01n = vpadd_f32(_ss0n, _ss1n);

  413. 

  414.                     *outptr0 = vget_lane_f32(_ss01, 0);

  415.                     *outptr1 = vget_lane_f32(_ss01, 1);

  416.                     *outptr0n = vget_lane_f32(_ss01n, 0);

  417.                     *outptr1n = vget_lane_f32(_ss01n, 1);

  418. #endif // __aarch64__

  419. #else

  420.                     float sum0 = 0.f;

  421.                     float sum0n = 0.f;

  422.                     float sum1 = 0.f;

  423.                     float sum1n = 0.f;

  424. 

  425.                     sum0 += r0[0] * k0[0];

  426.                     sum0 += r0[1] * k0[1];

  427.                     sum0 += r0[2] * k0[2];

  428.                     sum0 += r1[0] * k0[3];

  429.                     sum0 += r1[1] * k0[4];

  430.                     sum0 += r1[2] * k0[5];

  431.                     sum0 += r2[0] * k0[6];

  432.                     sum0 += r2[1] * k0[7];

  433.                     sum0 += r2[2] * k0[8];

  434. 

  435.                     sum1 += r0[0] * k1[0];

  436.                     sum1 += r0[1] * k1[1];

  437.                     sum1 += r0[2] * k1[2];

  438.                     sum1 += r1[0] * k1[3];

  439.                     sum1 += r1[1] * k1[4];

  440.                     sum1 += r1[2] * k1[5];

  441.                     sum1 += r2[0] * k1[6];

  442.                     sum1 += r2[1] * k1[7];

  443.                     sum1 += r2[2] * k1[8];

  444. 

  445.                     sum0n += r1[0] * k0[0];

  446.                     sum0n += r1[1] * k0[1];

  447.                     sum0n += r1[2] * k0[2];

  448.                     sum0n += r2[0] * k0[3];

  449.                     sum0n += r2[1] * k0[4];

  450.                     sum0n += r2[2] * k0[5];

  451.                     sum0n += r3[0] * k0[6];

  452.                     sum0n += r3[1] * k0[7];

  453.                     sum0n += r3[2] * k0[8];

  454. 

  455.                     sum1n += r1[0] * k1[0];

  456.                     sum1n += r1[1] * k1[1];

  457.                     sum1n += r1[2] * k1[2];

  458.                     sum1n += r2[0] * k1[3];

  459.                     sum1n += r2[1] * k1[4];

  460.                     sum1n += r2[2] * k1[5];

  461.                     sum1n += r3[0] * k1[6];

  462.                     sum1n += r3[1] * k1[7];

  463.                     sum1n += r3[2] * k1[8];

  464. 

  465.                     *outptr0 += sum0;

  466.                     *outptr1 += sum1;

  467.                     *outptr0n += sum0n;

  468.                     *outptr1n += sum1n;

  469. #endif // __ARM_NEON

  470.                     r0++;

  471.                     r1++;

  472.                     r2++;

  473.                     r3++;

  474.                     outptr0++;

  475.                     outptr1++;

  476.                     outptr0n++;

  477.                     outptr1n++;

  478.                 }

  479. 

  480.                 r0 += 2 + w;

  481.                 r1 += 2 + w;

  482.                 r2 += 2 + w;

  483.                 r3 += 2 + w;

  484. 

  485.                 outptr0 += outw;

  486.                 outptr1 += outw;

  487.                 outptr0n += outw;

  488.                 outptr1n += outw;

  489.             }

  490. 

  491.             for (; i < outh; i++)

  492.             {

  493. #if __ARM_NEON

  494.                 int nn = outw >> 2;

  495.                 int remain = outw & 3;

  496. #else

  497.                 int remain = outw;

  498. #endif // __ARM_NEON

  499. 

  500. #if __ARM_NEON

  501. #if __aarch64__

  502.                 if (nn > 0)

  503.                 {

  504.                 asm volatile(

  505.                     "0:                                 \n"

  506. 

  507.                     "prfm   pldl1keep, [%3, #256]       \n"

  508.                     "ld1    {v8.4s, v9.4s}, [%3]        \n"// r0

  509.                     "add    %3, %3, #16                 \n"

  510. 

  511.                     "prfm   pldl1keep, [%1, #128]       \n"

  512.                     "ld1    {v6.4s}, [%1]               \n"// _sum0

  513. 

  514.                     "prfm   pldl1keep, [%2, #128]       \n"

  515.                     "ld1    {v7.4s}, [%2]               \n"// _sum1

  516. 

  517.                     "fmul   v14.4s, v8.4s, %12.s[0]     \n"

  518.                     "fmul   v15.4s, v8.4s, %15.s[0]     \n"

  519. 

  520.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  521.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  522. 

  523.                     "fmla   v6.4s, v10.4s, %12.s[1]     \n"

  524.                     "fmla   v7.4s, v10.4s, %15.s[1]     \n"

  525. 

  526.                     "prfm   pldl1keep, [%4, #256]       \n"

  527.                     "ld1    {v8.4s, v9.4s}, [%4]        \n"// r1

  528.                     "add    %4, %4, #16                 \n"

  529. 

  530.                     "fmla   v14.4s, v11.4s, %12.s[2]    \n"

  531.                     "fmla   v15.4s, v11.4s, %15.s[2]    \n"

  532. 

  533.                     "fmla   v6.4s, v8.4s, %13.s[0]      \n"

  534.                     "fmla   v7.4s, v8.4s, %16.s[0]      \n"

  535. 

  536.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  537.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  538. 

  539.                     "fmla   v14.4s, v10.4s, %13.s[1]    \n"

  540.                     "fmla   v15.4s, v10.4s, %16.s[1]    \n"

  541. 

  542.                     "prfm   pldl1keep, [%5, #256]       \n"

  543.                     "ld1    {v8.4s, v9.4s}, [%5]        \n"// r2

  544.                     "add    %5, %5, #16                 \n"

  545. 

  546.                     "fmla   v6.4s, v11.4s, %13.s[2]     \n"

  547.                     "fmla   v7.4s, v11.4s, %16.s[2]     \n"

  548. 

  549.                     "fmla   v14.4s, v8.4s, %14.s[0]     \n"

  550.                     "fmla   v15.4s, v8.4s, %17.s[0]     \n"

  551. 

  552.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  553.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  554. 

  555.                     "fmla   v6.4s, v10.4s, %14.s[1]     \n"

  556.                     "fmla   v7.4s, v10.4s, %17.s[1]     \n"

  557. 

  558.                     "fmla   v14.4s, v11.4s, %14.s[2]    \n"

  559.                     "fmla   v15.4s, v11.4s, %17.s[2]    \n"

  560. 

  561.                     "fadd   v6.4s, v6.4s, v14.4s        \n"

  562.                     "fadd   v7.4s, v7.4s, v15.4s        \n"

  563. 

  564.                     "st1    {v6.4s}, [%1], #16          \n"

  565.                     "st1    {v7.4s}, [%2], #16          \n"

  566. 

  567.                     "subs   %w0, %w0, #1                \n"

  568.                     "bne    0b                          \n"

  569. 

  570.                     : "=r"(nn),         // %0

  571.                       "=r"(outptr0),    // %1

  572.                       "=r"(outptr1),    // %2

  573.                       "=r"(r0),         // %3

  574.                       "=r"(r1),         // %4

  575.                       "=r"(r2)          // %5

  576.                     : "0"(nn),

  577.                       "1"(outptr0),

  578.                       "2"(outptr1),

  579.                       "3"(r0),

  580.                       "4"(r1),

  581.                       "5"(r2),

  582.                       "w"(_k00),      // %12

  583.                       "w"(_k03),      // %13

  584.                       "w"(_k06),      // %14

  585.                       "w"(_k10),      // %15

  586.                       "w"(_k13),      // %16

  587.                       "w"(_k16)       // %17

  588.                     : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  589.                 );

  590.                 }

  591. #else

  592.                 if (nn > 0)

  593.                 {

  594.                 asm volatile(

  595.                     "0:                             \n"

  596. 

  597.                     "pld        [%3, #192]          \n"

  598.                     "vld1.f32   {d16-d18}, [%3]     \n"// r0

  599.                     "add        %3, #16             \n"

  600. 

  601.                     "pld        [%1, #128]          \n"

  602.                     "vld1.f32   {d12-d13}, [%1]     \n"// _sum0

  603. 

  604.                     "pld        [%2, #128]          \n"

  605.                     "vld1.f32   {d14-d15}, [%2]     \n"// _sum1

  606. 

  607.                     "vmul.f32   q14, q8, %e12[0]    \n"

  608.                     "vmul.f32   q15, q8, %e15[0]    \n"

  609. 

  610.                     "vext.32    q10, q8, q9, #1     \n"

  611.                     "vext.32    q11, q8, q9, #2     \n"

  612. 

  613.                     "vmla.f32   q6, q10, %e12[1]    \n"

  614.                     "vmla.f32   q7, q10, %e15[1]    \n"

  615. 

  616.                     "pld        [%4, #192]          \n"

  617.                     "vld1.f32   {d16-d18}, [%4]     \n"// r1

  618.                     "add        %4, #16             \n"

  619. 

  620.                     "vmla.f32   q14, q11, %f12[0]   \n"

  621.                     "vmla.f32   q15, q11, %f15[0]   \n"

  622. 

  623.                     "vmla.f32   q6, q8, %e13[0]     \n"

  624.                     "vmla.f32   q7, q8, %e16[0]     \n"

  625. 

  626.                     "vext.32    q10, q8, q9, #1     \n"

  627.                     "vext.32    q11, q8, q9, #2     \n"

  628. 

  629.                     "vmla.f32   q14, q10, %e13[1]   \n"

  630.                     "vmla.f32   q15, q10, %e16[1]   \n"

  631. 

  632.                     "pld        [%5, #192]          \n"

  633.                     "vld1.f32   {d16-d18}, [%5]     \n"// r2

  634.                     "add        %5, #16             \n"

  635. 

  636.                     "vmla.f32   q6, q11, %f13[0]    \n"

  637.                     "vmla.f32   q7, q11, %f16[0]    \n"

  638. 

  639.                     "vmla.f32   q14, q8, %e14[0]    \n"

  640.                     "vmla.f32   q15, q8, %e17[0]    \n"

  641. 

  642.                     "vext.32    q10, q8, q9, #1     \n"

  643.                     "vext.32    q11, q8, q9, #2     \n"

  644. 

  645.                     "vmla.f32   q6, q10, %e14[1]    \n"

  646.                     "vmla.f32   q7, q10, %e17[1]    \n"

  647. 

  648.                     "vmla.f32   q14, q11, %f14[0]   \n"

  649.                     "vmla.f32   q15, q11, %f17[0]   \n"

  650. 

  651.                     "vadd.f32   q6, q6, q14         \n"

  652.                     "vadd.f32   q7, q7, q15         \n"

  653. 

  654.                     "vst1.f32   {d12-d13}, [%1]!    \n"

  655. 

  656.                     "vst1.f32   {d14-d15}, [%2]!    \n"

  657. 

  658.                     "subs       %0, #1              \n"

  659.                     "bne        0b                  \n"

  660. 

  661.                     : "=r"(nn),         // %0

  662.                       "=r"(outptr0),    // %1

  663.                       "=r"(outptr1),    // %2

  664.                       "=r"(r0),         // %3

  665.                       "=r"(r1),         // %4

  666.                       "=r"(r2)          // %5

  667.                     : "0"(nn),

  668.                       "1"(outptr0),

  669.                       "2"(outptr1),

  670.                       "3"(r0),

  671.                       "4"(r1),

  672.                       "5"(r2),

  673.                       "w"(_k00),      // %12

  674.                       "w"(_k03),      // %13

  675.                       "w"(_k06),      // %14

  676.                       "w"(_k10),      // %15

  677.                       "w"(_k13),      // %16

  678.                       "w"(_k16)       // %17

  679.                     : "cc", "memory", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  680.                 );

  681.                 }

  682. #endif // __aarch64__

  683. #endif // __ARM_NEON

  684.                 for (; remain>0; remain--)

  685.                 {

  686. #if __ARM_NEON

  687.                     float32x4_t _r00 = vld1q_f32(r0);

  688.                     float32x4_t _r10 = vld1q_f32(r1);

  689.                     float32x4_t _r20 = vld1q_f32(r2);

  690. 

  691.                     float32x4_t _sum0 = vmulq_f32(_r00, _k00);

  692.                     float32x4_t _sum1 = vmulq_f32(_r00, _k10);

  693.                     _sum0 = vmlaq_f32(_sum0, _r10, _k03);

  694.                     _sum1 = vmlaq_f32(_sum1, _r10, _k13);

  695.                     _sum0 = vmlaq_f32(_sum0, _r20, _k06);

  696.                     _sum1 = vmlaq_f32(_sum1, _r20, _k16);

  697. 

  698.                     _sum0 = vsetq_lane_f32(*outptr0, _sum0, 3);

  699.                     _sum1 = vsetq_lane_f32(*outptr1, _sum1, 3);

  700. #if __aarch64__

  701.                     *outptr0 = vaddvq_f32(_sum0);

  702.                     *outptr1 = vaddvq_f32(_sum1);

  703. #else

  704.                     float32x2_t _ss0 = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));

  705.                     float32x2_t _ss1 = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1));

  706.                     float32x2_t _ss01 = vpadd_f32(_ss0, _ss1);

  707. 

  708.                     *outptr0 = vget_lane_f32(_ss01, 0);

  709.                     *outptr1 = vget_lane_f32(_ss01, 1);

  710. #endif // __aarch64__

  711. #else

  712.                     float sum0 = 0.f;

  713.                     float sum1 = 0.f;

  714. 

  715.                     sum0 += r0[0] * k0[0];

  716.                     sum0 += r0[1] * k0[1];

  717.                     sum0 += r0[2] * k0[2];

  718.                     sum0 += r1[0] * k0[3];

  719.                     sum0 += r1[1] * k0[4];

  720.                     sum0 += r1[2] * k0[5];

  721.                     sum0 += r2[0] * k0[6];

  722.                     sum0 += r2[1] * k0[7];

  723.                     sum0 += r2[2] * k0[8];

  724. 

  725.                     sum1 += r0[0] * k1[0];

  726.                     sum1 += r0[1] * k1[1];

  727.                     sum1 += r0[2] * k1[2];

  728.                     sum1 += r1[0] * k1[3];

  729.                     sum1 += r1[1] * k1[4];

  730.                     sum1 += r1[2] * k1[5];

  731.                     sum1 += r2[0] * k1[6];

  732.                     sum1 += r2[1] * k1[7];

  733.                     sum1 += r2[2] * k1[8];

  734. 

  735.                     *outptr0 += sum0;

  736.                     *outptr1 += sum1;

  737. #endif // __ARM_NEON

  738.                     r0++;

  739.                     r1++;

  740.                     r2++;

  741.                     outptr0++;

  742.                     outptr1++;

  743.                 }

  744. 

  745.                 r0 += 2;

  746.                 r1 += 2;

  747.                 r2 += 2;

  748.             }

  749. 

  750.             k0 += 9;

  751.             k1 += 9;

  752.         }

  753.     }

  754. 

  755.     #pragma omp parallel for num_threads(opt.num_threads)

  756.     for (int p=remain_outch_start; p<outch; p++)

  757.     {

  758.         Mat out = top_blob.channel(p);

  759. 

  760.         const float bias0 = bias ? bias[p] : 0.f;

  761. 

  762.         out.fill(bias0);

  763. 

  764.         const float* kernel0 = kernel + p*inch*9;

  765. 

  766.         for (int q=0; q<inch; q++)

  767.         {

  768.             float* outptr = out;

  769.             float* outptr2 = outptr + outw;

  770. 

  771.             const float* img0 = bottom_blob.channel(q);

  772. 

  773.             const float* r0 = img0;

  774.             const float* r1 = img0 + w;

  775.             const float* r2 = img0 + w*2;

  776.             const float* r3 = img0 + w*3;

  777. 

  778. #if __ARM_NEON

  779.             float32x4_t _k0123 = vld1q_f32(kernel0);

  780.             float32x4_t _k3456 = vld1q_f32(kernel0+3);

  781.             float32x4_t _k6789 = vld1q_f32(kernel0+6);

  782. #else

  783.             const float* k0 = kernel0;

  784.             const float* k1 = kernel0 + 3;

  785.             const float* k2 = kernel0 + 6;

  786. #endif // __ARM_NEON

  787. 

  788.             int i = 0;

  789. 

  790.             for (; i+1 < outh; i+=2)

  791.             {

  792. 

  793. #if __ARM_NEON

  794.                 int nn = outw >> 2;

  795.                 int remain = outw & 3;

  796. #else

  797.                 int remain = outw;

  798. #endif // __ARM_NEON

  799. 

  800. #if __ARM_NEON

  801. #if __aarch64__

  802.                 if (nn > 0)

  803.                 {

  804.                 asm volatile(

  805.                     "prfm   pldl1keep, [%3, #256]       \n"

  806.                     "ld1    {v9.4s, v10.4s}, [%3]       \n"// r0

  807.                     "add    %3, %3, #16                 \n"

  808. 

  809.                     "ext    v11.16b, v9.16b, v10.16b, #4 \n"

  810.                     "ext    v12.16b, v9.16b, v10.16b, #8 \n"

  811. 

  812.                     "0:                                 \n"

  813. 

  814.                     "prfm   pldl1keep, [%1, #128]       \n"

  815.                     "ld1    {v7.4s}, [%1]               \n"// _sum

  816. 

  817.                     "fmla   v7.4s, v9.4s, %14.s[0]      \n"

  818.                     "fmul   v6.4s, v11.4s, %14.s[1]     \n"

  819.                     "fmul   v13.4s, v12.4s, %14.s[2]    \n"

  820. 

  821.                     "prfm   pldl1keep, [%4, #256]       \n"

  822.                     "ld1    {v9.4s, v10.4s}, [%4]       \n"// r1

  823.                     "add    %4, %4, #16                 \n"

  824. 

  825.                     "fmla   v7.4s, v9.4s, %15.s[0]      \n"

  826. 

  827.                     "ext    v11.16b, v9.16b, v10.16b, #4 \n"

  828.                     "ext    v12.16b, v9.16b, v10.16b, #8 \n"

  829. 

  830.                     "fmla   v6.4s, v11.4s, %15.s[1]     \n"

  831.                     "fmla   v13.4s, v12.4s, %15.s[2]    \n"

  832. 

  833.                     "prfm   pldl1keep, [%2, #128]       \n"

  834.                     "ld1    {v8.4s}, [%2]               \n"// _sum2

  835. 

  836.                     "fmla   v8.4s, v9.4s, %14.s[0]      \n"

  837.                     "fmul   v14.4s, v11.4s, %14.s[1]    \n"

  838.                     "fmul   v15.4s, v12.4s, %14.s[2]    \n"

  839. 

  840.                     "prfm   pldl1keep, [%5, #256]       \n"

  841.                     "ld1    {v9.4s, v10.4s}, [%5]       \n"// r2

  842.                     "add    %5, %5, #16                 \n"

  843. 

  844.                     "fmla   v7.4s, v9.4s, %16.s[0]      \n"

  845. 

  846.                     "ext    v11.16b, v9.16b, v10.16b, #4 \n"

  847.                     "ext    v12.16b, v9.16b, v10.16b, #8 \n"

  848. 

  849.                     "fmla   v6.4s, v11.4s, %16.s[1]     \n"

  850.                     "fmla   v13.4s, v12.4s, %16.s[2]    \n"

  851. 

  852.                     "fmla   v8.4s, v9.4s, %15.s[0]      \n"

  853.                     "fmla   v14.4s, v11.4s, %15.s[1]    \n"

  854.                     "fmla   v15.4s, v12.4s, %15.s[2]    \n"

  855. 

  856.                     "prfm   pldl1keep, [%6, #256]       \n"

  857.                     "ld1    {v9.4s, v10.4s}, [%6]       \n"// r3

  858.                     "add    %6, %6, #16                 \n"

  859. 

  860.                     "fmla   v8.4s, v9.4s, %16.s[0]      \n"

  861. 

  862.                     "ext    v11.16b, v9.16b, v10.16b, #4 \n"

  863.                     "ext    v12.16b, v9.16b, v10.16b, #8 \n"

  864. 

  865.                     "fmla   v14.4s, v11.4s, %16.s[1]    \n"

  866.                     "fmla   v15.4s, v12.4s, %16.s[2]    \n"

  867. 

  868.                     "fadd   v7.4s, v7.4s, v6.4s         \n"

  869. 

  870.                     "prfm   pldl1keep, [%3, #256]       \n"

  871.                     "ld1    {v9.4s, v10.4s}, [%3]       \n"// r0

  872. 

  873.                     "fadd   v8.4s, v8.4s, v14.4s        \n"

  874.                     "fadd   v7.4s, v7.4s, v13.4s        \n"

  875.                     "fadd   v8.4s, v8.4s, v15.4s        \n"

  876. 

  877.                     "ext    v11.16b, v9.16b, v10.16b, #4 \n"

  878.                     "ext    v12.16b, v9.16b, v10.16b, #8 \n"

  879. 

  880.                     "add    %3, %3, #16                 \n"

  881. 

  882.                     "st1    {v7.4s}, [%1], #16          \n"

  883.                     "st1    {v8.4s}, [%2], #16          \n"

  884. 

  885.                     "subs   %w0, %w0, #1                \n"

  886.                     "bne    0b                          \n"

  887. 

  888.                     "sub    %3, %3, #16                 \n"

  889.                     : "=r"(nn),         // %0

  890.                       "=r"(outptr),     // %1

  891.                       "=r"(outptr2),    // %2

  892.                       "=r"(r0),         // %3

  893.                       "=r"(r1),         // %4

  894.                       "=r"(r2),         // %5

  895.                       "=r"(r3)          // %6

  896.                     : "0"(nn),

  897.                       "1"(outptr),

  898.                       "2"(outptr2),

  899.                       "3"(r0),

  900.                       "4"(r1),

  901.                       "5"(r2),

  902.                       "6"(r3),

  903.                       "w"(_k0123),      // %14

  904.                       "w"(_k3456),      // %15

  905.                       "w"(_k6789)       // %16

  906.                     : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  907.                 );

  908.                 }

  909. #else

  910.                 if (nn > 0)

  911.                 {

  912.                 asm volatile(

  913.                     "pld        [%3, #192]          \n"

  914.                     "vld1.f32   {d18-d20}, [%3 :64] \n"// r0

  915.                     "add        %3, #16             \n"

  916. 

  917.                     "vext.32    q11, q9, q10, #1    \n"

  918.                     "vext.32    q12, q9, q10, #2    \n"

  919. 

  920.                     "0:                             \n"

  921. 

  922.                     "pld        [%1, #128]          \n"

  923.                     "vld1.f32   {d14-d15}, [%1 :64] \n"// _sum

  924. 

  925.                     "vmla.f32   q7, q9, %e14[0]     \n"

  926.                     "vmul.f32   q6, q11, %e14[1]    \n"

  927.                     "vmul.f32   q13, q12, %f14[0]   \n"

  928. 

  929.                     "pld        [%4, #192]          \n"

  930.                     "vld1.f32   {d18-d20}, [%4]     \n"// r1

  931.                     "add        %4, #16             \n"

  932. 

  933.                     "vmla.f32   q7, q9, %e15[0]     \n"

  934. 

  935.                     "vext.32    q11, q9, q10, #1    \n"

  936.                     "vext.32    q12, q9, q10, #2    \n"

  937. 

  938.                     "vmla.f32   q6, q11, %e15[1]    \n"

  939.                     "vmla.f32   q13, q12, %f15[0]   \n"

  940. 

  941.                     "pld        [%2, #128]          \n"

  942.                     "vld1.f32   {d16-d17}, [%2]     \n"// _sum2

  943. 

  944.                     "vmla.f32   q8, q9, %e14[0]     \n"

  945.                     "vmul.f32   q14, q11, %e14[1]   \n"

  946.                     "vmul.f32   q15, q12, %f14[0]   \n"

  947. 

  948.                     "pld        [%5, #192]          \n"

  949.                     "vld1.f32   {d18-d20}, [%5 :64] \n"// r2

  950.                     "add        %5, #16             \n"

  951. 

  952.                     "vmla.f32   q7, q9, %e16[0]     \n"

  953. 

  954.                     "vext.32    q11, q9, q10, #1    \n"

  955.                     "vext.32    q12, q9, q10, #2    \n"

  956. 

  957.                     "vmla.f32   q6, q11, %e16[1]    \n"

  958.                     "vmla.f32   q13, q12, %f16[0]   \n"

  959. 

  960.                     "vmla.f32   q8, q9, %e15[0]     \n"

  961.                     "vmla.f32   q14, q11, %e15[1]   \n"

  962.                     "vmla.f32   q15, q12, %f15[0]   \n"

  963. 

  964.                     "pld        [%6, #192]          \n"

  965.                     "vld1.f32   {d18-d20}, [%6]     \n"// r3

  966.                     "add        %6, #16             \n"

  967. 

  968.                     "vmla.f32   q8, q9, %e16[0]     \n"

  969. 

  970.                     "vext.32    q11, q9, q10, #1    \n"

  971.                     "vext.32    q12, q9, q10, #2    \n"

  972. 

  973.                     "vmla.f32   q14, q11, %e16[1]   \n"

  974.                     "vmla.f32   q15, q12, %f16[0]   \n"

  975. 

  976.                     "vadd.f32   q7, q7, q6          \n"

  977. 

  978.                     "pld        [%3, #192]          \n"

  979.                     "vld1.f32   {d18-d20}, [%3 :64] \n"// r0

  980. 

  981.                     "vadd.f32   q8, q8, q14         \n"

  982.                     "vadd.f32   q7, q7, q13         \n"

  983.                     "vadd.f32   q8, q8, q15         \n"

  984. 

  985.                     "vext.32    q11, q9, q10, #1    \n"

  986.                     "vext.32    q12, q9, q10, #2    \n"

  987. 

  988.                     "add        %3, #16             \n"

  989. 

  990.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  991.                     "vst1.f32   {d16-d17}, [%2]!    \n"

  992. 

  993.                     "subs       %0, #1              \n"

  994.                     "bne        0b                  \n"

  995. 

  996.                     "sub        %3, #16             \n"

  997.                     : "=r"(nn),         // %0

  998.                       "=r"(outptr),     // %1

  999.                       "=r"(outptr2),    // %2

  1000.                       "=r"(r0),         // %3

  1001.                       "=r"(r1),         // %4

  1002.                       "=r"(r2),         // %5

  1003.                       "=r"(r3)          // %6

  1004.                     : "0"(nn),

  1005.                       "1"(outptr),

  1006.                       "2"(outptr2),

  1007.                       "3"(r0),

  1008.                       "4"(r1),

  1009.                       "5"(r2),

  1010.                       "6"(r3),

  1011.                       "w"(_k0123),      // %14

  1012.                       "w"(_k3456),      // %15

  1013.                       "w"(_k6789)       // %16

  1014.                     : "cc", "memory", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  1015.                 );

  1016.                 }

  1017. #endif // __aarch64__

  1018. #endif // __ARM_NEON

  1019.                 for (; remain>0; remain--)

  1020.                 {

  1021. #if __ARM_NEON

  1022.                     float32x4_t _r00 = vld1q_f32(r0);

  1023.                     float32x4_t _r10 = vld1q_f32(r1);

  1024.                     float32x4_t _r20 = vld1q_f32(r2);

  1025.                     float32x4_t _r30 = vld1q_f32(r3);

  1026. 

  1027.                     float32x4_t _sum = vmulq_f32(_r00, _k0123);

  1028.                     _sum = vmlaq_f32(_sum, _r10, _k3456);

  1029.                     _sum = vmlaq_f32(_sum, _r20, _k6789);

  1030. 

  1031.                     float32x4_t _sum2 = vmulq_f32(_r10, _k0123);

  1032.                     _sum2 = vmlaq_f32(_sum2, _r20, _k3456);

  1033.                     _sum2 = vmlaq_f32(_sum2, _r30, _k6789);

  1034. 

  1035.                     _sum = vsetq_lane_f32(*outptr, _sum, 3);

  1036.                     _sum2 = vsetq_lane_f32(*outptr2, _sum2, 3);

  1037. 

  1038. #if __aarch64__

  1039.                     *outptr = vaddvq_f32(_sum);

  1040.                     *outptr2 = vaddvq_f32(_sum2);

  1041. #else

  1042.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  1043.                     float32x2_t _ss2 = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2));

  1044. 

  1045.                     float32x2_t _sss2 = vpadd_f32(_ss, _ss2);

  1046. 

  1047.                     *outptr = vget_lane_f32(_sss2, 0);

  1048.                     *outptr2 = vget_lane_f32(_sss2, 1);

  1049. #endif // __aarch64__

  1050. #else

  1051.                     float sum = 0;

  1052.                     float sum2 = 0;

  1053. 

  1054.                     sum += r0[0] * k0[0];

  1055.                     sum += r0[1] * k0[1];

  1056.                     sum += r0[2] * k0[2];

  1057.                     sum += r1[0] * k1[0];

  1058.                     sum += r1[1] * k1[1];

  1059.                     sum += r1[2] * k1[2];

  1060.                     sum += r2[0] * k2[0];

  1061.                     sum += r2[1] * k2[1];

  1062.                     sum += r2[2] * k2[2];

  1063. 

  1064.                     sum2 += r1[0] * k0[0];

  1065.                     sum2 += r1[1] * k0[1];

  1066.                     sum2 += r1[2] * k0[2];

  1067.                     sum2 += r2[0] * k1[0];

  1068.                     sum2 += r2[1] * k1[1];

  1069.                     sum2 += r2[2] * k1[2];

  1070.                     sum2 += r3[0] * k2[0];

  1071.                     sum2 += r3[1] * k2[1];

  1072.                     sum2 += r3[2] * k2[2];

  1073. 

  1074.                     *outptr += sum;

  1075.                     *outptr2 += sum2;

  1076. #endif

  1077.                     r0++;

  1078.                     r1++;

  1079.                     r2++;

  1080.                     r3++;

  1081.                     outptr++;

  1082.                     outptr2++;

  1083.                 }

  1084. 

  1085.                 r0 += 2 + w;

  1086.                 r1 += 2 + w;

  1087.                 r2 += 2 + w;

  1088.                 r3 += 2 + w;

  1089. 

  1090.                 outptr += outw;

  1091.                 outptr2 += outw;

  1092.             }

  1093. 

  1094.             for (; i < outh; i++)

  1095.             {

  1096. 

  1097. #if __ARM_NEON

  1098.                 int nn = outw >> 2;

  1099.                 int remain = outw & 3;

  1100. #else

  1101.                 int remain = outw;

  1102. #endif // __ARM_NEON

  1103. 

  1104. #if __ARM_NEON

  1105. #if __aarch64__

  1106.                 if (nn > 0)

  1107.                 {

  1108.                 asm volatile(

  1109.                     "prfm   pldl1keep, [%2, #256]       \n"

  1110.                     "ld1    {v8.4s, v9.4s}, [%2]        \n"// r0

  1111.                     "add    %2, %2, #16                 \n"

  1112. 

  1113.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  1114.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  1115. 

  1116.                     "0:                                 \n"

  1117. 

  1118.                     "prfm   pldl1keep, [%1, #128]       \n"

  1119.                     "ld1    {v7.4s}, [%1]               \n"// _sum

  1120. 

  1121.                     "fmla   v7.4s, v8.4s, %10.s[0]      \n"

  1122.                     "fmul   v13.4s, v10.4s, %10.s[1]    \n"

  1123.                     "fmul   v14.4s, v11.4s, %10.s[2]    \n"

  1124. 

  1125.                     "prfm   pldl1keep, [%3, #256]       \n"

  1126.                     "ld1    {v8.4s, v9.4s}, [%3]        \n"// r1

  1127.                     "add    %3, %3, #16                 \n"

  1128. 

  1129.                     "fmla   v7.4s, v8.4s, %11.s[0]      \n"

  1130. 

  1131.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  1132.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  1133. 

  1134.                     "fmla   v13.4s, v10.4s, %11.s[1]    \n"

  1135.                     "fmla   v14.4s, v11.4s, %11.s[2]    \n"

  1136. 

  1137.                     "prfm   pldl1keep, [%4, #256]       \n"

  1138.                     "ld1    {v8.4s, v9.4s}, [%4]        \n"// r2

  1139.                     "add    %4, %4, #16                 \n"

  1140. 

  1141.                     "fmla   v7.4s, v8.4s, %12.s[0]      \n"

  1142. 

  1143.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  1144.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  1145. 

  1146.                     "fmla   v13.4s, v10.4s, %12.s[1]    \n"

  1147.                     "fmla   v14.4s, v11.4s, %12.s[2]    \n"

  1148. 

  1149.                     "prfm   pldl1keep, [%2, #256]       \n"

  1150.                     "ld1    {v8.4s, v9.4s}, [%2]        \n"// r0

  1151.                     "add    %2, %2, #16                 \n"

  1152. 

  1153.                     "fadd   v7.4s, v7.4s, v13.4s        \n"

  1154.                     "fadd   v7.4s, v7.4s, v14.4s        \n"

  1155. 

  1156.                     "ext    v10.16b, v8.16b, v9.16b, #4 \n"

  1157.                     "ext    v11.16b, v8.16b, v9.16b, #8 \n"

  1158. 

  1159.                     "st1    {v7.4s}, [%1], #16          \n"

  1160. 

  1161.                     "subs   %w0, %w0, #1                \n"

  1162.                     "bne    0b                          \n"

  1163. 

  1164.                     "sub    %2, %2, #16                 \n"

  1165.                     : "=r"(nn),         // %0

  1166.                       "=r"(outptr),     // %1

  1167.                       "=r"(r0),         // %2

  1168.                       "=r"(r1),         // %3

  1169.                       "=r"(r2)          // %4

  1170.                     : "0"(nn),

  1171.                       "1"(outptr),

  1172.                       "2"(r0),

  1173.                       "3"(r1),

  1174.                       "4"(r2),

  1175.                       "w"(_k0123),      // %10

  1176.                       "w"(_k3456),      // %11

  1177.                       "w"(_k6789)       // %12

  1178.                     : "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  1179.                 );

  1180.                 }

  1181. #else

  1182.                 if (nn > 0)

  1183.                 {

  1184.                 asm volatile(

  1185.                     "pld        [%2, #192]          \n"

  1186.                     "vld1.f32   {d16-d18}, [%2]     \n"// r0

  1187.                     "add        %2, #16             \n"

  1188. 

  1189.                     "vext.32    q10, q8, q9, #1     \n"

  1190.                     "vext.32    q11, q8, q9, #2     \n"

  1191. 

  1192.                     "0:                             \n"

  1193. 

  1194.                     "pld        [%1, #128]          \n"

  1195.                     "vld1.f32   {d14-d15}, [%1]     \n"// _sum

  1196. 

  1197.                     "vmla.f32   q7, q8, %e10[0]     \n"

  1198.                     "vmul.f32   q13, q10, %e10[1]   \n"

  1199.                     "vmul.f32   q14, q11, %f10[0]   \n"

  1200. 

  1201.                     "pld        [%3, #192]          \n"

  1202.                     "vld1.f32   {d16-d18}, [%3]     \n"// r1

  1203.                     "add        %3, #16             \n"

  1204. 

  1205.                     "vmla.f32   q7, q8, %e11[0]     \n"

  1206. 

  1207.                     "vext.32    q10, q8, q9, #1     \n"

  1208.                     "vext.32    q11, q8, q9, #2     \n"

  1209. 

  1210.                     "vmla.f32   q13, q10, %e11[1]   \n"

  1211.                     "vmla.f32   q14, q11, %f11[0]   \n"

  1212. 

  1213.                     "pld        [%4, #192]          \n"

  1214.                     "vld1.f32   {d16-d18}, [%4]     \n"// r2

  1215.                     "add        %4, #16             \n"

  1216. 

  1217.                     "vmla.f32   q7, q8, %e12[0]     \n"

  1218. 

  1219.                     "vext.32    q10, q8, q9, #1     \n"

  1220.                     "vext.32    q11, q8, q9, #2     \n"

  1221. 

  1222.                     "vmla.f32   q13, q10, %e12[1]   \n"

  1223.                     "vmla.f32   q14, q11, %f12[0]   \n"

  1224. 

  1225.                     "pld        [%2, #192]          \n"

  1226.                     "vld1.f32   {d16-d18}, [%2]     \n"// r0

  1227.                     "add        %2, #16             \n"

  1228. 

  1229.                     "vadd.f32   q7, q7, q13         \n"

  1230.                     "vadd.f32   q7, q7, q14         \n"

  1231. 

  1232.                     "vext.32    q10, q8, q9, #1     \n"

  1233.                     "vext.32    q11, q8, q9, #2     \n"

  1234. 

  1235.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  1236. 

  1237.                     "subs       %0, #1              \n"

  1238.                     "bne        0b                  \n"

  1239. 

  1240.                     "sub        %2, #16             \n"

  1241.                     : "=r"(nn),         // %0

  1242.                       "=r"(outptr),     // %1

  1243.                       "=r"(r0),         // %2

  1244.                       "=r"(r1),         // %3

  1245.                       "=r"(r2)          // %4

  1246.                     : "0"(nn),

  1247.                       "1"(outptr),

  1248.                       "2"(r0),

  1249.                       "3"(r1),

  1250.                       "4"(r2),

  1251.                       "w"(_k0123),      // %10

  1252.                       "w"(_k3456),      // %11

  1253.                       "w"(_k6789)       // %12

  1254.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  1255.                 );

  1256.                 }

  1257. #endif // __aarch64__

  1258. #endif // __ARM_NEON

  1259.                 for (; remain>0; remain--)

  1260.                 {

  1261. #if __ARM_NEON

  1262.                     float32x4_t _r00 = vld1q_f32(r0);

  1263.                     float32x4_t _r10 = vld1q_f32(r1);

  1264.                     float32x4_t _r20 = vld1q_f32(r2);

  1265. 

  1266.                     float32x4_t _sum = vmulq_f32(_r00, _k0123);

  1267.                     _sum = vmlaq_f32(_sum, _r10, _k3456);

  1268.                     _sum = vmlaq_f32(_sum, _r20, _k6789);

  1269. 

  1270.                     _sum = vsetq_lane_f32(*outptr, _sum, 3);

  1271. 

  1272. #if __aarch64__

  1273.                     *outptr = vaddvq_f32(_sum);

  1274. #else

  1275.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  1276.                     _ss = vpadd_f32(_ss, _ss);

  1277. 

  1278.                     *outptr = vget_lane_f32(_ss, 0);

  1279. #endif // __aarch64__

  1280. #else

  1281.                     float sum = 0;

  1282. 

  1283.                     sum += r0[0] * k0[0];

  1284.                     sum += r0[1] * k0[1];

  1285.                     sum += r0[2] * k0[2];

  1286.                     sum += r1[0] * k1[0];

  1287.                     sum += r1[1] * k1[1];

  1288.                     sum += r1[2] * k1[2];

  1289.                     sum += r2[0] * k2[0];

  1290.                     sum += r2[1] * k2[1];

  1291.                     sum += r2[2] * k2[2];

  1292. 

  1293.                     *outptr += sum;

  1294. #endif

  1295.                     r0++;

  1296.                     r1++;

  1297.                     r2++;

  1298.                     outptr++;

  1299.                 }

  1300. 

  1301.                 r0 += 2;

  1302.                 r1 += 2;

  1303.                 r2 += 2;

  1304.             }

  1305. 

  1306.             kernel0 += 9;

  1307.         }

  1308.     }

  1309. 

  1310. }

  1311. 

  1312. static void conv3x3s1_winograd64_transform_kernel_neon(const Mat& kernel, Mat& kernel_tm, int inch, int outch)

  1313. {

  1314.     kernel_tm.create(8*8, inch, outch);

  1315. 

  1316.     const float ktm[8][3] = {

  1317.         {   1.0f,     0.0f,     0.0f},

  1318.         {-2.0f/9,  -2.0f/9,  -2.0f/9},

  1319.         {-2.0f/9,   2.0f/9,  -2.0f/9},

  1320.         {1.0f/90,  1.0f/45,  2.0f/45},

  1321.         {1.0f/90, -1.0f/45,  2.0f/45},

  1322.         {1.0f/45,  1.0f/90, 1.0f/180},

  1323.         {1.0f/45, -1.0f/90, 1.0f/180},

  1324.         {   0.0f,     0.0f,     1.0f}

  1325.     };

  1326. 

  1327.     #pragma omp parallel for

  1328.     for (int p = 0; p<outch; p++)

  1329.     {

  1330.         for (int q = 0; q<inch; q++)

  1331.         {

  1332.             const float* kernel0 = (const float*)kernel + p*inch * 9 + q * 9;

  1333.             float* kernel_tm0 = kernel_tm.channel(p).row(q);

  1334. 

  1335.             // transform kernel, transposed

  1336.             const float* k0 = kernel0;

  1337.             const float* k1 = kernel0 + 3;

  1338.             const float* k2 = kernel0 + 6;

  1339. 

  1340.             // h

  1341.             float tmp[8][3];

  1342.             for (int i=0; i<8; i++)

  1343.             {

  1344.                 tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2];

  1345.                 tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2];

  1346.                 tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2];

  1347.             }

  1348. 

  1349.             // v

  1350.             for (int j=0; j<8; j++)

  1351.             {

  1352.                 float* tmpp = &tmp[j][0];

  1353. 

  1354.                 for (int i=0; i<8; i++)

  1355.                 {

  1356.                     kernel_tm0[j*8 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2];

  1357.                 }

  1358.             }

  1359.         }

  1360.     }

  1361. 

  1362.     // optimized layout for winograd4

  1363.     // interleave weights

  1364.     int nn_outch = outch >> 2;

  1365.     int remain_outch_start = nn_outch << 2;

  1366. 

  1367.     Mat kernel_tm2(8*8 * inch * 4, 1, nn_outch + (outch % 4 + 3) / 4);

  1368. 

  1369.     #pragma omp parallel for

  1370.     for (int pp=0; pp<nn_outch; pp++)

  1371.     {

  1372.         int p = pp * 4;

  1373. 

  1374.         float* ktm2 = kernel_tm2.channel(pp);

  1375. 

  1376.         const Mat kernel0_tm = kernel_tm.channel(p);

  1377.         const Mat kernel1_tm = kernel_tm.channel(p+1);

  1378.         const Mat kernel2_tm = kernel_tm.channel(p+2);

  1379.         const Mat kernel3_tm = kernel_tm.channel(p+3);

  1380. 

  1381.         int q=0;

  1382. 

  1383. #if __ARM_NEON && __aarch64__

  1384.         for (; q+3<inch; q+=4)

  1385.         {

  1386.             const float* k00 = kernel0_tm.row(q);

  1387.             const float* k01 = kernel0_tm.row(q+1);

  1388.             const float* k02 = kernel0_tm.row(q+2);

  1389.             const float* k03 = kernel0_tm.row(q+3);

  1390.             const float* k10 = kernel1_tm.row(q);

  1391.             const float* k11 = kernel1_tm.row(q+1);

  1392.             const float* k12 = kernel1_tm.row(q+2);

  1393.             const float* k13 = kernel1_tm.row(q+3);

  1394.             const float* k20 = kernel2_tm.row(q);

  1395.             const float* k21 = kernel2_tm.row(q+1);

  1396.             const float* k22 = kernel2_tm.row(q+2);

  1397.             const float* k23 = kernel2_tm.row(q+3);

  1398.             const float* k30 = kernel3_tm.row(q);

  1399.             const float* k31 = kernel3_tm.row(q+1);

  1400.             const float* k32 = kernel3_tm.row(q+2);

  1401.             const float* k33 = kernel3_tm.row(q+3);

  1402. 

  1403.             for (int r=0; r<16; r++)

  1404.             {

  1405.             // split into two asm blocks for gcc reject over 30 oprands :(

  1406.             asm volatile(

  1407.                 "ld1    {v0.4s}, [%1], #16  \n"

  1408.                 "ld1    {v1.4s}, [%2], #16  \n"

  1409.                 "ld1    {v2.4s}, [%3], #16  \n"

  1410.                 "ld1    {v3.4s}, [%4], #16  \n"

  1411.                 "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64  \n"

  1412. 

  1413.                 "ld1    {v0.4s}, [%5], #16  \n"

  1414.                 "ld1    {v1.4s}, [%6], #16  \n"

  1415.                 "ld1    {v2.4s}, [%7], #16  \n"

  1416.                 "ld1    {v3.4s}, [%8], #16  \n"

  1417.                 "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64  \n"

  1418. 

  1419.                 : "=r"(ktm2),   // %0

  1420.                   "=r"(k00),    // %1

  1421.                   "=r"(k01),    // %2

  1422.                   "=r"(k02),    // %3

  1423.                   "=r"(k03),    // %4

  1424.                   "=r"(k10),    // %5

  1425.                   "=r"(k11),    // %6

  1426.                   "=r"(k12),    // %7

  1427.                   "=r"(k13)     // %8

  1428.                 : "0"(ktm2),

  1429.                   "1"(k00),

  1430.                   "2"(k01),

  1431.                   "3"(k02),

  1432.                   "4"(k03),

  1433.                   "5"(k10),

  1434.                   "6"(k11),

  1435.                   "7"(k12),

  1436.                   "8"(k13)

  1437.                 : "cc", "memory", "v0", "v1", "v2", "v3"

  1438.             );

  1439.             asm volatile(

  1440.                 "ld1    {v0.4s}, [%1], #16  \n"

  1441.                 "ld1    {v1.4s}, [%2], #16  \n"

  1442.                 "ld1    {v2.4s}, [%3], #16  \n"

  1443.                 "ld1    {v3.4s}, [%4], #16  \n"

  1444.                 "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64  \n"

  1445. 

  1446.                 "ld1    {v0.4s}, [%5], #16  \n"

  1447.                 "ld1    {v1.4s}, [%6], #16  \n"

  1448.                 "ld1    {v2.4s}, [%7], #16  \n"

  1449.                 "ld1    {v3.4s}, [%8], #16  \n"

  1450.                 "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64  \n"

  1451. 

  1452.                 : "=r"(ktm2),   // %0

  1453.                   "=r"(k20),    // %1

  1454.                   "=r"(k21),    // %2

  1455.                   "=r"(k22),    // %3

  1456.                   "=r"(k23),    // %4

  1457.                   "=r"(k30),    // %5

  1458.                   "=r"(k31),    // %6

  1459.                   "=r"(k32),    // %7

  1460.                   "=r"(k33)     // %8

  1461.                 : "0"(ktm2),

  1462.                   "1"(k20),

  1463.                   "2"(k21),

  1464.                   "3"(k22),

  1465.                   "4"(k23),

  1466.                   "5"(k30),

  1467.                   "6"(k31),

  1468.                   "7"(k32),

  1469.                   "8"(k33)

  1470.                 : "cc", "memory", "v0", "v1", "v2", "v3"

  1471.             );

  1472.             }

  1473.         }

  1474. #endif // __ARM_NEON && __aarch64__

  1475. 

  1476.         for (; q+1<inch; q+=2)

  1477.         {

  1478.             const float* k00 = kernel0_tm.row(q);

  1479.             const float* k01 = kernel0_tm.row(q+1);

  1480.             const float* k10 = kernel1_tm.row(q);

  1481.             const float* k11 = kernel1_tm.row(q+1);

  1482.             const float* k20 = kernel2_tm.row(q);

  1483.             const float* k21 = kernel2_tm.row(q+1);

  1484.             const float* k30 = kernel3_tm.row(q);

  1485.             const float* k31 = kernel3_tm.row(q+1);

  1486. 

  1487.             for (int r=0; r<16; r++)

  1488.             {

  1489. #if __ARM_NEON

  1490. #if __aarch64__

  1491.             asm volatile(

  1492.                 "ld1    {v0.4s}, [%1], #16  \n"

  1493.                 "ld1    {v1.4s}, [%2], #16  \n"

  1494.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1495. 

  1496.                 "ld1    {v0.4s}, [%3], #16  \n"

  1497.                 "ld1    {v1.4s}, [%4], #16  \n"

  1498.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1499. 

  1500.                 "ld1    {v0.4s}, [%5], #16  \n"

  1501.                 "ld1    {v1.4s}, [%6], #16  \n"

  1502.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1503. 

  1504.                 "ld1    {v0.4s}, [%7], #16  \n"

  1505.                 "ld1    {v1.4s}, [%8], #16  \n"

  1506.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1507. 

  1508.                 : "=r"(ktm2),   // %0

  1509.                   "=r"(k00),    // %1

  1510.                   "=r"(k01),    // %2

  1511.                   "=r"(k10),    // %3

  1512.                   "=r"(k11),    // %4

  1513.                   "=r"(k20),    // %5

  1514.                   "=r"(k21),    // %6

  1515.                   "=r"(k30),    // %7

  1516.                   "=r"(k31)     // %8

  1517.                 : "0"(ktm2),

  1518.                   "1"(k00),

  1519.                   "2"(k01),

  1520.                   "3"(k10),

  1521.                   "4"(k11),

  1522.                   "5"(k20),

  1523.                   "6"(k21),

  1524.                   "7"(k30),

  1525.                   "8"(k31)

  1526.                 : "cc", "memory", "v0", "v1"

  1527.             );

  1528. #else

  1529.             asm volatile(

  1530.                 "vld1.f32   {d0-d1}, [%1 :128]! \n"

  1531.                 "vld1.f32   {d2-d3}, [%2 :128]! \n"

  1532.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1533. 

  1534.                 "vld1.f32   {d0-d1}, [%3 :128]! \n"

  1535.                 "vld1.f32   {d2-d3}, [%4 :128]! \n"

  1536.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1537. 

  1538.                 "vld1.f32   {d0-d1}, [%5 :128]! \n"

  1539.                 "vld1.f32   {d2-d3}, [%6 :128]! \n"

  1540.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1541. 

  1542.                 "vld1.f32   {d0-d1}, [%7 :128]! \n"

  1543.                 "vld1.f32   {d2-d3}, [%8 :128]! \n"

  1544.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1545. 

  1546.                 : "=r"(ktm2),   // %0

  1547.                   "=r"(k00),    // %1

  1548.                   "=r"(k01),    // %2

  1549.                   "=r"(k10),    // %3

  1550.                   "=r"(k11),    // %4

  1551.                   "=r"(k20),    // %5

  1552.                   "=r"(k21),    // %6

  1553.                   "=r"(k30),    // %7

  1554.                   "=r"(k31)     // %8

  1555.                 : "0"(ktm2),

  1556.                   "1"(k00),

  1557.                   "2"(k01),

  1558.                   "3"(k10),

  1559.                   "4"(k11),

  1560.                   "5"(k20),

  1561.                   "6"(k21),

  1562.                   "7"(k30),

  1563.                   "8"(k31)

  1564.                 : "cc", "memory", "q0", "q1"

  1565.             );

  1566. #endif // __aarch64__

  1567. #else

  1568.                 for (int m=0; m<4; m++)

  1569.                 {

  1570.                     ktm2[0 +m] = k00[m];

  1571.                     ktm2[4 +m] = k01[m];

  1572.                     ktm2[8 +m] = k10[m];

  1573.                     ktm2[12+m] = k11[m];

  1574.                     ktm2[16+m] = k20[m];

  1575.                     ktm2[20+m] = k21[m];

  1576.                     ktm2[24+m] = k30[m];

  1577.                     ktm2[28+m] = k31[m];

  1578.                 }

  1579. 

  1580.                 k00 += 4;

  1581.                 k01 += 4;

  1582.                 k10 += 4;

  1583.                 k11 += 4;

  1584.                 k20 += 4;

  1585.                 k21 += 4;

  1586.                 k30 += 4;

  1587.                 k31 += 4;

  1588.                 ktm2 += 32;

  1589. #endif // __ARM_NEON

  1590.             }

  1591.         }

  1592. 

  1593.         for (; q<inch; q++)

  1594.         {

  1595.             const float* k00 = kernel0_tm.row(q);

  1596.             const float* k10 = kernel1_tm.row(q);

  1597.             const float* k20 = kernel2_tm.row(q);

  1598.             const float* k30 = kernel3_tm.row(q);

  1599. 

  1600.             for (int r=0; r<16; r++)

  1601.             {

  1602. #if __ARM_NEON

  1603. #if __aarch64__

  1604.             asm volatile(

  1605.                 "ld1    {v0.4s}, [%1], #16  \n"

  1606.                 "ld1    {v1.4s}, [%2], #16  \n"

  1607.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1608. 

  1609.                 "ld1    {v0.4s}, [%3], #16  \n"

  1610.                 "ld1    {v1.4s}, [%4], #16  \n"

  1611.                 "st1    {v0.4s, v1.4s}, [%0], #32  \n"

  1612. 

  1613.                 : "=r"(ktm2),   // %0

  1614.                   "=r"(k00),    // %1

  1615.                   "=r"(k10),    // %2

  1616.                   "=r"(k20),    // %3

  1617.                   "=r"(k30)     // %4

  1618.                 : "0"(ktm2),

  1619.                   "1"(k00),

  1620.                   "2"(k10),

  1621.                   "3"(k20),

  1622.                   "4"(k30)

  1623.                 : "cc", "memory", "v0", "v1"

  1624.             );

  1625. #else

  1626.             asm volatile(

  1627.                 "vld1.f32   {d0-d1}, [%1 :128]! \n"

  1628.                 "vld1.f32   {d2-d3}, [%2 :128]! \n"

  1629.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1630. 

  1631.                 "vld1.f32   {d0-d1}, [%3 :128]! \n"

  1632.                 "vld1.f32   {d2-d3}, [%4 :128]! \n"

  1633.                 "vst1.f32   {d0-d3}, [%0 :128]! \n"

  1634. 

  1635.                 : "=r"(ktm2),   // %0

  1636.                   "=r"(k00),    // %1

  1637.                   "=r"(k10),    // %2

  1638.                   "=r"(k20),    // %3

  1639.                   "=r"(k30)     // %4

  1640.                 : "0"(ktm2),

  1641.                   "1"(k00),

  1642.                   "2"(k10),

  1643.                   "3"(k20),

  1644.                   "4"(k30)

  1645.                 : "cc", "memory", "q0", "q1"

  1646.             );

  1647. #endif // __aarch64__

  1648. #else

  1649.                 for (int m=0; m<4; m++)

  1650.                 {

  1651.                     ktm2[0 +m] = k00[m];

  1652.                     ktm2[4 +m] = k10[m];

  1653.                     ktm2[8 +m] = k20[m];

  1654.                     ktm2[12+m] = k30[m];

  1655.                 }

  1656. 

  1657.                 k00 += 4;

  1658.                 k10 += 4;

  1659.                 k20 += 4;

  1660.                 k30 += 4;

  1661.                 ktm2 += 16;

  1662. #endif // __ARM_NEON

  1663.             }

  1664.         }

  1665.     }

  1666. 

  1667.     #pragma omp parallel for

  1668.     for (int p = remain_outch_start; p<outch; p++)

  1669.     {

  1670.         float* ktm2 = (float*)kernel_tm2.channel(nn_outch) + 8*8 * inch * (p-remain_outch_start);

  1671. 

  1672.         const Mat kernel0_tm = kernel_tm.channel(p);

  1673. 

  1674.         int q = 0;

  1675. 

  1676.         for (; q<inch; q++)

  1677.         {

  1678.             const float* k00 = kernel0_tm.row(q);

  1679. 

  1680.             for (int r=0; r<16; r++)

  1681.             {

  1682. #if __ARM_NEON

  1683. #if __aarch64__

  1684.             asm volatile(

  1685.                 "ld1    {v0.4s}, [%1], #16  \n"

  1686.                 "st1    {v0.4s}, [%0], #16  \n"

  1687.                 : "=r"(ktm2),   // %0

  1688.                   "=r"(k00)     // %1

  1689.                 : "0"(ktm2),

  1690.                   "1"(k00)

  1691.                 : "cc", "memory", "v0"

  1692.             );

  1693. #else

  1694.             asm volatile(

  1695.                 "vld1.f32   {d0-d1}, [%1 :128]! \n"

  1696.                 "vst1.f32   {d0-d1}, [%0 :128]! \n"

  1697.                 : "=r"(ktm2),   // %0

  1698.                   "=r"(k00)     // %1

  1699.                 : "0"(ktm2),

  1700.                   "1"(k00)

  1701.                 : "cc", "memory", "q0"

  1702.             );

  1703. #endif // __aarch64__

  1704. #else

  1705.                 for (int m=0; m<4; m++)

  1706.                 {

  1707.                     ktm2[m] = k00[m];

  1708.                 }

  1709. 

  1710.                 k00 += 4;

  1711.                 ktm2 += 4;

  1712. #endif // __ARM_NEON

  1713.             }

  1714.         }

  1715.     }

  1716. 

  1717.     kernel_tm = kernel_tm2;

  1718. }

  1719. 

  1720. static void conv3x3s1_winograd64_transform_kernel_neon5(const Mat& kernel, Mat& kernel_tm, int inch, int outch)

  1721. {

  1722.     kernel_tm.create(8*8, inch, outch);

  1723. 

  1724.     const float ktm[8][3] = {

  1725.         {   1.0f,     0.0f,     0.0f},

  1726.         {-2.0f/9,  -2.0f/9,  -2.0f/9},

  1727.         {-2.0f/9,   2.0f/9,  -2.0f/9},

  1728.         {1.0f/90,  1.0f/45,  2.0f/45},

  1729.         {1.0f/90, -1.0f/45,  2.0f/45},

  1730.         {1.0f/45,  1.0f/90, 1.0f/180},

  1731.         {1.0f/45, -1.0f/90, 1.0f/180},

  1732.         {   0.0f,     0.0f,     1.0f}

  1733.     };

  1734. 

  1735.     #pragma omp parallel for

  1736.     for (int p = 0; p<outch; p++)

  1737.     {

  1738.         for (int q = 0; q<inch; q++)

  1739.         {

  1740.             const float* kernel0 = (const float*)kernel + p*inch * 9 + q * 9;

  1741.             float* kernel_tm0 = kernel_tm.channel(p).row(q);

  1742. 

  1743.             // transform kernel, transposed

  1744.             const float* k0 = kernel0;

  1745.             const float* k1 = kernel0 + 3;

  1746.             const float* k2 = kernel0 + 6;

  1747. 

  1748.             // h

  1749.             float tmp[8][3];

  1750.             for (int i=0; i<8; i++)

  1751.             {

  1752.                 tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2];

  1753.                 tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2];

  1754.                 tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2];

  1755.             }

  1756. 

  1757.             // v

  1758.             for (int j=0; j<8; j++)

  1759.             {

  1760.                 float* tmpp = &tmp[j][0];

  1761. 

  1762.                 for (int i=0; i<8; i++)

  1763.                 {

  1764.                     kernel_tm0[j*8 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2];

  1765.                 }

  1766.             }

  1767.         }

  1768.     }

  1769. 

  1770. 

  1771.     // optimized layout for winograd5

  1772.     // interleave weights

  1773. //     Mat kernel_tm2(8*8, inch, outch);

  1774. //     Mat kernel_tm2(inch, 64, outch);

  1775. #if __ARM_NEON && __aarch64__

  1776.     Mat kernel_tm2(8*4*(inch/4) + 8*(inch%4), 64, outch/8 + (outch%8)/4 + outch%4);

  1777. #else

  1778.     Mat kernel_tm2(4*4*(inch/4) + 4*(inch%4), 64, outch/4 + outch%4);

  1779. #endif

  1780. 

  1781.     int p=0;

  1782. #if __aarch64__

  1783.     for (; p+7<outch; p+=8)

  1784.     {

  1785.         const Mat kernel0_tm = kernel_tm.channel(p);

  1786.         const Mat kernel1_tm = kernel_tm.channel(p+1);

  1787.         const Mat kernel2_tm = kernel_tm.channel(p+2);

  1788.         const Mat kernel3_tm = kernel_tm.channel(p+3);

  1789.         const Mat kernel4_tm = kernel_tm.channel(p+4);

  1790.         const Mat kernel5_tm = kernel_tm.channel(p+5);

  1791.         const Mat kernel6_tm = kernel_tm.channel(p+6);

  1792.         const Mat kernel7_tm = kernel_tm.channel(p+7);

  1793. 

  1794.         Mat ktm2 = kernel_tm2.channel(p/8);

  1795. 

  1796.         for (int r=0; r<64; r++)

  1797.         {

  1798.             float* ktm2p = ktm2.row(r);

  1799. 

  1800.             for (int q=0; q<inch; q++)

  1801.             {

  1802.                 const float* ktm0_0 = kernel0_tm.row(q);

  1803.                 const float* ktm1_0 = kernel1_tm.row(q);

  1804.                 const float* ktm2_0 = kernel2_tm.row(q);

  1805.                 const float* ktm3_0 = kernel3_tm.row(q);

  1806.                 const float* ktm4_0 = kernel4_tm.row(q);

  1807.                 const float* ktm5_0 = kernel5_tm.row(q);

  1808.                 const float* ktm6_0 = kernel6_tm.row(q);

  1809.                 const float* ktm7_0 = kernel7_tm.row(q);

  1810. 

  1811.                 ktm2p[0] = ktm0_0[r];

  1812.                 ktm2p[1] = ktm1_0[r];

  1813.                 ktm2p[2] = ktm2_0[r];

  1814.                 ktm2p[3] = ktm3_0[r];

  1815.                 ktm2p[4] = ktm4_0[r];

  1816.                 ktm2p[5] = ktm5_0[r];

  1817.                 ktm2p[6] = ktm6_0[r];

  1818.                 ktm2p[7] = ktm7_0[r];

  1819. 

  1820.                 ktm2p += 8;

  1821.             }

  1822.         }

  1823.     }

  1824. #endif // __aarch64__

  1825.     for (; p+3<outch; p+=4)

  1826.     {

  1827.         const Mat kernel0_tm = kernel_tm.channel(p);

  1828.         const Mat kernel1_tm = kernel_tm.channel(p+1);

  1829.         const Mat kernel2_tm = kernel_tm.channel(p+2);

  1830.         const Mat kernel3_tm = kernel_tm.channel(p+3);

  1831. 

  1832. #if __ARM_NEON && __aarch64__

  1833.         Mat ktm2 = kernel_tm2.channel(p/8+(p%8)/4);

  1834. #else

  1835.         Mat ktm2 = kernel_tm2.channel(p/4);

  1836. #endif

  1837. 

  1838.         for (int r=0; r<64; r++)

  1839.         {

  1840.             float* ktm2p = ktm2.row(r);

  1841. 

  1842.             for (int q=0; q<inch; q++)

  1843.             {

  1844.                 const float* ktm0_0 = kernel0_tm.row(q);

  1845.                 const float* ktm1_0 = kernel1_tm.row(q);

  1846.                 const float* ktm2_0 = kernel2_tm.row(q);

  1847.                 const float* ktm3_0 = kernel3_tm.row(q);

  1848. 

  1849.                 ktm2p[0] = ktm0_0[r];

  1850.                 ktm2p[1] = ktm1_0[r];

  1851.                 ktm2p[2] = ktm2_0[r];

  1852.                 ktm2p[3] = ktm3_0[r];

  1853. 

  1854.                 ktm2p += 4;

  1855.             }

  1856.         }

  1857.     }

  1858.     for (; p<outch; p++)

  1859.     {

  1860.         const Mat kernel0_tm = kernel_tm.channel(p);

  1861. 

  1862. #if __ARM_NEON && __aarch64__

  1863.         Mat ktm2 = kernel_tm2.channel(p/8+(p%8)/4+p%4);

  1864. #else

  1865.         Mat ktm2 = kernel_tm2.channel(p/4+p%4);

  1866. #endif

  1867. 

  1868.         for (int r=0; r<64; r++)

  1869.         {

  1870.             float* ktm2p = ktm2.row(r);

  1871. 

  1872.             for (int q=0; q<inch; q++)

  1873.             {

  1874.                 const float* ktm0_0 = kernel0_tm.row(q);

  1875. 

  1876.                 ktm2p[0] = ktm0_0[r];

  1877. 

  1878.                 ktm2p += 1;

  1879.             }

  1880.         }

  1881.     }

  1882. 

  1883.     kernel_tm = kernel_tm2;

  1884. }

  1885. 

  1886. #if 0//TODO remove old code sometime later

  1887. static void conv3x3s1_winograd64_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias)

  1888. {

  1889.     int w = bottom_blob.w;

  1890.     int h = bottom_blob.h;

  1891.     int inch = bottom_blob.c;

  1892. 

  1893.     int outw = top_blob.w;

  1894.     int outh = top_blob.h;

  1895.     int outch = top_blob.c;

  1896. 

  1897.     // pad to 6n+2

  1898.     Mat bottom_blob_bordered = bottom_blob;

  1899. 

  1900.     outw = (outw + 5) / 6 * 6;

  1901.     outh = (outh + 5) / 6 * 6;

  1902. 

  1903.     w = outw + 2;

  1904.     h = outh + 2;

  1905.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f);

  1906. 

  1907.     const float* bias = _bias;

  1908. 

  1909.     // BEGIN transform input

  1910.     Mat bottom_blob_tm;

  1911.     {

  1912.         int w_tm = outw / 6 * 8;

  1913.         int h_tm = outh / 6 * 8;

  1914.         bottom_blob_tm.create(8*8, w_tm/8 * h_tm/8, inch);

  1915. 

  1916. //         const float itm[8][8] = {

  1917. //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  1918. //

  1919. //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  1920. //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  1921. //

  1922. //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  1923. //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  1924. //

  1925. //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  1926. //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  1927. //

  1928. //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  1929. //         };

  1930. 

  1931.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  1932.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  1933. 

  1934.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  1935.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  1936. 

  1937.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  1938.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  1939. 

  1940.         // reuse r04 * 1.25

  1941.         // reuse r03 * 2.5

  1942.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  1943.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  1944. 

  1945.         #pragma omp parallel for

  1946.         for (int q = 0; q<inch; q++)

  1947.         {

  1948.             const Mat img0 = bottom_blob_bordered.channel(q);

  1949.             Mat img0_tm = bottom_blob_tm.channel(q);

  1950. 

  1951.             float tmp[8][8];

  1952. 

  1953.             // tile

  1954.             for (int i=0; i<h_tm/8; i++)

  1955.             {

  1956.                 for (int j=0; j<w_tm/8; j++)

  1957.                 {

  1958.                     const float* r0 = img0.row(i * 6) + j * 6;

  1959.                     float* r0_tm = img0_tm.row(i * w_tm/8 + j);

  1960. 

  1961.                     // TODO neon optimize

  1962.                     for (int m=0; m<8; m++)

  1963.                     {

  1964.                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25;

  1965.                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25;

  1966. 

  1967.                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25);

  1968.                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25);

  1969. 

  1970.                         tmp[1][m] = tmp12a + tmp12b;

  1971.                         tmp[2][m] = tmp12a - tmp12b;

  1972. 

  1973.                         float tmp34a = (r0[6] + r0[2] * 0.25 - r0[4] * 1.25);

  1974.                         float tmp34b = (r0[1] * 0.5 - r0[3] * 2.5 + r0[5] * 2);

  1975. 

  1976.                         tmp[3][m] = tmp34a + tmp34b;

  1977.                         tmp[4][m] = tmp34a - tmp34b;

  1978. 

  1979.                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25) * 4);

  1980.                         float tmp56b = (r0[1] * 2 - r0[3] * 2.5 + r0[5] * 0.5);

  1981. 

  1982.                         tmp[5][m] = tmp56a + tmp56b;

  1983.                         tmp[6][m] = tmp56a - tmp56b;

  1984. 

  1985.                         r0 += w;

  1986.                     }

  1987. 

  1988.                     for (int m=0; m<8; m++)

  1989.                     {

  1990.                         const float* tmp0 = tmp[m];

  1991. 

  1992.                         r0_tm[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25;

  1993.                         r0_tm[7] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25;

  1994. 

  1995.                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25);

  1996.                         float tmp12b = (tmp0[1] - tmp0[3] * 4.25 + tmp0[5]);

  1997. 

  1998.                         r0_tm[1] = tmp12a + tmp12b;

  1999.                         r0_tm[2] = tmp12a - tmp12b;

  2000. 

  2001.                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25 - tmp0[4] * 1.25);

  2002.                         float tmp34b = (tmp0[1] * 0.5 - tmp0[3] * 2.5 + tmp0[5] * 2);

  2003. 

  2004.                         r0_tm[3] = tmp34a + tmp34b;

  2005.                         r0_tm[4] = tmp34a - tmp34b;

  2006. 

  2007.                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25) * 4);

  2008.                         float tmp56b = (tmp0[1] * 2 - tmp0[3] * 2.5 + tmp0[5] * 0.5);

  2009. 

  2010.                         r0_tm[5] = tmp56a + tmp56b;

  2011.                         r0_tm[6] = tmp56a - tmp56b;

  2012. 

  2013.                         r0_tm += 8;

  2014.                     }

  2015.                 }

  2016.             }

  2017.         }

  2018. 

  2019.     }

  2020.     bottom_blob_bordered = Mat();

  2021.     // END transform input

  2022. 

  2023.     // BEGIN dot

  2024.     Mat top_blob_tm;

  2025.     {

  2026.         int w_tm = outw / 6 * 8;

  2027.         int h_tm = outh / 6 * 8;

  2028.         top_blob_tm.create(8*8, w_tm/8 * h_tm/8, outch);

  2029. 

  2030.         int nn_outch = outch >> 2;

  2031.         int remain_outch_start = nn_outch << 2;

  2032. 

  2033.         #pragma omp parallel for

  2034.         for (int pp=0; pp<nn_outch; pp++)

  2035.         {

  2036.             int p = pp * 4;

  2037. 

  2038.             Mat out0_tm = top_blob_tm.channel(p);

  2039.             Mat out1_tm = top_blob_tm.channel(p+1);

  2040.             Mat out2_tm = top_blob_tm.channel(p+2);

  2041.             Mat out3_tm = top_blob_tm.channel(p+3);

  2042.             const Mat kernel0_tm = kernel_tm.channel(p);

  2043.             const Mat kernel1_tm = kernel_tm.channel(p+1);

  2044.             const Mat kernel2_tm = kernel_tm.channel(p+2);

  2045.             const Mat kernel3_tm = kernel_tm.channel(p+3);

  2046. 

  2047.             out0_tm.fill(0.f);

  2048.             out1_tm.fill(0.f);

  2049.             out2_tm.fill(0.f);

  2050.             out3_tm.fill(0.f);

  2051. 

  2052.             int q = 0;

  2053.             for (; q+3<inch; q+=4)

  2054.             {

  2055.                 const float* r0 = bottom_blob_tm.channel(q);

  2056.                 const float* r1 = bottom_blob_tm.channel(q+1);

  2057.                 const float* r2 = bottom_blob_tm.channel(q+2);

  2058.                 const float* r3 = bottom_blob_tm.channel(q+3);

  2059. 

  2060.                 const float* k00 = kernel0_tm.row(q);

  2061.                 const float* k10 = kernel1_tm.row(q);

  2062.                 const float* k20 = kernel2_tm.row(q);

  2063.                 const float* k30 = kernel3_tm.row(q);

  2064. 

  2065.                 float* output0_tm = out0_tm;

  2066.                 float* output1_tm = out1_tm;

  2067.                 float* output2_tm = out2_tm;

  2068.                 float* output3_tm = out3_tm;

  2069. 

  2070.                 // tile

  2071.                 for (int i=0; i<h_tm/8 * w_tm/8; i++)

  2072.                 {

  2073. #if __ARM_NEON

  2074. #if __aarch64__

  2075.                     for (int m=0; m+7<64; m+=8)

  2076.                     {

  2077.                         float32x4_t _output0_tm = vld1q_f32(output0_tm);

  2078.                         float32x4_t _output1_tm = vld1q_f32(output1_tm);

  2079.                         float32x4_t _output2_tm = vld1q_f32(output2_tm);

  2080.                         float32x4_t _output3_tm = vld1q_f32(output3_tm);

  2081. 

  2082.                         float32x4_t _r0 = vld1q_f32(r0);

  2083.                         float32x4_t _r1 = vld1q_f32(r1);

  2084.                         float32x4_t _r2 = vld1q_f32(r2);

  2085.                         float32x4_t _r3 = vld1q_f32(r3);

  2086. 

  2087.                         float32x4_t _k00 = vld1q_f32(k00);

  2088.                         k00 += 64;

  2089.                         float32x4_t _k01 = vld1q_f32(k00);

  2090.                         k00 += 64;

  2091.                         float32x4_t _k02 = vld1q_f32(k00);

  2092.                         k00 += 64;

  2093.                         float32x4_t _k03 = vld1q_f32(k00);

  2094.                         k00 += 64;

  2095. 

  2096.                         k00 -= 64*4;

  2097. 

  2098.                         _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  2099.                         _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  2100.                         _output0_tm = vmlaq_f32(_output0_tm, _r2, _k02);

  2101.                         _output0_tm = vmlaq_f32(_output0_tm, _r3, _k03);

  2102. 

  2103.                         float32x4_t _k10 = vld1q_f32(k10);

  2104.                         k10 += 64;

  2105.                         float32x4_t _k11 = vld1q_f32(k10);

  2106.                         k10 += 64;

  2107.                         float32x4_t _k12 = vld1q_f32(k10);

  2108.                         k10 += 64;

  2109.                         float32x4_t _k13 = vld1q_f32(k10);

  2110.                         k10 += 64;

  2111. 

  2112.                         k10 -= 64*4;

  2113. 

  2114.                         _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  2115.                         _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  2116.                         _output1_tm = vmlaq_f32(_output1_tm, _r2, _k12);

  2117.                         _output1_tm = vmlaq_f32(_output1_tm, _r3, _k13);

  2118. 

  2119.                         float32x4_t _k20 = vld1q_f32(k20);

  2120.                         k20 += 64;

  2121.                         float32x4_t _k21 = vld1q_f32(k20);

  2122.                         k20 += 64;

  2123.                         float32x4_t _k22 = vld1q_f32(k20);

  2124.                         k20 += 64;

  2125.                         float32x4_t _k23 = vld1q_f32(k20);

  2126.                         k20 += 64;

  2127. 

  2128.                         k20 -= 64*4;

  2129. 

  2130.                         _output2_tm = vmlaq_f32(_output2_tm, _r0, _k20);

  2131.                         _output2_tm = vmlaq_f32(_output2_tm, _r1, _k21);

  2132.                         _output2_tm = vmlaq_f32(_output2_tm, _r2, _k22);

  2133.                         _output2_tm = vmlaq_f32(_output2_tm, _r3, _k23);

  2134. 

  2135.                         float32x4_t _k30 = vld1q_f32(k30);

  2136.                         k30 += 64;

  2137.                         float32x4_t _k31 = vld1q_f32(k30);

  2138.                         k30 += 64;

  2139.                         float32x4_t _k32 = vld1q_f32(k30);

  2140.                         k30 += 64;

  2141.                         float32x4_t _k33 = vld1q_f32(k30);

  2142.                         k30 += 64;

  2143. 

  2144.                         k30 -= 64*4;

  2145. 

  2146.                         _output3_tm = vmlaq_f32(_output3_tm, _r0, _k30);

  2147.                         _output3_tm = vmlaq_f32(_output3_tm, _r1, _k31);

  2148.                         _output3_tm = vmlaq_f32(_output3_tm, _r2, _k32);

  2149.                         _output3_tm = vmlaq_f32(_output3_tm, _r3, _k33);

  2150. 

  2151.                         vst1q_f32(output0_tm, _output0_tm);

  2152.                         vst1q_f32(output1_tm, _output1_tm);

  2153.                         vst1q_f32(output2_tm, _output2_tm);

  2154.                         vst1q_f32(output3_tm, _output3_tm);

  2155. 

  2156.                         output0_tm += 4;

  2157.                         output1_tm += 4;

  2158.                         output2_tm += 4;

  2159.                         output3_tm += 4;

  2160. 

  2161.                         r0 += 4;

  2162.                         r1 += 4;

  2163.                         r2 += 4;

  2164.                         r3 += 4;

  2165. 

  2166.                         k00 += 4;

  2167.                         k10 += 4;

  2168.                         k20 += 4;

  2169.                         k30 += 4;

  2170. 

  2171.                         float32x4_t _output0_tmn = vld1q_f32(output0_tm);

  2172.                         float32x4_t _output1_tmn = vld1q_f32(output1_tm);

  2173.                         float32x4_t _output2_tmn = vld1q_f32(output2_tm);

  2174.                         float32x4_t _output3_tmn = vld1q_f32(output3_tm);

  2175. 

  2176.                         float32x4_t _r0n = vld1q_f32(r0);

  2177.                         float32x4_t _r1n = vld1q_f32(r1);

  2178.                         float32x4_t _r2n = vld1q_f32(r2);

  2179.                         float32x4_t _r3n = vld1q_f32(r3);

  2180. 

  2181.                         float32x4_t _k00n = vld1q_f32(k00);

  2182.                         k00 += 64;

  2183.                         float32x4_t _k01n = vld1q_f32(k00);

  2184.                         k00 += 64;

  2185.                         float32x4_t _k02n = vld1q_f32(k00);

  2186.                         k00 += 64;

  2187.                         float32x4_t _k03n = vld1q_f32(k00);

  2188.                         k00 += 64;

  2189. 

  2190.                         k00 -= 64*4;

  2191. 

  2192.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  2193.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  2194.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r2n, _k02n);

  2195.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r3n, _k03n);

  2196. 

  2197.                         float32x4_t _k10n = vld1q_f32(k10);

  2198.                         k10 += 64;

  2199.                         float32x4_t _k11n = vld1q_f32(k10);

  2200.                         k10 += 64;

  2201.                         float32x4_t _k12n = vld1q_f32(k10);

  2202.                         k10 += 64;

  2203.                         float32x4_t _k13n = vld1q_f32(k10);

  2204.                         k10 += 64;

  2205. 

  2206.                         k10 -= 64*4;

  2207. 

  2208.                         _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  2209.                         _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  2210.                         _output1_tmn = vmlaq_f32(_output1_tmn, _r2n, _k12n);

  2211.                         _output1_tmn = vmlaq_f32(_output1_tmn, _r3n, _k13n);

  2212. 

  2213.                         float32x4_t _k20n = vld1q_f32(k20);

  2214.                         k20 += 64;

  2215.                         float32x4_t _k21n = vld1q_f32(k20);

  2216.                         k20 += 64;

  2217.                         float32x4_t _k22n = vld1q_f32(k20);

  2218.                         k20 += 64;

  2219.                         float32x4_t _k23n = vld1q_f32(k20);

  2220.                         k20 += 64;

  2221. 

  2222.                         k20 -= 64*4;

  2223. 

  2224.                         _output2_tmn = vmlaq_f32(_output2_tmn, _r0n, _k20n);

  2225.                         _output2_tmn = vmlaq_f32(_output2_tmn, _r1n, _k21n);

  2226.                         _output2_tmn = vmlaq_f32(_output2_tmn, _r2n, _k22n);

  2227.                         _output2_tmn = vmlaq_f32(_output2_tmn, _r3n, _k23n);

  2228. 

  2229.                         float32x4_t _k30n = vld1q_f32(k30);

  2230.                         k30 += 64;

  2231.                         float32x4_t _k31n = vld1q_f32(k30);

  2232.                         k30 += 64;

  2233.                         float32x4_t _k32n = vld1q_f32(k30);

  2234.                         k30 += 64;

  2235.                         float32x4_t _k33n = vld1q_f32(k30);

  2236.                         k30 += 64;

  2237. 

  2238.                         k30 -= 64*4;

  2239. 

  2240.                         _output3_tmn = vmlaq_f32(_output3_tmn, _r0n, _k30n);

  2241.                         _output3_tmn = vmlaq_f32(_output3_tmn, _r1n, _k31n);

  2242.                         _output3_tmn = vmlaq_f32(_output3_tmn, _r2n, _k32n);

  2243.                         _output3_tmn = vmlaq_f32(_output3_tmn, _r3n, _k33n);

  2244. 

  2245.                         vst1q_f32(output0_tm, _output0_tmn);

  2246.                         vst1q_f32(output1_tm, _output1_tmn);

  2247.                         vst1q_f32(output2_tm, _output2_tmn);

  2248.                         vst1q_f32(output3_tm, _output3_tmn);

  2249. 

  2250.                         output0_tm += 4;

  2251.                         output1_tm += 4;

  2252.                         output2_tm += 4;

  2253.                         output3_tm += 4;

  2254. 

  2255.                         r0 += 4;

  2256.                         r1 += 4;

  2257.                         r2 += 4;

  2258.                         r3 += 4;

  2259. 

  2260.                         k00 += 4;

  2261.                         k10 += 4;

  2262.                         k20 += 4;

  2263.                         k30 += 4;

  2264.                     }

  2265. #else // __aarch64__

  2266.                     asm volatile(

  2267.                         "mov        r4, #8              \n"

  2268. 

  2269.                         "pld        [%0, #256]          \n"

  2270.                         "vld1.f32   {d16-d19}, [%0 :128]\n"//q8 q9 = _output0_tm

  2271. 

  2272.                         "0:                             \n"

  2273. 

  2274.                         "pld        [%4, #256]          \n"

  2275.                         "vld1.f32   {d0-d3}, [%4 :128]! \n"//q0 q1 = _r0

  2276. 

  2277.                         "pld        [%8, #256]          \n"

  2278.                         "vld1.f32   {d20-d23}, [%8 :128]\n"//q10 q11 = _k00

  2279.                         "add        %8, %8, #256        \n"

  2280. 

  2281.                         "vmla.f32   q8, q0, q10         \n"

  2282.                         "vmla.f32   q9, q1, q11         \n"

  2283. 

  2284.                         "pld        [%1, #256]          \n"

  2285.                         "vld1.f32   {d24-d27}, [%1 :128]\n"//q12 q13 = _output1_tm

  2286. 

  2287.                         "pld        [%9, #256]          \n"

  2288.                         "vld1.f32   {d28-d31}, [%9 :128]\n"//q14 q15 = _k10

  2289.                         "add        %9, %9, #256        \n"

  2290. 

  2291.                         "vmla.f32   q12, q0, q14        \n"

  2292.                         "vmla.f32   q13, q1, q15        \n"

  2293. 

  2294.                         "pld        [%5, #256]          \n"

  2295.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"//q2 q3 = _r1

  2296. 

  2297.                         "pld        [%8, #256]          \n"

  2298.                         "vld1.f32   {d20-d23}, [%8 :128]\n"//q10 q11 = _k01

  2299.                         "add        %8, %8, #256        \n"

  2300. 

  2301.                         "vmla.f32   q8, q2, q10         \n"

  2302.                         "vmla.f32   q9, q3, q11         \n"

  2303. 

  2304.                         "pld        [%9, #256]          \n"

  2305.                         "vld1.f32   {d28-d31}, [%9 :128]\n"//q14 q15 = _k11

  2306.                         "add        %9, %9, #256        \n"

  2307. 

  2308.                         "vmla.f32   q12, q2, q14        \n"

  2309.                         "vmla.f32   q13, q3, q15        \n"

  2310. 

  2311.                         "pld        [%6, #256]          \n"

  2312.                         "vld1.f32   {d8-d11}, [%6 :128]!\n"//q4 q5 = _r2

  2313. 

  2314.                         "pld        [%8, #256]          \n"

  2315.                         "vld1.f32   {d20-d23}, [%8 :128]\n"//q10 q11 = _k02

  2316.                         "add        %8, %8, #256        \n"

  2317. 

  2318.                         "vmla.f32   q8, q4, q10         \n"

  2319.                         "vmla.f32   q9, q5, q11         \n"

  2320. 

  2321.                         "pld        [%9, #256]          \n"

  2322.                         "vld1.f32   {d28-d31}, [%9 :128]\n"//q14 q15 = _k12

  2323.                         "add        %9, %9, #256        \n"

  2324. 

  2325.                         "vmla.f32   q12, q4, q14        \n"

  2326.                         "vmla.f32   q13, q5, q15        \n"

  2327. 

  2328.                         "pld        [%7, #256]          \n"

  2329.                         "vld1.f32   {d12-d15}, [%7 :128]!\n"//q6 q7 = _r3

  2330. 

  2331.                         "pld        [%8, #256]          \n"

  2332.                         "vld1.f32   {d20-d23}, [%8 :128]\n"//q10 q11 = _k03

  2333.                         "sub        %8, %8, #736        \n"

  2334. 

  2335.                         "vmla.f32   q8, q6, q10         \n"

  2336.                         "vmla.f32   q9, q7, q11         \n"

  2337. 

  2338.                         "pld        [%9, #256]          \n"

  2339.                         "vld1.f32   {d28-d31}, [%9 :128]\n"//q14 q15 = _k13

  2340.                         "sub        %9, %9, #736        \n"

  2341. 

  2342.                         "vmla.f32   q12, q6, q14        \n"

  2343.                         "vmla.f32   q13, q7, q15        \n"

  2344. 

  2345.                         "vst1.f32   {d16-d19}, [%0 :128]!\n"

  2346. 

  2347.                         "pld        [%2, #256]          \n"

  2348.                         "vld1.f32   {d16-d19}, [%2 :128]\n"//q8 q9 = _output2_tm

  2349. 

  2350.                         "pld        [%10, #256]         \n"

  2351.                         "vld1.f32   {d20-d23}, [%10 :128]\n"//q10 q11 = _k20

  2352.                         "add        %10, %10, #256      \n"

  2353. 

  2354.                         "vmla.f32   q8, q0, q10         \n"

  2355.                         "vmla.f32   q9, q1, q11         \n"

  2356. 

  2357.                         "vst1.f32   {d24-d27}, [%1 :128]!\n"

  2358. 

  2359.                         "pld        [%3, #256]          \n"

  2360.                         "vld1.f32   {d24-d27}, [%3 :128]\n"//q12 q13 = _output3_tm

  2361. 

  2362.                         "pld        [%11, #256]         \n"

  2363.                         "vld1.f32   {d28-d31}, [%11 :128]\n"//q14 q15 = _k30

  2364.                         "add        %11, %11, #256      \n"

  2365. 

  2366.                         "vmla.f32   q12, q0, q14        \n"

  2367.                         "vmla.f32   q13, q1, q15        \n"

  2368. 

  2369.                         "pld        [%10, #256]         \n"

  2370.                         "vld1.f32   {d20-d23}, [%10 :128]\n"//q10 q11 = _k21

  2371.                         "add        %10, %10, #256      \n"

  2372. 

  2373.                         "vmla.f32   q8, q2, q10         \n"

  2374.                         "vmla.f32   q9, q3, q11         \n"

  2375. 

  2376.                         "pld        [%11, #256]         \n"

  2377.                         "vld1.f32   {d28-d31}, [%11 :128]\n"//q14 q15 = _k31

  2378.                         "add        %11, %11, #256      \n"

  2379. 

  2380.                         "vmla.f32   q12, q2, q14        \n"

  2381.                         "vmla.f32   q13, q3, q15        \n"

  2382. 

  2383.                         "pld        [%10, #256]         \n"

  2384.                         "vld1.f32   {d20-d23}, [%10 :128]\n"//q10 q11 = _k22

  2385.                         "add        %10, %10, #256      \n"

  2386. 

  2387.                         "vmla.f32   q8, q4, q10         \n"

  2388.                         "vmla.f32   q9, q5, q11         \n"

  2389. 

  2390.                         "pld        [%11, #256]         \n"

  2391.                         "vld1.f32   {d28-d31}, [%11 :128]\n"//q14 q15 = _k32

  2392.                         "add        %11, %11, #256      \n"

  2393. 

  2394.                         "vmla.f32   q12, q4, q14        \n"

  2395.                         "vmla.f32   q13, q5, q15        \n"

  2396. 

  2397.                         "pld        [%10, #256]         \n"

  2398.                         "vld1.f32   {d20-d23}, [%10 :128]\n"//q10 q11 = _k23

  2399.                         "sub        %10, %10, #736      \n"

  2400. 

  2401.                         "vmla.f32   q8, q6, q10         \n"

  2402.                         "vmla.f32   q9, q7, q11         \n"

  2403. 

  2404.                         "pld        [%11, #256]         \n"

  2405.                         "vld1.f32   {d28-d31}, [%11 :128]\n"//q14 q15 = _k33

  2406.                         "sub        %11, %11, #736      \n"

  2407. 

  2408.                         "vmla.f32   q12, q6, q14        \n"

  2409.                         "vmla.f32   q13, q7, q15        \n"

  2410. 

  2411.                         "vst1.f32   {d16-d19}, [%2 :128]!\n"

  2412. 

  2413.                         "pld        [%0, #256]          \n"

  2414.                         "vld1.f32   {d16-d19}, [%0 :128]\n"//q8 q9 = _output0_tm

  2415. 

  2416.                         "subs       r4, r4, #1          \n"

  2417. 

  2418.                         "vst1.f32   {d24-d27}, [%3 :128]!\n"

  2419. 

  2420.                         "bne        0b                  \n"

  2421. 

  2422.                         : "=r"(output0_tm), // %0

  2423.                           "=r"(output1_tm), // %1

  2424.                           "=r"(output2_tm), // %2

  2425.                           "=r"(output3_tm), // %3

  2426.                           "=r"(r0),         // %4

  2427.                           "=r"(r1),         // %5

  2428.                           "=r"(r2),         // %6

  2429.                           "=r"(r3),         // %7

  2430.                           "=r"(k00),        // %8

  2431.                           "=r"(k10),        // %9

  2432.                           "=r"(k20),        // %10

  2433.                           "=r"(k30)         // %11

  2434.                         : "0"(output0_tm),

  2435.                           "1"(output1_tm),

  2436.                           "2"(output2_tm),

  2437.                           "3"(output3_tm),

  2438.                           "4"(r0),

  2439.                           "5"(r1),

  2440.                           "6"(r2),

  2441.                           "7"(r3),

  2442.                           "8"(k00),

  2443.                           "9"(k10),

  2444.                           "10"(k20),

  2445.                           "11"(k30)

  2446.                         : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  2447.                     );

  2448. #endif // __aarch64__

  2449. 

  2450.                     k00 -= 64;

  2451.                     k10 -= 64;

  2452.                     k20 -= 64;

  2453.                     k30 -= 64;

  2454. #else

  2455.                     for (int m=0; m<64; m++)

  2456.                     {

  2457.                         output0_tm[m] += r0[m] * k00[m];

  2458.                         k00 += 64;

  2459.                         output0_tm[m] += r1[m] * k00[m];

  2460.                         k00 += 64;

  2461.                         output0_tm[m] += r2[m] * k00[m];

  2462.                         k00 += 64;

  2463.                         output0_tm[m] += r3[m] * k00[m];

  2464.                         k00 += 64;

  2465. 

  2466.                         k00 -= 64 * 4;

  2467. 

  2468.                         output1_tm[m] += r0[m] * k10[m];

  2469.                         k10 += 64;

  2470.                         output1_tm[m] += r1[m] * k10[m];

  2471.                         k10 += 64;

  2472.                         output1_tm[m] += r2[m] * k10[m];

  2473.                         k10 += 64;

  2474.                         output1_tm[m] += r3[m] * k10[m];

  2475.                         k10 += 64;

  2476. 

  2477.                         k10 -= 64 * 4;

  2478. 

  2479.                         output2_tm[m] += r0[m] * k20[m];

  2480.                         k20 += 64;

  2481.                         output2_tm[m] += r1[m] * k20[m];

  2482.                         k20 += 64;

  2483.                         output2_tm[m] += r2[m] * k20[m];

  2484.                         k20 += 64;

  2485.                         output2_tm[m] += r3[m] * k20[m];

  2486.                         k20 += 64;

  2487. 

  2488.                         k20 -= 64 * 4;

  2489. 

  2490.                         output3_tm[m] += r0[m] * k30[m];

  2491.                         k30 += 64;

  2492.                         output3_tm[m] += r1[m] * k30[m];

  2493.                         k30 += 64;

  2494.                         output3_tm[m] += r2[m] * k30[m];

  2495.                         k30 += 64;

  2496.                         output3_tm[m] += r3[m] * k30[m];

  2497.                         k30 += 64;

  2498. 

  2499.                         k30 -= 64 * 4;

  2500.                     }

  2501. 

  2502.                     r0 += 64;

  2503.                     r1 += 64;

  2504.                     r2 += 64;

  2505.                     r3 += 64;

  2506.                     output0_tm += 64;

  2507.                     output1_tm += 64;

  2508.                     output2_tm += 64;

  2509.                     output3_tm += 64;

  2510. #endif // __ARM_NEON

  2511.                 }

  2512.             }

  2513. 

  2514.             for (; q<inch; q++)

  2515.             {

  2516.                 const float* r0 = bottom_blob_tm.channel(q);

  2517. 

  2518.                 const float* k0 = kernel0_tm.row(q);

  2519.                 const float* k1 = kernel1_tm.row(q);

  2520.                 const float* k2 = kernel2_tm.row(q);

  2521.                 const float* k3 = kernel3_tm.row(q);

  2522. 

  2523.                 float* output0_tm = out0_tm;

  2524.                 float* output1_tm = out1_tm;

  2525.                 float* output2_tm = out2_tm;

  2526.                 float* output3_tm = out3_tm;

  2527. 

  2528.                 // tile

  2529.                 for (int i=0; i<h_tm/8 * w_tm/8; i++)

  2530.                 {

  2531.                     // TODO neon optimize

  2532.                     for (int m=0; m<64; m++)

  2533.                     {

  2534.                         output0_tm[m] += r0[m] * k0[m];

  2535.                         output1_tm[m] += r0[m] * k1[m];

  2536.                         output2_tm[m] += r0[m] * k2[m];

  2537.                         output3_tm[m] += r0[m] * k3[m];

  2538.                     }

  2539. 

  2540.                     r0 += 64;

  2541.                     output0_tm += 64;

  2542.                     output1_tm += 64;

  2543.                     output2_tm += 64;

  2544.                     output3_tm += 64;

  2545.                 }

  2546. 

  2547.             }

  2548.         }

  2549. 

  2550.         #pragma omp parallel for

  2551.         for (int p=remain_outch_start; p<outch; p++)

  2552.         {

  2553.             Mat out0_tm = top_blob_tm.channel(p);

  2554.             const Mat kernel0_tm = kernel_tm.channel(p);

  2555. 

  2556.             out0_tm.fill(0.f);

  2557. 

  2558.             int q = 0;

  2559.             for (; q+3<inch; q+=4)

  2560.             {

  2561.                 const float* r0 = bottom_blob_tm.channel(q);

  2562.                 const float* r1 = bottom_blob_tm.channel(q+1);

  2563.                 const float* r2 = bottom_blob_tm.channel(q+2);

  2564.                 const float* r3 = bottom_blob_tm.channel(q+3);

  2565. 

  2566.                 const float* k0 = kernel0_tm.row(q);

  2567.                 const float* k1 = kernel0_tm.row(q+1);

  2568.                 const float* k2 = kernel0_tm.row(q+2);

  2569.                 const float* k3 = kernel0_tm.row(q+3);

  2570. 

  2571.                 float* output0_tm = out0_tm;

  2572. 

  2573.                 // tile

  2574.                 for (int i=0; i<h_tm/8 * w_tm/8; i++)

  2575.                 {

  2576. #if __ARM_NEON

  2577. #if __aarch64__

  2578.                     for (int m=0; m+7<64; m+=8)

  2579.                     {

  2580.                         float32x4_t _output0_tm = vld1q_f32(output0_tm);

  2581. 

  2582.                         float32x4_t _r0 = vld1q_f32(r0);

  2583.                         float32x4_t _r1 = vld1q_f32(r1);

  2584.                         float32x4_t _r2 = vld1q_f32(r2);

  2585.                         float32x4_t _r3 = vld1q_f32(r3);

  2586. 

  2587.                         float32x4_t _k0 = vld1q_f32(k0);

  2588.                         float32x4_t _k1 = vld1q_f32(k1);

  2589.                         float32x4_t _k2 = vld1q_f32(k2);

  2590.                         float32x4_t _k3 = vld1q_f32(k3);

  2591. 

  2592.                         _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  2593.                         _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  2594.                         _output0_tm = vmlaq_f32(_output0_tm, _r2, _k2);

  2595.                         _output0_tm = vmlaq_f32(_output0_tm, _r3, _k3);

  2596. 

  2597.                         vst1q_f32(output0_tm, _output0_tm);

  2598. 

  2599.                         output0_tm += 4;

  2600. 

  2601.                         r0 += 4;

  2602.                         r1 += 4;

  2603.                         r2 += 4;

  2604.                         r3 += 4;

  2605. 

  2606.                         k0 += 4;

  2607.                         k1 += 4;

  2608.                         k2 += 4;

  2609.                         k3 += 4;

  2610. 

  2611.                         float32x4_t _output0_tmn = vld1q_f32(output0_tm);

  2612. 

  2613.                         float32x4_t _r0n = vld1q_f32(r0);

  2614.                         float32x4_t _r1n = vld1q_f32(r1);

  2615.                         float32x4_t _r2n = vld1q_f32(r2);

  2616.                         float32x4_t _r3n = vld1q_f32(r3);

  2617. 

  2618.                         float32x4_t _k0n = vld1q_f32(k0);

  2619.                         float32x4_t _k1n = vld1q_f32(k1);

  2620.                         float32x4_t _k2n = vld1q_f32(k2);

  2621.                         float32x4_t _k3n = vld1q_f32(k3);

  2622. 

  2623.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  2624.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  2625.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r2n, _k2n);

  2626.                         _output0_tmn = vmlaq_f32(_output0_tmn, _r3n, _k3n);

  2627. 

  2628.                         vst1q_f32(output0_tm, _output0_tmn);

  2629. 

  2630.                         output0_tm += 4;

  2631. 

  2632.                         r0 += 4;

  2633.                         r1 += 4;

  2634.                         r2 += 4;

  2635.                         r3 += 4;

  2636. 

  2637.                         k0 += 4;

  2638.                         k1 += 4;

  2639.                         k2 += 4;

  2640.                         k3 += 4;

  2641.                     }

  2642. #else

  2643.                     asm volatile(

  2644.                         "pld        [%1, #256]          \n"

  2645.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2646. 

  2647.                         "mov        r4, %0              \n"

  2648. 

  2649.                         "pld        [%0, #256]          \n"

  2650.                         "vld1.f32   {d24-d27}, [%0 :128]!\n"//q12 q13 = output0_tm

  2651. 

  2652.                         "pld        [%5, #256]          \n"

  2653.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2654. 

  2655.                         "vmla.f32   q12, q0, q2         \n"

  2656. 

  2657.                         "pld        [%2, #256]          \n"

  2658.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2659.                         "vmla.f32   q13, q1, q3         \n"

  2660. 

  2661.                         "pld        [%6, #256]          \n"

  2662.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2663. 

  2664.                         "vmla.f32   q12, q8, q10        \n"

  2665. 

  2666.                         "pld        [%3, #256]          \n"

  2667.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2668.                         "vmla.f32   q13, q9, q11        \n"

  2669. 

  2670.                         "pld        [%7, #256]          \n"

  2671.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2672. 

  2673.                         "vmla.f32   q12, q0, q2         \n"

  2674. 

  2675.                         "pld        [%4, #256]          \n"

  2676.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2677.                         "vmla.f32   q13, q1, q3         \n"

  2678. 

  2679.                         "pld        [%8, #256]          \n"

  2680.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2681. 

  2682.                         "vmla.f32   q12, q8, q10        \n"

  2683. 

  2684.                         "pld        [%0, #256]          \n"

  2685.                         "vld1.f32   {d28-d31}, [%0 :128]!\n"//q14 q15 = output0_tm

  2686.                         "vmla.f32   q13, q9, q11        \n"

  2687. 

  2688.                         "pld        [%1, #256]          \n"

  2689.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2690. 

  2691.                         "pld        [%5, #256]          \n"

  2692.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2693. 

  2694.                         "vmla.f32   q14, q0, q2         \n"

  2695. 

  2696.                         "vst1.f32   {d24-d27}, [r4 :128]!\n"

  2697. 

  2698.                         "pld        [%2, #256]          \n"

  2699.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2700.                         "vmla.f32   q15, q1, q3         \n"

  2701. 

  2702.                         "pld        [%6, #256]          \n"

  2703.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2704. 

  2705.                         "vmla.f32   q14, q8, q10        \n"

  2706. 

  2707.                         "pld        [%3, #256]          \n"

  2708.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2709.                         "vmla.f32   q15, q9, q11        \n"

  2710. 

  2711.                         "pld        [%7, #256]          \n"

  2712.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2713. 

  2714.                         "vmla.f32   q14, q0, q2         \n"

  2715. 

  2716.                         "pld        [%4, #256]          \n"

  2717.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2718.                         "vmla.f32   q15, q1, q3         \n"

  2719. 

  2720.                         "pld        [%8, #256]          \n"

  2721.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2722. 

  2723.                         "vmla.f32   q14, q8, q10        \n"

  2724. 

  2725.                         "pld        [%0, #256]          \n"

  2726.                         "vld1.f32   {d24-d27}, [%0 :128]!\n"//q12 q13 = output0_tm

  2727.                         "vmla.f32   q15, q9, q11        \n"

  2728. 

  2729.                         "pld        [%1, #256]          \n"

  2730.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2731. 

  2732.                         "pld        [%5, #256]          \n"

  2733.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2734. 

  2735.                         "vmla.f32   q12, q0, q2         \n"

  2736. 

  2737.                         "vst1.f32   {d28-d31}, [r4 :128]!\n"

  2738. 

  2739.                         "pld        [%2, #256]          \n"

  2740.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2741.                         "vmla.f32   q13, q1, q3         \n"

  2742. 

  2743.                         "pld        [%6, #256]          \n"

  2744.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2745. 

  2746.                         "vmla.f32   q12, q8, q10        \n"

  2747. 

  2748.                         "pld        [%3, #256]          \n"

  2749.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2750.                         "vmla.f32   q13, q9, q11        \n"

  2751. 

  2752.                         "pld        [%7, #256]          \n"

  2753.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2754. 

  2755.                         "vmla.f32   q12, q0, q2         \n"

  2756. 

  2757.                         "pld        [%4, #256]          \n"

  2758.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2759.                         "vmla.f32   q13, q1, q3         \n"

  2760. 

  2761.                         "pld        [%8, #256]          \n"

  2762.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2763. 

  2764.                         "vmla.f32   q12, q8, q10        \n"

  2765. 

  2766.                         "pld        [%0, #256]          \n"

  2767.                         "vld1.f32   {d28-d31}, [%0 :128]!\n"//q14 q15 = output0_tm

  2768.                         "vmla.f32   q13, q9, q11        \n"

  2769. 

  2770.                         "pld        [%1, #256]          \n"

  2771.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2772. 

  2773.                         "pld        [%5, #256]          \n"

  2774.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2775. 

  2776.                         "vmla.f32   q14, q0, q2         \n"

  2777. 

  2778.                         "vst1.f32   {d24-d27}, [r4 :128]!\n"

  2779. 

  2780.                         "pld        [%2, #256]          \n"

  2781.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2782.                         "vmla.f32   q15, q1, q3         \n"

  2783. 

  2784.                         "pld        [%6, #256]          \n"

  2785.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2786. 

  2787.                         "vmla.f32   q14, q8, q10        \n"

  2788. 

  2789.                         "pld        [%3, #256]          \n"

  2790.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2791.                         "vmla.f32   q15, q9, q11        \n"

  2792. 

  2793.                         "pld        [%7, #256]          \n"

  2794.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2795. 

  2796.                         "vmla.f32   q14, q0, q2         \n"

  2797. 

  2798.                         "pld        [%4, #256]          \n"

  2799.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2800.                         "vmla.f32   q15, q1, q3         \n"

  2801. 

  2802.                         "pld        [%8, #256]          \n"

  2803.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2804. 

  2805.                         "vmla.f32   q14, q8, q10        \n"

  2806. 

  2807.                         "pld        [%0, #256]          \n"

  2808.                         "vld1.f32   {d24-d27}, [%0 :128]!\n"//q12 q13 = output0_tm

  2809.                         "vmla.f32   q15, q9, q11        \n"

  2810. 

  2811.                         "pld        [%1, #256]          \n"

  2812.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2813. 

  2814.                         "pld        [%5, #256]          \n"

  2815.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2816. 

  2817.                         "vmla.f32   q12, q0, q2         \n"

  2818. 

  2819.                         "vst1.f32   {d28-d31}, [r4 :128]!\n"

  2820. 

  2821.                         "pld        [%2, #256]          \n"

  2822.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2823.                         "vmla.f32   q13, q1, q3         \n"

  2824. 

  2825.                         "pld        [%6, #256]          \n"

  2826.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2827. 

  2828.                         "vmla.f32   q12, q8, q10        \n"

  2829. 

  2830.                         "pld        [%3, #256]          \n"

  2831.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2832.                         "vmla.f32   q13, q9, q11        \n"

  2833. 

  2834.                         "pld        [%7, #256]          \n"

  2835.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2836. 

  2837.                         "vmla.f32   q12, q0, q2         \n"

  2838. 

  2839.                         "pld        [%4, #256]          \n"

  2840.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2841.                         "vmla.f32   q13, q1, q3         \n"

  2842. 

  2843.                         "pld        [%8, #256]          \n"

  2844.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2845. 

  2846.                         "vmla.f32   q12, q8, q10        \n"

  2847. 

  2848.                         "pld        [%0, #256]          \n"

  2849.                         "vld1.f32   {d28-d31}, [%0 :128]!\n"//q14 q15 = output0_tm

  2850.                         "vmla.f32   q13, q9, q11        \n"

  2851. 

  2852.                         "pld        [%1, #256]          \n"

  2853.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2854. 

  2855.                         "pld        [%5, #256]          \n"

  2856.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2857. 

  2858.                         "vmla.f32   q14, q0, q2         \n"

  2859. 

  2860.                         "vst1.f32   {d24-d27}, [r4 :128]!\n"

  2861. 

  2862.                         "pld        [%2, #256]          \n"

  2863.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2864.                         "vmla.f32   q15, q1, q3         \n"

  2865. 

  2866.                         "pld        [%6, #256]          \n"

  2867.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2868. 

  2869.                         "vmla.f32   q14, q8, q10        \n"

  2870. 

  2871.                         "pld        [%3, #256]          \n"

  2872.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2873.                         "vmla.f32   q15, q9, q11        \n"

  2874. 

  2875.                         "pld        [%7, #256]          \n"

  2876.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2877. 

  2878.                         "vmla.f32   q14, q0, q2         \n"

  2879. 

  2880.                         "pld        [%4, #256]          \n"

  2881.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2882.                         "vmla.f32   q15, q1, q3         \n"

  2883. 

  2884.                         "pld        [%8, #256]          \n"

  2885.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2886. 

  2887.                         "vmla.f32   q14, q8, q10        \n"

  2888. 

  2889.                         "pld        [%0, #256]          \n"

  2890.                         "vld1.f32   {d24-d27}, [%0 :128]!\n"//q12 q13 = output0_tm

  2891.                         "vmla.f32   q15, q9, q11        \n"

  2892. 

  2893.                         "pld        [%1, #256]          \n"

  2894.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2895. 

  2896.                         "pld        [%5, #256]          \n"

  2897.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2898. 

  2899.                         "vmla.f32   q12, q0, q2         \n"

  2900. 

  2901.                         "vst1.f32   {d28-d31}, [r4 :128]!\n"

  2902. 

  2903.                         "pld        [%2, #256]          \n"

  2904.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2905.                         "vmla.f32   q13, q1, q3         \n"

  2906. 

  2907.                         "pld        [%6, #256]          \n"

  2908.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2909. 

  2910.                         "vmla.f32   q12, q8, q10        \n"

  2911. 

  2912.                         "pld        [%3, #256]          \n"

  2913.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2914.                         "vmla.f32   q13, q9, q11        \n"

  2915. 

  2916.                         "pld        [%7, #256]          \n"

  2917.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2918. 

  2919.                         "vmla.f32   q12, q0, q2         \n"

  2920. 

  2921.                         "pld        [%4, #256]          \n"

  2922.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2923.                         "vmla.f32   q13, q1, q3         \n"

  2924. 

  2925.                         "pld        [%8, #256]          \n"

  2926.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2927. 

  2928.                         "vmla.f32   q12, q8, q10        \n"

  2929. 

  2930.                         "pld        [%0, #256]          \n"

  2931.                         "vld1.f32   {d28-d31}, [%0 :128]!\n"//q14 q15 = output0_tm

  2932.                         "vmla.f32   q13, q9, q11        \n"

  2933. 

  2934.                         "pld        [%1, #256]          \n"

  2935.                         "vld1.f32   {d0-d3}, [%1 :128]! \n"

  2936. 

  2937.                         "pld        [%5, #256]          \n"

  2938.                         "vld1.f32   {d4-d7}, [%5 :128]! \n"

  2939. 

  2940.                         "vmla.f32   q14, q0, q2         \n"

  2941. 

  2942.                         "vst1.f32   {d24-d27}, [r4 :128]!\n"

  2943. 

  2944.                         "pld        [%2, #256]          \n"

  2945.                         "vld1.f32   {d16-d19}, [%2 :128]!\n"

  2946.                         "vmla.f32   q15, q1, q3         \n"

  2947. 

  2948.                         "pld        [%6, #256]          \n"

  2949.                         "vld1.f32   {d20-d23}, [%6 :128]!\n"

  2950. 

  2951.                         "vmla.f32   q14, q8, q10        \n"

  2952. 

  2953.                         "pld        [%3, #256]          \n"

  2954.                         "vld1.f32   {d0-d3}, [%3 :128]! \n"

  2955.                         "vmla.f32   q15, q9, q11        \n"

  2956. 

  2957.                         "pld        [%7, #256]          \n"

  2958.                         "vld1.f32   {d4-d7}, [%7 :128]! \n"

  2959. 

  2960.                         "vmla.f32   q14, q0, q2         \n"

  2961. 

  2962.                         "pld        [%4, #256]          \n"

  2963.                         "vld1.f32   {d16-d19}, [%4 :128]!\n"

  2964.                         "vmla.f32   q15, q1, q3         \n"

  2965. 

  2966.                         "pld        [%8, #256]          \n"

  2967.                         "vld1.f32   {d20-d23}, [%8 :128]!\n"

  2968. 

  2969.                         "vmla.f32   q14, q8, q10        \n"

  2970.                         "vmla.f32   q15, q9, q11        \n"

  2971. 

  2972.                         "vst1.f32   {d28-d31}, [r4 :128]!\n"

  2973. 

  2974.                         : "=r"(output0_tm), // %0

  2975.                           "=r"(r0),         // %1

  2976.                           "=r"(r1),         // %2

  2977.                           "=r"(r2),         // %3

  2978.                           "=r"(r3),         // %4

  2979.                           "=r"(k0),         // %5

  2980.                           "=r"(k1),         // %6

  2981.                           "=r"(k2),         // %7

  2982.                           "=r"(k3)          // %8

  2983.                         : "0"(output0_tm),

  2984.                           "1"(r0),

  2985.                           "2"(r1),

  2986.                           "3"(r2),

  2987.                           "4"(r3),

  2988.                           "5"(k0),

  2989.                           "6"(k1),

  2990.                           "7"(k2),

  2991.                           "8"(k3)

  2992.                         : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  2993.                     );

  2994. #endif // __aarch64__

  2995. 

  2996.                     k0 -= 64;

  2997.                     k1 -= 64;

  2998.                     k2 -= 64;

  2999.                     k3 -= 64;

  3000. #else

  3001.                     for (int m=0; m<64; m++)

  3002.                     {

  3003.                         output0_tm[m] += r0[m] * k0[m];

  3004.                         output0_tm[m] += r1[m] * k1[m];

  3005.                         output0_tm[m] += r2[m] * k2[m];

  3006.                         output0_tm[m] += r3[m] * k3[m];

  3007.                     }

  3008. 

  3009.                     r0 += 64;

  3010.                     r1 += 64;

  3011.                     r2 += 64;

  3012.                     r3 += 64;

  3013.                     output0_tm += 64;

  3014. #endif // __ARM_NEON

  3015.                 }

  3016.             }

  3017. 

  3018.             for (; q<inch; q++)

  3019.             {

  3020.                 const float* r0 = bottom_blob_tm.channel(q);

  3021. 

  3022.                 const float* k0 = kernel0_tm.row(q);

  3023. 

  3024.                 float* output0_tm = out0_tm;

  3025. 

  3026.                 // tile

  3027.                 for (int i=0; i<h_tm/8 * w_tm/8; i++)

  3028.                 {

  3029.                     // TODO neon optimize

  3030.                     for (int m=0; m<64; m++)

  3031.                     {

  3032.                         output0_tm[m] += r0[m] * k0[m];

  3033.                     }

  3034. 

  3035.                     r0 += 64;

  3036.                     output0_tm += 64;

  3037.                 }

  3038. 

  3039.             }

  3040.         }

  3041.     }

  3042.     bottom_blob_tm = Mat();

  3043.     // END dot

  3044. 

  3045.     // BEGIN transform output

  3046.     Mat top_blob_bordered;

  3047.     top_blob_bordered.create(outw, outh, outch);

  3048.     {

  3049. //         const float otm[6][8] = {

  3050. //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  3051. //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  3052. //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  3053. //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  3054. //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  3055. //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  3056. //         };

  3057. 

  3058.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  3059.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  3060.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  3061.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  3062.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  3063.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  3064. 

  3065.         int w_tm = outw / 6 * 8;

  3066. 

  3067.         #pragma omp parallel for

  3068.         for (int p = 0; p<outch; p++)

  3069.         {

  3070.             const Mat out0_tm = top_blob_tm.channel(p);

  3071.             Mat out0 = top_blob_bordered.channel(p);

  3072. 

  3073.             const float bias0 = bias ? bias[p] : 0.f;

  3074. 

  3075.             float tmp[6][8];

  3076. 

  3077.             // tile

  3078.             for (int i=0; i<outh/6; i++)

  3079.             {

  3080.                 for (int j=0; j<outw/6; j++)

  3081.                 {

  3082.                     const float* output0_tm = out0_tm.row(i * w_tm/8 + j);

  3083.                     float* output0 = out0.row(i * 6) + j * 6;

  3084. 

  3085.                     // TODO neon optimize

  3086.                     for (int m=0; m<8; m++)

  3087.                     {

  3088.                         float tmp024a = output0_tm[1] + output0_tm[2];

  3089.                         float tmp135a = output0_tm[1] - output0_tm[2];

  3090. 

  3091.                         float tmp024b = output0_tm[3] + output0_tm[4];

  3092.                         float tmp135b = output0_tm[3] - output0_tm[4];

  3093. 

  3094.                         float tmp024c = output0_tm[5] + output0_tm[6];

  3095.                         float tmp135c = output0_tm[5] - output0_tm[6];

  3096. 

  3097.                         tmp[0][m] = output0_tm[0] + tmp024a + tmp024b + tmp024c * 32;

  3098.                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  3099.                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  3100. 

  3101.                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  3102.                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  3103.                         tmp[5][m] = output0_tm[7] + tmp135a + tmp135b * 32 + tmp135c;

  3104. 

  3105.                         output0_tm += 8;

  3106.                     }

  3107. 

  3108.                     for (int m=0; m<6; m++)

  3109.                     {

  3110.                         const float* tmp0 = tmp[m];

  3111. 

  3112.                         float tmp024a = tmp0[1] + tmp0[2];

  3113.                         float tmp135a = tmp0[1] - tmp0[2];

  3114. 

  3115.                         float tmp024b = tmp0[3] + tmp0[4];

  3116.                         float tmp135b = tmp0[3] - tmp0[4];

  3117. 

  3118.                         float tmp024c = tmp0[5] + tmp0[6];

  3119.                         float tmp135c = tmp0[5] - tmp0[6];

  3120. 

  3121.                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  3122.                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  3123.                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  3124. 

  3125.                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  3126.                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  3127.                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  3128. 

  3129.                         output0 += outw;

  3130.                     }

  3131.                 }

  3132.             }

  3133.         }

  3134.     }

  3135.     // END transform output

  3136. 

  3137.     // cut result pad

  3138.     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w);

  3139. }

  3140. 

  3141. static void conv3x3s1_winograd64_neon2(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias)

  3142. {

  3143.     int w = bottom_blob.w;

  3144.     int h = bottom_blob.h;

  3145.     int inch = bottom_blob.c;

  3146. 

  3147.     int outw = top_blob.w;

  3148.     int outh = top_blob.h;

  3149.     int outch = top_blob.c;

  3150. 

  3151.     // pad to 6n+2

  3152.     Mat bottom_blob_bordered = bottom_blob;

  3153. 

  3154.     outw = (outw + 5) / 6 * 6;

  3155.     outh = (outh + 5) / 6 * 6;

  3156. 

  3157.     w = outw + 2;

  3158.     h = outh + 2;

  3159.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f);

  3160. 

  3161.     const float* bias = _bias;

  3162. 

  3163.     // BEGIN transform input

  3164.     Mat bottom_blob_tm;

  3165.     {

  3166.         int w_tm = outw / 6 * 8;

  3167.         int h_tm = outh / 6 * 8;

  3168.         bottom_blob_tm.create(2*8, 4 * w_tm/8 * h_tm/8, inch);

  3169.         const int tiles = w_tm/8 * h_tm/8;

  3170. 

  3171. //         const float itm[8][8] = {

  3172. //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  3173. //

  3174. //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  3175. //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  3176. //

  3177. //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  3178. //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  3179. //

  3180. //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  3181. //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  3182. //

  3183. //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  3184. //         };

  3185. 

  3186.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  3187.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  3188. 

  3189.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  3190.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  3191. 

  3192.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  3193.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  3194. 

  3195.         // reuse r04 * 1.25

  3196.         // reuse r03 * 2.5

  3197.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  3198.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  3199. 

  3200.         #pragma omp parallel for

  3201.         for (int q = 0; q<inch; q++)

  3202.         {

  3203.             const Mat img0 = bottom_blob_bordered.channel(q);

  3204.             Mat img0_tm = bottom_blob_tm.channel(q);

  3205. 

  3206.             float tmp[8][8];

  3207. 

  3208.             // tile

  3209.             for (int i=0; i<h_tm/8; i++)

  3210.             {

  3211.                 for (int j=0; j<w_tm/8; j++)

  3212.                 {

  3213.                     const float* r0 = img0.row(i * 6) + j * 6;

  3214.                     float* r0_tm01 = img0_tm.row(i * w_tm/8 + j);

  3215.                     float* r0_tm23 = img0_tm.row(tiles + i * w_tm/8 + j);

  3216.                     float* r0_tm45 = img0_tm.row(tiles * 2 + i * w_tm/8 + j);

  3217.                     float* r0_tm67 = img0_tm.row(tiles * 3 + i * w_tm/8 + j);

  3218. 

  3219.                     for (int m=0; m<8; m++)

  3220.                     {

  3221.                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25;

  3222.                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25;

  3223. 

  3224.                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25);

  3225.                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25);

  3226. 

  3227.                         tmp[1][m] = tmp12a + tmp12b;

  3228.                         tmp[2][m] = tmp12a - tmp12b;

  3229. 

  3230.                         float tmp34a = (r0[6] + r0[2] * 0.25 - r0[4] * 1.25);

  3231.                         float tmp34b = (r0[1] * 0.5 - r0[3] * 2.5 + r0[5] * 2);

  3232. 

  3233.                         tmp[3][m] = tmp34a + tmp34b;

  3234.                         tmp[4][m] = tmp34a - tmp34b;

  3235. 

  3236.                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25) * 4);

  3237.                         float tmp56b = (r0[1] * 2 - r0[3] * 2.5 + r0[5] * 0.5);

  3238. 

  3239.                         tmp[5][m] = tmp56a + tmp56b;

  3240.                         tmp[6][m] = tmp56a - tmp56b;

  3241. 

  3242.                         r0 += w;

  3243.                     }

  3244. 

  3245.                     float* r0_tms[4] = { r0_tm01, r0_tm23, r0_tm45, r0_tm67 };

  3246. 

  3247.                     for (int m=0; m<8; m++)

  3248.                     {

  3249.                         const float* tmp0 = tmp[m];

  3250. 

  3251.                         float* r0_tm = r0_tms[m/2] + (m%2) * 8;

  3252. 

  3253.                         r0_tm[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25;

  3254.                         r0_tm[7] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25;

  3255. 

  3256.                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25);

  3257.                         float tmp12b = (tmp0[1] - tmp0[3] * 4.25 + tmp0[5]);

  3258. 

  3259.                         r0_tm[1] = tmp12a + tmp12b;

  3260.                         r0_tm[2] = tmp12a - tmp12b;

  3261. 

  3262.                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25 - tmp0[4] * 1.25);

  3263.                         float tmp34b = (tmp0[1] * 0.5 - tmp0[3] * 2.5 + tmp0[5] * 2);

  3264. 

  3265.                         r0_tm[3] = tmp34a + tmp34b;

  3266.                         r0_tm[4] = tmp34a - tmp34b;

  3267. 

  3268.                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25) * 4);

  3269.                         float tmp56b = (tmp0[1] * 2 - tmp0[3] * 2.5 + tmp0[5] * 0.5);

  3270. 

  3271.                         r0_tm[5] = tmp56a + tmp56b;

  3272.                         r0_tm[6] = tmp56a - tmp56b;

  3273.                     }

  3274.                 }

  3275.             }

  3276.         }

  3277. 

  3278.     }

  3279.     bottom_blob_bordered = Mat();

  3280.     // END transform input

  3281. 

  3282.     // BEGIN dot

  3283.     Mat top_blob_tm;

  3284.     {

  3285.         int w_tm = outw / 6 * 8;

  3286.         int h_tm = outh / 6 * 8;

  3287.         top_blob_tm.create(2*8, 4 * w_tm/8 * h_tm/8, outch);

  3288. 

  3289.         const int tiles = h_tm/8 * w_tm/8;

  3290. 

  3291.         #pragma omp parallel for

  3292.         for (int p = 0; p<outch; p++)

  3293.         {

  3294.             Mat out0_tm = top_blob_tm.channel(p);

  3295.             const Mat kernel0_tm = kernel_tm.channel(p);

  3296. 

  3297.             out0_tm.fill(0.f);

  3298. 

  3299.             int q = 0;

  3300.             for (; q+1<inch; q+=2)

  3301.             {

  3302.                 const float* r0 = bottom_blob_tm.channel(q);

  3303.                 const float* r1 = bottom_blob_tm.channel(q+1);

  3304. 

  3305.                 const float* k0 = kernel0_tm.row(q);

  3306.                 const float* k1 = kernel0_tm.row(q+1);

  3307. 

  3308.                 float* output0_tm = out0_tm;

  3309. 

  3310.                 for (int r=0; r<4; r++)

  3311.                 {

  3312. #if __ARM_NEON

  3313. #if __aarch64__

  3314.                 float32x4_t _k0 = vld1q_f32(k0);

  3315.                 float32x4_t _k0n = vld1q_f32(k0+4);

  3316.                 float32x4_t _k0nn = vld1q_f32(k0+8);

  3317.                 float32x4_t _k0nnn = vld1q_f32(k0+12);

  3318.                 float32x4_t _k1 = vld1q_f32(k1);

  3319.                 float32x4_t _k1n = vld1q_f32(k1+4);

  3320.                 float32x4_t _k1nn = vld1q_f32(k1+8);

  3321.                 float32x4_t _k1nnn = vld1q_f32(k1+12);

  3322. #else

  3323.                 float32x4_t _k0;

  3324.                 float32x4_t _k0n;

  3325.                 float32x4_t _k0nn;

  3326.                 float32x4_t _k0nnn;

  3327.                 float32x4_t _k1;

  3328.                 float32x4_t _k1n;

  3329.                 float32x4_t _k1nn;

  3330.                 float32x4_t _k1nnn;

  3331. 

  3332.                 asm volatile(

  3333.                     "pld        [%0, #512]              \n"

  3334.                     "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"

  3335.                     "pld        [%1, #512]              \n"

  3336.                     "vld1.f32   {%e4-%f4}, [%1 :128]!   \n"

  3337. 

  3338.                     "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"

  3339.                     "vld1.f32   {%e5-%f5}, [%1 :128]!   \n"

  3340. 

  3341.                     "vld1.f32   {%e6-%f6}, [%0 :128]!   \n"

  3342.                     "vld1.f32   {%e8-%f8}, [%1 :128]!   \n"

  3343. 

  3344.                     "vld1.f32   {%e7-%f7}, [%0 :128]!   \n"

  3345.                     "vld1.f32   {%e9-%f9}, [%1 :128]!   \n"

  3346.                     : "=r"(k0),     // %0

  3347.                       "=r"(k1),     // %1

  3348.                       "=w"(_k0),    // %2

  3349.                       "=w"(_k0n),   // %3

  3350.                       "=w"(_k1),    // %4

  3351.                       "=w"(_k1n),   // %5

  3352.                       "=w"(_k0nn),  // %6

  3353.                       "=w"(_k0nnn), // %7

  3354.                       "=w"(_k1nn),  // %8

  3355.                       "=w"(_k1nnn)  // %9

  3356.                     : "0"(k0),

  3357.                       "1"(k1)

  3358.                     : "cc", "memory"

  3359.                 );

  3360. #endif // __aarch64__

  3361. #endif // __ARM_NEON

  3362. 

  3363.                 // tile

  3364. #if __ARM_NEON

  3365.                 int nn = tiles >> 2;

  3366.                 int remain = tiles & 3;

  3367. #else

  3368.                 int remain = tiles;

  3369. #endif // __ARM_NEON

  3370. 

  3371. #if __ARM_NEON

  3372. #if __aarch64__

  3373.                 for (; nn>0; nn--)

  3374.                 {

  3375.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  3376.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  3377. 

  3378.                     float32x4_t _r0 = vld1q_f32(r0);

  3379.                     float32x4_t _r0n = vld1q_f32(r0+4);

  3380.                     float32x4_t _r1 = vld1q_f32(r1);

  3381.                     float32x4_t _r1n = vld1q_f32(r1+4);

  3382. 

  3383.                     r0 += 8;

  3384.                     r1 += 8;

  3385. 

  3386.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3387.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3388.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  3389.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  3390. 

  3391.                     vst1q_f32(output0_tm, _output0_tm);

  3392.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3393. 

  3394.                     output0_tm += 8;

  3395. 

  3396.                     _output0_tm = vld1q_f32(output0_tm);

  3397.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3398. 

  3399.                     _r0 = vld1q_f32(r0);

  3400.                     _r0n = vld1q_f32(r0+4);

  3401.                     _r1 = vld1q_f32(r1);

  3402.                     _r1n = vld1q_f32(r1+4);

  3403. 

  3404.                     r0 += 8;

  3405.                     r1 += 8;

  3406. 

  3407.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3408.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3409.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);

  3410.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);

  3411. 

  3412.                     vst1q_f32(output0_tm, _output0_tm);

  3413.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3414. 

  3415.                     output0_tm += 8;

  3416. 

  3417.                     _output0_tm = vld1q_f32(output0_tm);

  3418.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3419. 

  3420.                     _r0 = vld1q_f32(r0);

  3421.                     _r0n = vld1q_f32(r0+4);

  3422.                     _r1 = vld1q_f32(r1);

  3423.                     _r1n = vld1q_f32(r1+4);

  3424. 

  3425.                     r0 += 8;

  3426.                     r1 += 8;

  3427. 

  3428.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3429.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3430.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  3431.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  3432. 

  3433.                     vst1q_f32(output0_tm, _output0_tm);

  3434.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3435. 

  3436.                     output0_tm += 8;

  3437. 

  3438.                     _output0_tm = vld1q_f32(output0_tm);

  3439.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3440. 

  3441.                     _r0 = vld1q_f32(r0);

  3442.                     _r0n = vld1q_f32(r0+4);

  3443.                     _r1 = vld1q_f32(r1);

  3444.                     _r1n = vld1q_f32(r1+4);

  3445. 

  3446.                     r0 += 8;

  3447.                     r1 += 8;

  3448. 

  3449.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3450.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3451.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);

  3452.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);

  3453. 

  3454.                     vst1q_f32(output0_tm, _output0_tm);

  3455.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3456. 

  3457.                     output0_tm += 8;

  3458. 

  3459.                     _output0_tm = vld1q_f32(output0_tm);

  3460.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3461. 

  3462.                     _r0 = vld1q_f32(r0);

  3463.                     _r0n = vld1q_f32(r0+4);

  3464.                     _r1 = vld1q_f32(r1);

  3465.                     _r1n = vld1q_f32(r1+4);

  3466. 

  3467.                     r0 += 8;

  3468.                     r1 += 8;

  3469. 

  3470.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3471.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3472.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  3473.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  3474. 

  3475.                     vst1q_f32(output0_tm, _output0_tm);

  3476.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3477. 

  3478.                     output0_tm += 8;

  3479. 

  3480.                     _output0_tm = vld1q_f32(output0_tm);

  3481.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3482. 

  3483.                     _r0 = vld1q_f32(r0);

  3484.                     _r0n = vld1q_f32(r0+4);

  3485.                     _r1 = vld1q_f32(r1);

  3486.                     _r1n = vld1q_f32(r1+4);

  3487. 

  3488.                     r0 += 8;

  3489.                     r1 += 8;

  3490. 

  3491.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3492.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3493.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);

  3494.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);

  3495. 

  3496.                     vst1q_f32(output0_tm, _output0_tm);

  3497.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3498. 

  3499.                     output0_tm += 8;

  3500. 

  3501.                     _output0_tm = vld1q_f32(output0_tm);

  3502.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3503. 

  3504.                     _r0 = vld1q_f32(r0);

  3505.                     _r0n = vld1q_f32(r0+4);

  3506.                     _r1 = vld1q_f32(r1);

  3507.                     _r1n = vld1q_f32(r1+4);

  3508. 

  3509.                     r0 += 8;

  3510.                     r1 += 8;

  3511. 

  3512.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3513.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3514.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  3515.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  3516. 

  3517.                     vst1q_f32(output0_tm, _output0_tm);

  3518.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3519. 

  3520.                     output0_tm += 8;

  3521. 

  3522.                     _output0_tm = vld1q_f32(output0_tm);

  3523.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3524. 

  3525.                     _r0 = vld1q_f32(r0);

  3526.                     _r0n = vld1q_f32(r0+4);

  3527.                     _r1 = vld1q_f32(r1);

  3528.                     _r1n = vld1q_f32(r1+4);

  3529. 

  3530.                     r0 += 8;

  3531.                     r1 += 8;

  3532. 

  3533.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3534.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3535.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);

  3536.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);

  3537. 

  3538.                     vst1q_f32(output0_tm, _output0_tm);

  3539.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3540. 

  3541.                     output0_tm += 8;

  3542.                 }

  3543. #else

  3544.                 if (nn > 0)

  3545.                 {

  3546.                     asm volatile(

  3547.                         "mov        r4, %1                  \n"

  3548. 

  3549.                         "pld        [%2, #256]              \n"

  3550.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3551. 

  3552.                         "pld        [%1, #256]              \n"

  3553.                         "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm

  3554. 

  3555.                         "vmla.f32   q8, q12, %q8            \n"

  3556.                         "vmla.f32   q9, q13, %q9            \n"

  3557. 

  3558.                         "pld        [%2, #256]              \n"

  3559.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3560. 

  3561.                         "0:                                 \n"

  3562. 

  3563.                         "pld        [%1, #256]              \n"

  3564.                         "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm

  3565. 

  3566.                         "vmla.f32   q10, q12, %q12          \n"

  3567.                         "vmla.f32   q11, q13, %q13          \n"

  3568. 

  3569.                         "pld        [%3, #256]              \n"

  3570.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3571. 

  3572.                         "vmla.f32   q8, q14, %q10           \n"

  3573.                         "vmla.f32   q9, q15, %q11           \n"

  3574. 

  3575.                         "pld        [%3, #256]              \n"

  3576.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3577. 

  3578.                         "pld        [%2, #256]              \n"

  3579.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3580. 

  3581.                         "vmla.f32   q10, q14, %q14          \n"

  3582.                         "vmla.f32   q11, q15, %q15          \n"

  3583. 

  3584.                         "vst1.f32   {d16-d19}, [r4 :128]!   \n"

  3585. 

  3586.                         "pld        [%1, #256]              \n"

  3587.                         "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm

  3588. 

  3589.                         "vmla.f32   q8, q12, %q8            \n"

  3590.                         "vmla.f32   q9, q13, %q9            \n"

  3591. 

  3592.                         "pld        [%2, #256]              \n"

  3593.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3594. 

  3595.                         "vst1.f32   {d20-d23}, [r4 :128]!   \n"

  3596. 

  3597.                         "pld        [%1, #256]              \n"

  3598.                         "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm

  3599. 

  3600.                         "vmla.f32   q10, q12, %q12          \n"

  3601.                         "vmla.f32   q11, q13, %q13          \n"

  3602. 

  3603.                         "pld        [%3, #256]              \n"

  3604.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3605. 

  3606.                         "vmla.f32   q8, q14, %q10           \n"

  3607.                         "vmla.f32   q9, q15, %q11           \n"

  3608. 

  3609.                         "pld        [%3, #256]              \n"

  3610.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3611. 

  3612.                         "pld        [%2, #256]              \n"

  3613.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3614. 

  3615.                         "vmla.f32   q10, q14, %q14          \n"

  3616.                         "vmla.f32   q11, q15, %q15          \n"

  3617. 

  3618.                         "vst1.f32   {d16-d19}, [r4 :128]!   \n"

  3619. 

  3620.                         "pld        [%1, #256]              \n"

  3621.                         "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm

  3622. 

  3623.                         "vmla.f32   q8, q12, %q8            \n"

  3624.                         "vmla.f32   q9, q13, %q9            \n"

  3625. 

  3626.                         "pld        [%2, #256]              \n"

  3627.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3628. 

  3629.                         "vst1.f32   {d20-d23}, [r4 :128]!   \n"

  3630. 

  3631.                         "pld        [%1, #256]              \n"

  3632.                         "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm

  3633. 

  3634.                         "vmla.f32   q10, q12, %q12          \n"

  3635.                         "vmla.f32   q11, q13, %q13          \n"

  3636. 

  3637.                         "pld        [%3, #256]              \n"

  3638.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3639. 

  3640.                         "vmla.f32   q8, q14, %q10           \n"

  3641.                         "vmla.f32   q9, q15, %q11           \n"

  3642. 

  3643.                         "pld        [%3, #256]              \n"

  3644.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3645. 

  3646.                         "pld        [%2, #256]              \n"

  3647.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3648. 

  3649.                         "vmla.f32   q10, q14, %q14          \n"

  3650.                         "vmla.f32   q11, q15, %q15          \n"

  3651. 

  3652.                         "vst1.f32   {d16-d19}, [r4 :128]!   \n"

  3653. 

  3654.                         "pld        [%1, #256]              \n"

  3655.                         "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm

  3656. 

  3657.                         "vmla.f32   q8, q12, %q8            \n"

  3658.                         "vmla.f32   q9, q13, %q9            \n"

  3659. 

  3660.                         "pld        [%2, #256]              \n"

  3661.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3662. 

  3663.                         "vst1.f32   {d20-d23}, [r4 :128]!   \n"

  3664. 

  3665.                         "pld        [%1, #256]              \n"

  3666.                         "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm

  3667. 

  3668.                         "vmla.f32   q10, q12, %q12          \n"

  3669.                         "vmla.f32   q11, q13, %q13          \n"

  3670. 

  3671.                         "pld        [%3, #256]              \n"

  3672.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3673. 

  3674.                         "vmla.f32   q8, q14, %q10           \n"

  3675.                         "vmla.f32   q9, q15, %q11           \n"

  3676. 

  3677.                         "pld        [%3, #256]              \n"

  3678.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  3679. 

  3680.                         "pld        [%2, #256]              \n"

  3681.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3682. 

  3683.                         "vmla.f32   q10, q14, %q14          \n"

  3684.                         "vmla.f32   q11, q15, %q15          \n"

  3685. 

  3686.                         "vst1.f32   {d16-d19}, [r4 :128]!   \n"

  3687. 

  3688.                         "pld        [%1, #256]              \n"

  3689.                         "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm

  3690. 

  3691.                         "vmla.f32   q8, q12, %q8            \n"

  3692.                         "vmla.f32   q9, q13, %q9            \n"

  3693. 

  3694.                         "pld        [%2, #256]              \n"

  3695.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  3696. 

  3697.                         "subs       %0, #1                  \n"

  3698. 

  3699.                         "vst1.f32   {d20-d23}, [r4 :128]!   \n"

  3700. 

  3701.                         "bne        0b                      \n"

  3702. 

  3703.                         "sub        %1, #32                 \n"

  3704.                         "sub        %2, #64                 \n"

  3705.                         : "=r"(nn),         // %0

  3706.                           "=r"(output0_tm), // %1

  3707.                           "=r"(r0),         // %2

  3708.                           "=r"(r1)          // %3

  3709.                         : "0"(nn),

  3710.                           "1"(output0_tm),

  3711.                           "2"(r0),

  3712.                           "3"(r1),

  3713.                           "w"(_k0),         // %8

  3714.                           "w"(_k0n),        // %9

  3715.                           "w"(_k1),         // %10

  3716.                           "w"(_k1n),        // %11

  3717.                           "w"(_k0nn),       // %12

  3718.                           "w"(_k0nnn),      // %13

  3719.                           "w"(_k1nn),       // %14

  3720.                           "w"(_k1nnn)       // %15

  3721.                         : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  3722.                     );

  3723.                 }

  3724. #endif // __aarch64__

  3725. #endif // __ARM_NEON

  3726.                 for (; remain>0; remain--)

  3727.                 {

  3728. #if __ARM_NEON

  3729. #if __aarch64__

  3730.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  3731.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  3732. 

  3733.                     float32x4_t _r0 = vld1q_f32(r0);

  3734.                     float32x4_t _r0n = vld1q_f32(r0+4);

  3735.                     float32x4_t _r1 = vld1q_f32(r1);

  3736.                     float32x4_t _r1n = vld1q_f32(r1+4);

  3737. 

  3738.                     r0 += 8;

  3739.                     r1 += 8;

  3740. 

  3741.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3742.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3743.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);

  3744.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);

  3745. 

  3746.                     vst1q_f32(output0_tm, _output0_tm);

  3747.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3748. 

  3749.                     output0_tm += 8;

  3750. 

  3751.                     _output0_tm = vld1q_f32(output0_tm);

  3752.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3753. 

  3754.                     _r0 = vld1q_f32(r0);

  3755.                     _r0n = vld1q_f32(r0+4);

  3756.                     _r1 = vld1q_f32(r1);

  3757.                     _r1n = vld1q_f32(r1+4);

  3758. 

  3759.                     r0 += 8;

  3760.                     r1 += 8;

  3761. 

  3762.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3763.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3764.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);

  3765.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);

  3766. 

  3767.                     vst1q_f32(output0_tm, _output0_tm);

  3768.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3769. 

  3770.                     output0_tm += 8;

  3771. #else

  3772.                     asm volatile(

  3773.                         "mov        r4, %0                  \n"

  3774. 

  3775.                         "pld        [%1, #256]              \n"

  3776.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  3777. 

  3778.                         "pld        [%0, #256]              \n"

  3779.                         "vld1.f32   {d16-d19}, [%0 :128]!   \n"// q8 q9 = _output0_tm

  3780. 

  3781.                         "vmla.f32   q8, q12, %q6            \n"

  3782. 

  3783.                         "pld        [%2, #256]              \n"

  3784.                         "vld1.f32   {d28-d31}, [%2 :128]!   \n"// q14 q15 = _r1

  3785.                         "vmla.f32   q9, q13, %q7            \n"

  3786. 

  3787.                         "pld        [%1, #256]              \n"

  3788.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  3789. 

  3790.                         "vmla.f32   q8, q14, %q8            \n"

  3791. 

  3792.                         "pld        [%0, #256]              \n"

  3793.                         "vld1.f32   {d20-d23}, [%0 :128]    \n"// q10 q11 = _output0_tm

  3794.                         "vmla.f32   q9, q15, %q9            \n"

  3795. 

  3796.                         "vmla.f32   q10, q12, %q10          \n"

  3797.                         "vmla.f32   q11, q13, %q11          \n"

  3798. 

  3799.                         "vst1.f32   {d16-d19}, [r4 :128]    \n"

  3800. 

  3801.                         "pld        [%2, #256]              \n"

  3802.                         "vld1.f32   {d28-d31}, [%2 :128]!   \n"// q14 q15 = _r1

  3803. 

  3804.                         "vmla.f32   q10, q14, %q12          \n"

  3805.                         "vmla.f32   q11, q15, %q13          \n"

  3806. 

  3807.                         "vst1.f32   {d20-d23}, [%0 :128]!   \n"

  3808.                         : "=r"(output0_tm), // %0

  3809.                           "=r"(r0),         // %1

  3810.                           "=r"(r1)          // %2

  3811.                         : "0"(output0_tm),

  3812.                           "1"(r0),

  3813.                           "2"(r1),

  3814.                           "w"(_k0),         // %6

  3815.                           "w"(_k0n),        // %7

  3816.                           "w"(_k1),         // %8

  3817.                           "w"(_k1n),        // %9

  3818.                           "w"(_k0nn),       // %10

  3819.                           "w"(_k0nnn),      // %11

  3820.                           "w"(_k1nn),       // %12

  3821.                           "w"(_k1nnn)       // %13

  3822.                         : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  3823.                     );

  3824. #endif // __aarch64__

  3825. #else

  3826.                     for (int m=0; m<16; m++)

  3827.                     {

  3828.                         output0_tm[m] += r0[m] * k0[m];

  3829.                         output0_tm[m] += r1[m] * k1[m];

  3830.                     }

  3831. 

  3832.                     r0 += 16;

  3833.                     r1 += 16;

  3834.                     output0_tm += 16;

  3835. #endif // __ARM_NEON

  3836.                 }

  3837. 

  3838. #if __ARM_NEON

  3839. #if __aarch64__

  3840.                 k0 += 16;

  3841.                 k1 += 16;

  3842. #endif // __aarch64__

  3843. #else

  3844.                 k0 += 16;

  3845.                 k1 += 16;

  3846. #endif // __ARM_NEON

  3847.                 }

  3848.             }

  3849. 

  3850.             for (; q<inch; q++)

  3851.             {

  3852.                 const float* r0 = bottom_blob_tm.channel(q);

  3853. 

  3854.                 const float* k0 = kernel0_tm.row(q);

  3855. 

  3856.                 float* output0_tm = out0_tm;

  3857. 

  3858.                 for (int r=0; r<4; r++)

  3859.                 {

  3860. #if __ARM_NEON

  3861. #if __aarch64__

  3862.                 float32x4_t _k0 = vld1q_f32(k0);

  3863.                 float32x4_t _k0n = vld1q_f32(k0+4);

  3864.                 float32x4_t _k0nn = vld1q_f32(k0+8);

  3865.                 float32x4_t _k0nnn = vld1q_f32(k0+12);

  3866. #else

  3867.                 float32x4_t _k0;

  3868.                 float32x4_t _k0n;

  3869.                 float32x4_t _k0nn;

  3870.                 float32x4_t _k0nnn;

  3871. 

  3872.                 asm volatile(

  3873.                     "pld        [%0, #512]              \n"

  3874.                     "vld1.f32   {%e1-%f1}, [%0 :128]!   \n"

  3875.                     "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"

  3876.                     "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"

  3877.                     "vld1.f32   {%e4-%f4}, [%0 :128]!   \n"

  3878.                     : "=r"(k0),     // %0

  3879.                       "=w"(_k0),    // %1

  3880.                       "=w"(_k0n),   // %2

  3881.                       "=w"(_k0nn),  // %3

  3882.                       "=w"(_k0nnn)  // %4

  3883.                     : "0"(k0)

  3884.                     : "cc", "memory"

  3885.                 );

  3886. #endif // __aarch64__

  3887. #endif // __ARM_NEON

  3888. 

  3889.                 // tile

  3890.                 for (int i=0; i<tiles; i++)

  3891.                 {

  3892. #if __ARM_NEON

  3893. #if __aarch64__

  3894.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  3895.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  3896. 

  3897.                     float32x4_t _r0 = vld1q_f32(r0);

  3898.                     float32x4_t _r0n = vld1q_f32(r0+4);

  3899. 

  3900.                     r0 += 8;

  3901. 

  3902.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);

  3903.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);

  3904. 

  3905.                     vst1q_f32(output0_tm, _output0_tm);

  3906.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3907. 

  3908.                     output0_tm += 8;

  3909. 

  3910.                     _output0_tm = vld1q_f32(output0_tm);

  3911.                     _output0_tmn = vld1q_f32(output0_tm+4);

  3912. 

  3913.                     _r0 = vld1q_f32(r0);

  3914.                     _r0n = vld1q_f32(r0+4);

  3915. 

  3916.                     r0 += 8;

  3917. 

  3918.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);

  3919.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);

  3920. 

  3921.                     vst1q_f32(output0_tm, _output0_tm);

  3922.                     vst1q_f32(output0_tm+4, _output0_tmn);

  3923. 

  3924.                     output0_tm += 8;

  3925. #else

  3926.                     asm volatile(

  3927.                         "mov        r4, %0                  \n"

  3928. 

  3929.                         "pld        [%1, #256]              \n"

  3930.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  3931. 

  3932.                         "pld        [%0, #256]              \n"

  3933.                         "vld1.f32   {d16-d19}, [%0 :128]!   \n"// q8 q9 = _output0_tm

  3934. 

  3935.                         "vmla.f32   q8, q12, %q4            \n"

  3936.                         "vmla.f32   q9, q13, %q5            \n"

  3937. 

  3938.                         "pld        [%1, #256]              \n"

  3939.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  3940. 

  3941.                         "pld        [%0, #256]              \n"

  3942.                         "vld1.f32   {d20-d23}, [%0 :128]    \n"// q10 q11 = _output0_tm

  3943. 

  3944.                         "vmla.f32   q10, q12, %q6           \n"

  3945. 

  3946.                         "vst1.f32   {d16-d19}, [r4 :128]    \n"

  3947. 

  3948.                         "vmla.f32   q11, q13, %q7           \n"

  3949. 

  3950.                         "vst1.f32   {d20-d23}, [%0 :128]!   \n"

  3951.                         : "=r"(output0_tm), // %0

  3952.                           "=r"(r0)          // %1

  3953.                         : "0"(output0_tm),

  3954.                           "1"(r0),

  3955.                           "w"(_k0),         // %4

  3956.                           "w"(_k0n),        // %5

  3957.                           "w"(_k0nn),       // %6

  3958.                           "w"(_k0nnn)       // %7

  3959.                         : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  3960.                     );

  3961. #endif // __aarch64__

  3962. #else

  3963.                     for (int m=0; m<16; m++)

  3964.                     {

  3965.                         output0_tm[m] += r0[m] * k0[m];

  3966.                     }

  3967. 

  3968.                     r0 += 16;

  3969.                     output0_tm += 16;

  3970. #endif // __ARM_NEON

  3971.                 }

  3972. 

  3973. #if __ARM_NEON

  3974. #if __aarch64__

  3975.                 k0 += 16;

  3976. #endif // __aarch64__

  3977. #else

  3978.                 k0 += 16;

  3979. #endif // __ARM_NEON

  3980.                 }

  3981.             }

  3982.         }

  3983.     }

  3984.     bottom_blob_tm = Mat();

  3985.     // END dot

  3986. 

  3987.     // BEGIN transform output

  3988.     Mat top_blob_bordered;

  3989.     top_blob_bordered.create(outw, outh, outch);

  3990.     {

  3991. //         const float otm[6][8] = {

  3992. //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  3993. //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  3994. //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  3995. //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  3996. //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  3997. //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  3998. //         };

  3999. 

  4000.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  4001.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  4002.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  4003.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  4004.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  4005.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  4006. 

  4007.         int w_tm = outw / 6 * 8;

  4008.         int h_tm = outh / 6 * 8;

  4009.         const int tiles = w_tm/8 * h_tm/8;

  4010. 

  4011.         #pragma omp parallel for

  4012.         for (int p = 0; p<outch; p++)

  4013.         {

  4014.             const Mat out0_tm = top_blob_tm.channel(p);

  4015.             Mat out0 = top_blob_bordered.channel(p);

  4016. 

  4017.             const float bias0 = bias ? bias[p] : 0.f;

  4018. 

  4019.             float tmp[6][8];

  4020. 

  4021.             // tile

  4022.             for (int i=0; i<outh/6; i++)

  4023.             {

  4024.                 for (int j=0; j<outw/6; j++)

  4025.                 {

  4026.                     const float* output0_tm01 = out0_tm.row(i * w_tm/8 + j);

  4027.                     const float* output0_tm23 = out0_tm.row(tiles + i * w_tm/8 + j);

  4028.                     const float* output0_tm45 = out0_tm.row(tiles * 2 + i * w_tm/8 + j);

  4029.                     const float* output0_tm67 = out0_tm.row(tiles * 3 + i * w_tm/8 + j);

  4030.                     float* output0 = out0.row(i * 6) + j * 6;

  4031. 

  4032.                     const float* output0_tms[4] = { output0_tm01, output0_tm23, output0_tm45, output0_tm67 };

  4033. 

  4034.                     for (int m=0; m<8; m++)

  4035.                     {

  4036.                         const float* output0_tm = output0_tms[m/2] + (m%2) * 8;

  4037. 

  4038.                         float tmp024a = output0_tm[1] + output0_tm[2];

  4039.                         float tmp135a = output0_tm[1] - output0_tm[2];

  4040. 

  4041.                         float tmp024b = output0_tm[3] + output0_tm[4];

  4042.                         float tmp135b = output0_tm[3] - output0_tm[4];

  4043. 

  4044.                         float tmp024c = output0_tm[5] + output0_tm[6];

  4045.                         float tmp135c = output0_tm[5] - output0_tm[6];

  4046. 

  4047.                         tmp[0][m] = output0_tm[0] + tmp024a + tmp024b + tmp024c * 32;

  4048.                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  4049.                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  4050. 

  4051.                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  4052.                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  4053.                         tmp[5][m] = output0_tm[7] + tmp135a + tmp135b * 32 + tmp135c;

  4054.                     }

  4055. 

  4056.                     for (int m=0; m<6; m++)

  4057.                     {

  4058.                         const float* tmp0 = tmp[m];

  4059. 

  4060.                         float tmp024a = tmp0[1] + tmp0[2];

  4061.                         float tmp135a = tmp0[1] - tmp0[2];

  4062. 

  4063.                         float tmp024b = tmp0[3] + tmp0[4];

  4064.                         float tmp135b = tmp0[3] - tmp0[4];

  4065. 

  4066.                         float tmp024c = tmp0[5] + tmp0[6];

  4067.                         float tmp135c = tmp0[5] - tmp0[6];

  4068. 

  4069.                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  4070.                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  4071.                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  4072. 

  4073.                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  4074.                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  4075.                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  4076. 

  4077.                         output0 += outw;

  4078.                     }

  4079.                 }

  4080.             }

  4081.         }

  4082.     }

  4083.     // END transform output

  4084. 

  4085.     // cut result pad

  4086.     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w);

  4087. }

  4088. 

  4089. static void conv3x3s1_winograd64_neon3(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias)

  4090. {

  4091.     int w = bottom_blob.w;

  4092.     int h = bottom_blob.h;

  4093.     int inch = bottom_blob.c;

  4094. 

  4095.     int outw = top_blob.w;

  4096.     int outh = top_blob.h;

  4097.     int outch = top_blob.c;

  4098. 

  4099.     // pad to 6n+2

  4100.     Mat bottom_blob_bordered = bottom_blob;

  4101. 

  4102.     outw = (outw + 5) / 6 * 6;

  4103.     outh = (outh + 5) / 6 * 6;

  4104. 

  4105.     w = outw + 2;

  4106.     h = outh + 2;

  4107.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f);

  4108. 

  4109.     const float* bias = _bias;

  4110. 

  4111.     // BEGIN transform input

  4112.     Mat bottom_blob_tm;

  4113.     {

  4114.         int w_tm = outw / 6 * 8;

  4115.         int h_tm = outh / 6 * 8;

  4116.         bottom_blob_tm.create(8, 8 * w_tm/8 * h_tm/8, inch);

  4117.         const int tiles = w_tm/8 * h_tm/8;

  4118. 

  4119. //         const float itm[8][8] = {

  4120. //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  4121. //

  4122. //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  4123. //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  4124. //

  4125. //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  4126. //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  4127. //

  4128. //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  4129. //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  4130. //

  4131. //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  4132. //         };

  4133. 

  4134.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  4135.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  4136. 

  4137.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  4138.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  4139. 

  4140.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  4141.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  4142. 

  4143.         // reuse r04 * 1.25

  4144.         // reuse r03 * 2.5

  4145.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  4146.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  4147. 

  4148.         #pragma omp parallel for

  4149.         for (int q = 0; q<inch; q++)

  4150.         {

  4151.             const Mat img0 = bottom_blob_bordered.channel(q);

  4152.             Mat img0_tm = bottom_blob_tm.channel(q);

  4153. 

  4154.             float tmp[8][8];

  4155. 

  4156.             // tile

  4157.             for (int i=0; i<h_tm/8; i++)

  4158.             {

  4159.                 for (int j=0; j<w_tm/8; j++)

  4160.                 {

  4161.                     const float* r0 = img0.row(i * 6) + j * 6;

  4162.                     float* r0_tm0 = img0_tm.row(i * w_tm/8 + j);

  4163.                     float* r0_tm1 = img0_tm.row(i * w_tm/8 + j + tiles);

  4164.                     float* r0_tm2 = img0_tm.row(i * w_tm/8 + j + tiles * 2);

  4165.                     float* r0_tm3 = img0_tm.row(i * w_tm/8 + j + tiles * 3);

  4166.                     float* r0_tm4 = img0_tm.row(i * w_tm/8 + j + tiles * 4);

  4167.                     float* r0_tm5 = img0_tm.row(i * w_tm/8 + j + tiles * 5);

  4168.                     float* r0_tm6 = img0_tm.row(i * w_tm/8 + j + tiles * 6);

  4169.                     float* r0_tm7 = img0_tm.row(i * w_tm/8 + j + tiles * 7);

  4170. 

  4171.                     for (int m=0; m<8; m++)

  4172.                     {

  4173.                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25;

  4174.                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25;

  4175. 

  4176.                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25);

  4177.                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25);

  4178. 

  4179.                         tmp[1][m] = tmp12a + tmp12b;

  4180.                         tmp[2][m] = tmp12a - tmp12b;

  4181. 

  4182.                         float tmp34a = (r0[6] + r0[2] * 0.25 - r0[4] * 1.25);

  4183.                         float tmp34b = (r0[1] * 0.5 - r0[3] * 2.5 + r0[5] * 2);

  4184. 

  4185.                         tmp[3][m] = tmp34a + tmp34b;

  4186.                         tmp[4][m] = tmp34a - tmp34b;

  4187. 

  4188.                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25) * 4);

  4189.                         float tmp56b = (r0[1] * 2 - r0[3] * 2.5 + r0[5] * 0.5);

  4190. 

  4191.                         tmp[5][m] = tmp56a + tmp56b;

  4192.                         tmp[6][m] = tmp56a - tmp56b;

  4193. 

  4194.                         r0 += w;

  4195.                     }

  4196. 

  4197.                     float* r0_tms[8] = { r0_tm0, r0_tm1, r0_tm2, r0_tm3, r0_tm4, r0_tm5, r0_tm6, r0_tm7 };

  4198. 

  4199.                     for (int m=0; m<8; m++)

  4200.                     {

  4201.                         const float* tmp0 = tmp[m];

  4202. 

  4203.                         float* r0_tm = r0_tms[m];

  4204. 

  4205.                         r0_tm[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25;

  4206.                         r0_tm[7] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25;

  4207. 

  4208.                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25);

  4209.                         float tmp12b = (tmp0[1] - tmp0[3] * 4.25 + tmp0[5]);

  4210. 

  4211.                         r0_tm[1] = tmp12a + tmp12b;

  4212.                         r0_tm[2] = tmp12a - tmp12b;

  4213. 

  4214.                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25 - tmp0[4] * 1.25);

  4215.                         float tmp34b = (tmp0[1] * 0.5 - tmp0[3] * 2.5 + tmp0[5] * 2);

  4216. 

  4217.                         r0_tm[3] = tmp34a + tmp34b;

  4218.                         r0_tm[4] = tmp34a - tmp34b;

  4219. 

  4220.                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25) * 4);

  4221.                         float tmp56b = (tmp0[1] * 2 - tmp0[3] * 2.5 + tmp0[5] * 0.5);

  4222. 

  4223.                         r0_tm[5] = tmp56a + tmp56b;

  4224.                         r0_tm[6] = tmp56a - tmp56b;

  4225.                     }

  4226.                 }

  4227.             }

  4228.         }

  4229. 

  4230.     }

  4231.     bottom_blob_bordered = Mat();

  4232.     // END transform input

  4233. 

  4234.     // BEGIN dot

  4235.     Mat top_blob_tm;

  4236.     {

  4237.         int w_tm = outw / 6 * 8;

  4238.         int h_tm = outh / 6 * 8;

  4239.         top_blob_tm.create(8, 8 * w_tm/8 * h_tm/8, outch);

  4240. 

  4241.         const int tiles = h_tm/8 * w_tm/8;

  4242. 

  4243.         int nn_outch = outch >> 1;

  4244.         int remain_outch_start = nn_outch << 1;

  4245. 

  4246.         #pragma omp parallel for

  4247.         for (int pp=0; pp<nn_outch; pp++)

  4248.         {

  4249.             int p = pp * 2;

  4250. 

  4251.             Mat out0_tm = top_blob_tm.channel(p);

  4252.             Mat out1_tm = top_blob_tm.channel(p+1);

  4253.             const Mat kernel0_tm = kernel_tm.channel(p);

  4254.             const Mat kernel1_tm = kernel_tm.channel(p+1);

  4255. 

  4256.             out0_tm.fill(0.f);

  4257.             out1_tm.fill(0.f);

  4258. 

  4259.             int q = 0;

  4260.             for (; q+1<inch; q+=2)

  4261.             {

  4262.                 const float* r0 = bottom_blob_tm.channel(q);

  4263.                 const float* r1 = bottom_blob_tm.channel(q+1);

  4264. 

  4265.                 const float* k00 = kernel0_tm.row(q);

  4266.                 const float* k01 = kernel0_tm.row(q+1);

  4267.                 const float* k10 = kernel1_tm.row(q);

  4268.                 const float* k11 = kernel1_tm.row(q+1);

  4269. 

  4270.                 float* output0_tm = out0_tm;

  4271.                 float* output1_tm = out1_tm;

  4272. 

  4273.                 for (int r=0; r<8; r++)

  4274.                 {

  4275. #if __ARM_NEON

  4276. #if __aarch64__

  4277.                 float32x4_t _k00 = vld1q_f32(k00);

  4278.                 float32x4_t _k00n = vld1q_f32(k00+4);

  4279.                 float32x4_t _k01 = vld1q_f32(k01);

  4280.                 float32x4_t _k01n = vld1q_f32(k01+4);

  4281.                 float32x4_t _k10 = vld1q_f32(k10);

  4282.                 float32x4_t _k10n = vld1q_f32(k10+4);

  4283.                 float32x4_t _k11 = vld1q_f32(k11);

  4284.                 float32x4_t _k11n = vld1q_f32(k11+4);

  4285. #else

  4286.                 float32x4_t _k00;

  4287.                 float32x4_t _k00n;

  4288.                 float32x4_t _k01;

  4289.                 float32x4_t _k01n;

  4290.                 float32x4_t _k10;

  4291.                 float32x4_t _k10n;

  4292.                 float32x4_t _k11;

  4293.                 float32x4_t _k11n;

  4294. 

  4295.                 asm volatile(

  4296.                     "pld        [%0, #256]              \n"

  4297.                     "vld1.f32   {%e4-%f4}, [%0 :128]!   \n"

  4298.                     "pld        [%1, #256]              \n"

  4299.                     "vld1.f32   {%e6-%f6}, [%1 :128]!   \n"

  4300.                     "pld        [%2, #256]              \n"

  4301.                     "vld1.f32   {%e8-%f8}, [%2 :128]!   \n"

  4302.                     "pld        [%3, #256]              \n"

  4303.                     "vld1.f32   {%e10-%f10}, [%3 :128]! \n"

  4304. 

  4305.                     "vld1.f32   {%e5-%f5}, [%0 :128]!   \n"

  4306.                     "vld1.f32   {%e7-%f7}, [%1 :128]!   \n"

  4307.                     "vld1.f32   {%e9-%f9}, [%2 :128]!   \n"

  4308.                     "vld1.f32   {%e11-%f11}, [%3 :128]! \n"

  4309.                     : "=r"(k00),    // %0

  4310.                       "=r"(k01),    // %1

  4311.                       "=r"(k10),    // %2

  4312.                       "=r"(k11),    // %3

  4313.                       "=w"(_k00),   // %4

  4314.                       "=w"(_k00n),  // %5

  4315.                       "=w"(_k01),   // %6

  4316.                       "=w"(_k01n),  // %7

  4317.                       "=w"(_k10),   // %8

  4318.                       "=w"(_k10n),  // %9

  4319.                       "=w"(_k11),   // %10

  4320.                       "=w"(_k11n)   // %11

  4321.                     : "0"(k00),

  4322.                       "1"(k01),

  4323.                       "2"(k10),

  4324.                       "3"(k11)

  4325.                     : "cc", "memory"

  4326.                 );

  4327. #endif // __aarch64__

  4328. #endif // __ARM_NEON

  4329. 

  4330.                 // tile

  4331. #if __ARM_NEON

  4332.                 int nn = tiles >> 2;

  4333.                 int remain = tiles & 3;

  4334. #else

  4335.                 int remain = tiles;

  4336. #endif // __ARM_NEON

  4337. 

  4338. #if __ARM_NEON

  4339. #if __aarch64__

  4340.                 for (; nn>0; nn--)

  4341.                 {

  4342.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  4343.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  4344. 

  4345.                     float32x4_t _output1_tm = vld1q_f32(output1_tm);

  4346.                     float32x4_t _output1_tmn = vld1q_f32(output1_tm+4);

  4347. 

  4348.                     float32x4_t _r0 = vld1q_f32(r0);

  4349.                     float32x4_t _r0n = vld1q_f32(r0+4);

  4350.                     float32x4_t _r1 = vld1q_f32(r1);

  4351.                     float32x4_t _r1n = vld1q_f32(r1+4);

  4352. 

  4353.                     r0 += 8;

  4354.                     r1 += 8;

  4355. 

  4356.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4357.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4358.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  4359.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  4360. 

  4361.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4362.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4363.                     _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  4364.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  4365. 

  4366.                     vst1q_f32(output0_tm, _output0_tm);

  4367.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4368. 

  4369.                     vst1q_f32(output1_tm, _output1_tm);

  4370.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4371. 

  4372.                     output0_tm += 8;

  4373.                     output1_tm += 8;

  4374. 

  4375.                     _output0_tm = vld1q_f32(output0_tm);

  4376.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4377. 

  4378.                     _output1_tm = vld1q_f32(output1_tm);

  4379.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4380. 

  4381.                     _r0 = vld1q_f32(r0);

  4382.                     _r0n = vld1q_f32(r0+4);

  4383.                     _r1 = vld1q_f32(r1);

  4384.                     _r1n = vld1q_f32(r1+4);

  4385. 

  4386.                     r0 += 8;

  4387.                     r1 += 8;

  4388. 

  4389.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4390.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4391.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  4392.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  4393. 

  4394.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4395.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4396.                     _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  4397.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  4398. 

  4399.                     vst1q_f32(output0_tm, _output0_tm);

  4400.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4401. 

  4402.                     vst1q_f32(output1_tm, _output1_tm);

  4403.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4404. 

  4405.                     output0_tm += 8;

  4406.                     output1_tm += 8;

  4407. 

  4408.                     _output0_tm = vld1q_f32(output0_tm);

  4409.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4410. 

  4411.                     _output1_tm = vld1q_f32(output1_tm);

  4412.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4413. 

  4414.                     _r0 = vld1q_f32(r0);

  4415.                     _r0n = vld1q_f32(r0+4);

  4416.                     _r1 = vld1q_f32(r1);

  4417.                     _r1n = vld1q_f32(r1+4);

  4418. 

  4419.                     r0 += 8;

  4420.                     r1 += 8;

  4421. 

  4422.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4423.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4424.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  4425.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  4426. 

  4427.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4428.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4429.                     _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  4430.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  4431. 

  4432.                     vst1q_f32(output0_tm, _output0_tm);

  4433.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4434. 

  4435.                     vst1q_f32(output1_tm, _output1_tm);

  4436.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4437. 

  4438.                     output0_tm += 8;

  4439.                     output1_tm += 8;

  4440. 

  4441.                     _output0_tm = vld1q_f32(output0_tm);

  4442.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4443. 

  4444.                     _output1_tm = vld1q_f32(output1_tm);

  4445.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4446. 

  4447.                     _r0 = vld1q_f32(r0);

  4448.                     _r0n = vld1q_f32(r0+4);

  4449.                     _r1 = vld1q_f32(r1);

  4450.                     _r1n = vld1q_f32(r1+4);

  4451. 

  4452.                     r0 += 8;

  4453.                     r1 += 8;

  4454. 

  4455.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4456.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4457.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  4458.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  4459. 

  4460.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4461.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4462.                     _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  4463.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  4464. 

  4465.                     vst1q_f32(output0_tm, _output0_tm);

  4466.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4467. 

  4468.                     vst1q_f32(output1_tm, _output1_tm);

  4469.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4470. 

  4471.                     output0_tm += 8;

  4472.                     output1_tm += 8;

  4473.                 }

  4474. #else

  4475.                 if (nn > 0)

  4476.                 {

  4477.                     asm volatile(

  4478.                         "0:                                 \n"

  4479. 

  4480.                         "pld        [%3, #256]              \n"

  4481.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4482. 

  4483.                         "pld        [%1, #256]              \n"

  4484.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4485. 

  4486.                         "vmla.f32   q8, q12, %q10           \n"

  4487.                         "vmla.f32   q9, q13, %q11           \n"

  4488. 

  4489.                         "pld        [%4, #256]              \n"

  4490.                         "vld1.f32   {d28-d31}, [%4 :128]!   \n"// q14 q15 = _r1

  4491. 

  4492.                         "vmla.f32   q8, q14, %q12           \n"

  4493.                         "vmla.f32   q9, q15, %q13           \n"

  4494. 

  4495.                         "pld        [%2, #256]              \n"

  4496.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4497. 

  4498.                         "vmla.f32   q10, q12, %q14          \n"

  4499.                         "vmla.f32   q11, q13, %q15          \n"

  4500. 

  4501.                         "pld        [%3, #256]              \n"

  4502.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4503. 

  4504.                         "vmla.f32   q10, q14, %q16          \n"

  4505.                         "vmla.f32   q11, q15, %q17          \n"

  4506. 

  4507.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4508. 

  4509.                         "pld        [%1, #256]              \n"

  4510.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4511. 

  4512.                         "vmla.f32   q8, q12, %q10           \n"

  4513.                         "vmla.f32   q9, q13, %q11           \n"

  4514. 

  4515.                         "pld        [%4, #256]              \n"

  4516.                         "vld1.f32   {d28-d31}, [%4 :128]!   \n"// q14 q15 = _r1

  4517. 

  4518.                         "vmla.f32   q8, q14, %q12           \n"

  4519.                         "vmla.f32   q9, q15, %q13           \n"

  4520. 

  4521.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4522. 

  4523.                         "pld        [%2, #256]              \n"

  4524.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4525. 

  4526.                         "vmla.f32   q10, q12, %q14          \n"

  4527.                         "vmla.f32   q11, q13, %q15          \n"

  4528. 

  4529.                         "pld        [%3, #256]              \n"

  4530.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4531. 

  4532.                         "vmla.f32   q10, q14, %q16          \n"

  4533.                         "vmla.f32   q11, q15, %q17          \n"

  4534. 

  4535.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4536. 

  4537.                         "pld        [%1, #256]              \n"

  4538.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4539. 

  4540.                         "vmla.f32   q8, q12, %q10           \n"

  4541.                         "vmla.f32   q9, q13, %q11           \n"

  4542. 

  4543.                         "pld        [%4, #256]              \n"

  4544.                         "vld1.f32   {d28-d31}, [%4 :128]!   \n"// q14 q15 = _r1

  4545. 

  4546.                         "vmla.f32   q8, q14, %q12           \n"

  4547.                         "vmla.f32   q9, q15, %q13           \n"

  4548. 

  4549.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4550. 

  4551.                         "pld        [%2, #256]              \n"

  4552.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4553. 

  4554.                         "vmla.f32   q10, q12, %q14          \n"

  4555.                         "vmla.f32   q11, q13, %q15          \n"

  4556. 

  4557.                         "pld        [%3, #256]              \n"

  4558.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4559. 

  4560.                         "vmla.f32   q10, q14, %q16          \n"

  4561.                         "vmla.f32   q11, q15, %q17          \n"

  4562. 

  4563.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4564. 

  4565.                         "pld        [%1, #256]              \n"

  4566.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4567. 

  4568.                         "vmla.f32   q8, q12, %q10           \n"

  4569.                         "vmla.f32   q9, q13, %q11           \n"

  4570. 

  4571.                         "pld        [%4, #256]              \n"

  4572.                         "vld1.f32   {d28-d31}, [%4 :128]!   \n"// q14 q15 = _r1

  4573. 

  4574.                         "vmla.f32   q8, q14, %q12           \n"

  4575.                         "vmla.f32   q9, q15, %q13           \n"

  4576. 

  4577.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4578. 

  4579.                         "pld        [%2, #256]              \n"

  4580.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4581. 

  4582.                         "vmla.f32   q10, q12, %q14          \n"

  4583.                         "vmla.f32   q11, q13, %q15          \n"

  4584. 

  4585.                         "vmla.f32   q10, q14, %q16          \n"

  4586.                         "vmla.f32   q11, q15, %q17          \n"

  4587. 

  4588.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4589.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4590. 

  4591.                         "subs       %0, #1                  \n"

  4592. 

  4593.                         "bne        0b                      \n"

  4594. 

  4595.                         : "=r"(nn),         // %0

  4596.                           "=r"(output0_tm), // %1

  4597.                           "=r"(output1_tm), // %2

  4598.                           "=r"(r0),         // %3

  4599.                           "=r"(r1)          // %4

  4600.                         : "0"(nn),

  4601.                           "1"(output0_tm),

  4602.                           "2"(output1_tm),

  4603.                           "3"(r0),

  4604.                           "4"(r1),

  4605.                           "w"(_k00),         // %10

  4606.                           "w"(_k00n),        // %11

  4607.                           "w"(_k01),         // %12

  4608.                           "w"(_k01n),        // %13

  4609.                           "w"(_k10),         // %14

  4610.                           "w"(_k10n),        // %15

  4611.                           "w"(_k11),         // %16

  4612.                           "w"(_k11n)         // %17

  4613.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  4614.                     );

  4615.                 }

  4616. #endif // __aarch64__

  4617. #endif // __ARM_NEON

  4618.                 for (; remain>0; remain--)

  4619.                 {

  4620. #if __ARM_NEON

  4621. #if __aarch64__

  4622.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  4623.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  4624. 

  4625.                     float32x4_t _output1_tm = vld1q_f32(output1_tm);

  4626.                     float32x4_t _output1_tmn = vld1q_f32(output1_tm+4);

  4627. 

  4628.                     float32x4_t _r0 = vld1q_f32(r0);

  4629.                     float32x4_t _r0n = vld1q_f32(r0+4);

  4630.                     float32x4_t _r1 = vld1q_f32(r1);

  4631.                     float32x4_t _r1n = vld1q_f32(r1+4);

  4632. 

  4633.                     r0 += 8;

  4634.                     r1 += 8;

  4635. 

  4636.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4637.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4638.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  4639.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  4640. 

  4641.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4642.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4643.                     _output1_tm = vmlaq_f32(_output1_tm, _r1, _k11);

  4644.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r1n, _k11n);

  4645. 

  4646.                     vst1q_f32(output0_tm, _output0_tm);

  4647.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4648. 

  4649.                     vst1q_f32(output1_tm, _output1_tm);

  4650.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4651. 

  4652.                     output0_tm += 8;

  4653.                     output1_tm += 8;

  4654. #else

  4655.                     asm volatile(

  4656.                         "pld        [%2, #256]              \n"

  4657.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  4658. 

  4659.                         "pld        [%0, #256]              \n"

  4660.                         "vld1.f32   {d16-d19}, [%0 :128]    \n"// q8 q9 = _output0_tm

  4661. 

  4662.                         "vmla.f32   q8, q12, %q8            \n"

  4663.                         "vmla.f32   q9, q13, %q9            \n"

  4664. 

  4665.                         "pld        [%3, #256]              \n"

  4666.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  4667. 

  4668.                         "vmla.f32   q8, q14, %q10           \n"

  4669.                         "vmla.f32   q9, q15, %q11           \n"

  4670. 

  4671.                         "pld        [%1, #256]              \n"

  4672.                         "vld1.f32   {d20-d23}, [%1 :128]    \n"// q10 q11 = _output1_tm

  4673. 

  4674.                         "vmla.f32   q10, q12, %q12          \n"

  4675.                         "vmla.f32   q11, q13, %q13          \n"

  4676. 

  4677.                         "vmla.f32   q10, q14, %q14          \n"

  4678.                         "vmla.f32   q11, q15, %q15          \n"

  4679. 

  4680.                         "vst1.f32   {d16-d19}, [%0 :128]!   \n"

  4681.                         "vst1.f32   {d20-d23}, [%1 :128]!   \n"

  4682. 

  4683.                         : "=r"(output0_tm), // %0

  4684.                           "=r"(output1_tm), // %1

  4685.                           "=r"(r0),         // %2

  4686.                           "=r"(r1)          // %3

  4687.                         : "0"(output0_tm),

  4688.                           "1"(output1_tm),

  4689.                           "2"(r0),

  4690.                           "3"(r1),

  4691.                           "w"(_k00),         // %8

  4692.                           "w"(_k00n),        // %9

  4693.                           "w"(_k01),         // %10

  4694.                           "w"(_k01n),        // %11

  4695.                           "w"(_k10),         // %12

  4696.                           "w"(_k10n),        // %13

  4697.                           "w"(_k11),         // %14

  4698.                           "w"(_k11n)         // %15

  4699.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  4700.                     );

  4701. #endif // __aarch64__

  4702. #else

  4703.                     for (int m=0; m<8; m++)

  4704.                     {

  4705.                         output0_tm[m] += r0[m] * k00[m];

  4706.                         output0_tm[m] += r1[m] * k01[m];

  4707.                         output1_tm[m] += r0[m] * k10[m];

  4708.                         output1_tm[m] += r1[m] * k11[m];

  4709.                     }

  4710. 

  4711.                     r0 += 8;

  4712.                     r1 += 8;

  4713.                     output0_tm += 8;

  4714.                     output1_tm += 8;

  4715. #endif // __ARM_NEON

  4716.                 }

  4717. 

  4718. #if __ARM_NEON

  4719. #if __aarch64__

  4720.                 k00 += 8;

  4721.                 k01 += 8;

  4722.                 k10 += 8;

  4723.                 k11 += 8;

  4724. #endif // __aarch64__

  4725. #else

  4726.                 k00 += 8;

  4727.                 k01 += 8;

  4728.                 k10 += 8;

  4729.                 k11 += 8;

  4730. #endif // __ARM_NEON

  4731.                 }

  4732.             }

  4733. 

  4734.             for (; q<inch; q++)

  4735.             {

  4736.                 const float* r0 = bottom_blob_tm.channel(q);

  4737. 

  4738.                 const float* k00 = kernel0_tm.row(q);

  4739.                 const float* k10 = kernel1_tm.row(q);

  4740. 

  4741.                 float* output0_tm = out0_tm;

  4742.                 float* output1_tm = out1_tm;

  4743. 

  4744.                 for (int r=0; r<8; r++)

  4745.                 {

  4746. #if __ARM_NEON

  4747. #if __aarch64__

  4748.                 float32x4_t _k00 = vld1q_f32(k00);

  4749.                 float32x4_t _k00n = vld1q_f32(k00+4);

  4750.                 float32x4_t _k10 = vld1q_f32(k10);

  4751.                 float32x4_t _k10n = vld1q_f32(k10+4);

  4752. #else

  4753.                 float32x4_t _k00;

  4754.                 float32x4_t _k00n;

  4755.                 float32x4_t _k10;

  4756.                 float32x4_t _k10n;

  4757. 

  4758.                 asm volatile(

  4759.                     "pld        [%0, #256]              \n"

  4760.                     "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"

  4761.                     "pld        [%1, #256]              \n"

  4762.                     "vld1.f32   {%e4-%f4}, [%1 :128]!   \n"

  4763.                     "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"

  4764.                     "vld1.f32   {%e5-%f5}, [%1 :128]!   \n"

  4765.                     : "=r"(k00),    // %0

  4766.                       "=r"(k10),    // %1

  4767.                       "=w"(_k00),   // %2

  4768.                       "=w"(_k00n),  // %3

  4769.                       "=w"(_k10),   // %4

  4770.                       "=w"(_k10n)   // %5

  4771.                     : "0"(k00),

  4772.                       "1"(k10)

  4773.                     : "cc", "memory"

  4774.                 );

  4775. #endif // __aarch64__

  4776. #endif // __ARM_NEON

  4777. 

  4778.                 // tile

  4779. #if __ARM_NEON

  4780.                 int nn = tiles >> 2;

  4781.                 int remain = tiles & 3;

  4782. #else

  4783.                 int remain = tiles;

  4784. #endif // __ARM_NEON

  4785. 

  4786. #if __ARM_NEON

  4787. #if __aarch64__

  4788.                 for (; nn>0; nn--)

  4789.                 {

  4790.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  4791.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  4792. 

  4793.                     float32x4_t _output1_tm = vld1q_f32(output1_tm);

  4794.                     float32x4_t _output1_tmn = vld1q_f32(output1_tm+4);

  4795. 

  4796.                     float32x4_t _r0 = vld1q_f32(r0);

  4797.                     float32x4_t _r0n = vld1q_f32(r0+4);

  4798. 

  4799.                     r0 += 8;

  4800. 

  4801.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4802.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4803. 

  4804.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4805.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4806. 

  4807.                     vst1q_f32(output0_tm, _output0_tm);

  4808.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4809. 

  4810.                     vst1q_f32(output1_tm, _output1_tm);

  4811.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4812. 

  4813.                     output0_tm += 8;

  4814.                     output1_tm += 8;

  4815. 

  4816.                     _output0_tm = vld1q_f32(output0_tm);

  4817.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4818. 

  4819.                     _output1_tm = vld1q_f32(output1_tm);

  4820.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4821. 

  4822.                     _r0 = vld1q_f32(r0);

  4823.                     _r0n = vld1q_f32(r0+4);

  4824. 

  4825.                     r0 += 8;

  4826. 

  4827.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4828.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4829. 

  4830.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4831.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4832. 

  4833.                     vst1q_f32(output0_tm, _output0_tm);

  4834.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4835. 

  4836.                     vst1q_f32(output1_tm, _output1_tm);

  4837.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4838. 

  4839.                     output0_tm += 8;

  4840.                     output1_tm += 8;

  4841. 

  4842.                     _output0_tm = vld1q_f32(output0_tm);

  4843.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4844. 

  4845.                     _output1_tm = vld1q_f32(output1_tm);

  4846.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4847. 

  4848.                     _r0 = vld1q_f32(r0);

  4849.                     _r0n = vld1q_f32(r0+4);

  4850. 

  4851.                     r0 += 8;

  4852. 

  4853.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4854.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4855. 

  4856.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4857.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4858. 

  4859.                     vst1q_f32(output0_tm, _output0_tm);

  4860.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4861. 

  4862.                     vst1q_f32(output1_tm, _output1_tm);

  4863.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4864. 

  4865.                     output0_tm += 8;

  4866.                     output1_tm += 8;

  4867. 

  4868.                     _output0_tm = vld1q_f32(output0_tm);

  4869.                     _output0_tmn = vld1q_f32(output0_tm+4);

  4870. 

  4871.                     _output1_tm = vld1q_f32(output1_tm);

  4872.                     _output1_tmn = vld1q_f32(output1_tm+4);

  4873. 

  4874.                     _r0 = vld1q_f32(r0);

  4875.                     _r0n = vld1q_f32(r0+4);

  4876. 

  4877.                     r0 += 8;

  4878. 

  4879.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  4880.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  4881. 

  4882.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  4883.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  4884. 

  4885.                     vst1q_f32(output0_tm, _output0_tm);

  4886.                     vst1q_f32(output0_tm+4, _output0_tmn);

  4887. 

  4888.                     vst1q_f32(output1_tm, _output1_tm);

  4889.                     vst1q_f32(output1_tm+4, _output1_tmn);

  4890. 

  4891.                     output0_tm += 8;

  4892.                     output1_tm += 8;

  4893.                 }

  4894. #else

  4895.                 if (nn > 0)

  4896.                 {

  4897.                     asm volatile(

  4898.                         "0:                                 \n"

  4899. 

  4900.                         "pld        [%3, #256]              \n"

  4901.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4902. 

  4903.                         "pld        [%1, #256]              \n"

  4904.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4905. 

  4906.                         "vmla.f32   q8, q12, %q8            \n"

  4907.                         "vmla.f32   q9, q13, %q9            \n"

  4908. 

  4909.                         "pld        [%2, #256]              \n"

  4910.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4911. 

  4912.                         "vmla.f32   q10, q12, %q10          \n"

  4913.                         "vmla.f32   q11, q13, %q11          \n"

  4914. 

  4915.                         "pld        [%3, #256]              \n"

  4916.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4917. 

  4918.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4919. 

  4920.                         "pld        [%1, #256]              \n"

  4921.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4922. 

  4923.                         "vmla.f32   q8, q12, %q8            \n"

  4924.                         "vmla.f32   q9, q13, %q9            \n"

  4925. 

  4926.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4927. 

  4928.                         "pld        [%2, #256]              \n"

  4929.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4930. 

  4931.                         "vmla.f32   q10, q12, %q10          \n"

  4932.                         "vmla.f32   q11, q13, %q11          \n"

  4933. 

  4934.                         "pld        [%3, #256]              \n"

  4935.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4936. 

  4937.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4938. 

  4939.                         "pld        [%1, #256]              \n"

  4940.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4941. 

  4942.                         "vmla.f32   q8, q12, %q8            \n"

  4943.                         "vmla.f32   q9, q13, %q9            \n"

  4944. 

  4945.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4946. 

  4947.                         "pld        [%2, #256]              \n"

  4948.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4949. 

  4950.                         "vmla.f32   q10, q12, %q10          \n"

  4951.                         "vmla.f32   q11, q13, %q11          \n"

  4952. 

  4953.                         "pld        [%3, #256]              \n"

  4954.                         "vld1.f32   {d24-d27}, [%3 :128]!   \n"// q12 q13 = _r0

  4955. 

  4956.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4957. 

  4958.                         "pld        [%1, #256]              \n"

  4959.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  4960. 

  4961.                         "vmla.f32   q8, q12, %q8            \n"

  4962.                         "vmla.f32   q9, q13, %q9            \n"

  4963. 

  4964.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4965. 

  4966.                         "pld        [%2, #256]              \n"

  4967.                         "vld1.f32   {d20-d23}, [%2 :128]    \n"// q10 q11 = _output1_tm

  4968. 

  4969.                         "vmla.f32   q10, q12, %q10          \n"

  4970.                         "vmla.f32   q11, q13, %q11          \n"

  4971. 

  4972.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  4973.                         "vst1.f32   {d20-d23}, [%2 :128]!   \n"

  4974. 

  4975.                         "subs       %0, #1                  \n"

  4976. 

  4977.                         "bne        0b                      \n"

  4978. 

  4979.                         : "=r"(nn),         // %0

  4980.                           "=r"(output0_tm), // %1

  4981.                           "=r"(output1_tm), // %2

  4982.                           "=r"(r0)          // %3

  4983.                         : "0"(nn),

  4984.                           "1"(output0_tm),

  4985.                           "2"(output1_tm),

  4986.                           "3"(r0),

  4987.                           "w"(_k00),        // %8

  4988.                           "w"(_k00n),       // %9

  4989.                           "w"(_k10),        // %10

  4990.                           "w"(_k10n)        // %11

  4991.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13"

  4992.                     );

  4993.                 }

  4994. #endif // __aarch64__

  4995. #endif // __ARM_NEON

  4996.                 for (; remain>0; remain--)

  4997.                 {

  4998. #if __ARM_NEON

  4999. #if __aarch64__

  5000.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  5001.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  5002. 

  5003.                     float32x4_t _output1_tm = vld1q_f32(output1_tm);

  5004.                     float32x4_t _output1_tmn = vld1q_f32(output1_tm+4);

  5005. 

  5006.                     float32x4_t _r0 = vld1q_f32(r0);

  5007.                     float32x4_t _r0n = vld1q_f32(r0+4);

  5008. 

  5009.                     r0 += 8;

  5010. 

  5011.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5012.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5013. 

  5014.                     _output1_tm = vmlaq_f32(_output1_tm, _r0, _k10);

  5015.                     _output1_tmn = vmlaq_f32(_output1_tmn, _r0n, _k10n);

  5016. 

  5017.                     vst1q_f32(output0_tm, _output0_tm);

  5018.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5019. 

  5020.                     vst1q_f32(output1_tm, _output1_tm);

  5021.                     vst1q_f32(output1_tm+4, _output1_tmn);

  5022. 

  5023.                     output0_tm += 8;

  5024.                     output1_tm += 8;

  5025. #else

  5026.                     asm volatile(

  5027.                         "pld        [%2, #256]              \n"

  5028.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  5029. 

  5030.                         "pld        [%0, #256]              \n"

  5031.                         "vld1.f32   {d16-d19}, [%0 :128]    \n"// q8 q9 = _output0_tm

  5032. 

  5033.                         "vmla.f32   q8, q12, %q6            \n"

  5034.                         "vmla.f32   q9, q13, %q7            \n"

  5035. 

  5036.                         "pld        [%1, #256]              \n"

  5037.                         "vld1.f32   {d20-d23}, [%1 :128]    \n"// q10 q11 = _output1_tm

  5038. 

  5039.                         "vmla.f32   q10, q12, %q8           \n"

  5040.                         "vmla.f32   q11, q13, %q9           \n"

  5041. 

  5042.                         "vst1.f32   {d16-d19}, [%0 :128]!   \n"

  5043.                         "vst1.f32   {d20-d23}, [%1 :128]!   \n"

  5044. 

  5045.                         : "=r"(output0_tm), // %0

  5046.                           "=r"(output1_tm), // %1

  5047.                           "=r"(r0)          // %2

  5048.                         : "0"(output0_tm),

  5049.                           "1"(output1_tm),

  5050.                           "2"(r0),

  5051.                           "w"(_k00),        // %6

  5052.                           "w"(_k00n),       // %7

  5053.                           "w"(_k10),        // %8

  5054.                           "w"(_k10n)        // %9

  5055.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13"

  5056.                     );

  5057. #endif // __aarch64__

  5058. #else

  5059.                     for (int m=0; m<8; m++)

  5060.                     {

  5061.                         output0_tm[m] += r0[m] * k00[m];

  5062.                         output1_tm[m] += r0[m] * k10[m];

  5063.                     }

  5064. 

  5065.                     r0 += 8;

  5066.                     output0_tm += 8;

  5067.                     output1_tm += 8;

  5068. #endif // __ARM_NEON

  5069.                 }

  5070. 

  5071. #if __ARM_NEON

  5072. #if __aarch64__

  5073.                 k00 += 8;

  5074.                 k10 += 8;

  5075. #endif // __aarch64__

  5076. #else

  5077.                 k00 += 8;

  5078.                 k10 += 8;

  5079. #endif // __ARM_NEON

  5080.                 }

  5081.             }

  5082.         }

  5083. 

  5084.         #pragma omp parallel for

  5085.         for (int p = remain_outch_start; p<outch; p++)

  5086.         {

  5087.             Mat out0_tm = top_blob_tm.channel(p);

  5088.             const Mat kernel0_tm = kernel_tm.channel(p);

  5089. 

  5090.             out0_tm.fill(0.f);

  5091. 

  5092.             int q = 0;

  5093.             for (; q+1<inch; q+=2)

  5094.             {

  5095.                 const float* r0 = bottom_blob_tm.channel(q);

  5096.                 const float* r1 = bottom_blob_tm.channel(q+1);

  5097. 

  5098.                 const float* k00 = kernel0_tm.row(q);

  5099.                 const float* k01 = kernel0_tm.row(q+1);

  5100. 

  5101.                 float* output0_tm = out0_tm;

  5102. 

  5103.                 for (int r=0; r<8; r++)

  5104.                 {

  5105. #if __ARM_NEON

  5106. #if __aarch64__

  5107.                 float32x4_t _k00 = vld1q_f32(k00);

  5108.                 float32x4_t _k00n = vld1q_f32(k00+4);

  5109.                 float32x4_t _k01 = vld1q_f32(k01);

  5110.                 float32x4_t _k01n = vld1q_f32(k01+4);

  5111. #else

  5112.                 float32x4_t _k00;

  5113.                 float32x4_t _k00n;

  5114.                 float32x4_t _k01;

  5115.                 float32x4_t _k01n;

  5116. 

  5117.                 asm volatile(

  5118.                     "pld        [%0, #256]              \n"

  5119.                     "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"

  5120.                     "pld        [%1, #256]              \n"

  5121.                     "vld1.f32   {%e4-%f4}, [%1 :128]!   \n"

  5122. 

  5123.                     "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"

  5124.                     "vld1.f32   {%e5-%f5}, [%1 :128]!   \n"

  5125.                     : "=r"(k00),    // %0

  5126.                       "=r"(k01),    // %1

  5127.                       "=w"(_k00),   // %2

  5128.                       "=w"(_k00n),  // %3

  5129.                       "=w"(_k01),   // %4

  5130.                       "=w"(_k01n)   // %5

  5131.                     : "0"(k00),

  5132.                       "1"(k01)

  5133.                     : "cc", "memory"

  5134.                 );

  5135. #endif // __aarch64__

  5136. #endif // __ARM_NEON

  5137. 

  5138.                 // tile

  5139. #if __ARM_NEON

  5140.                 int nn = tiles >> 2;

  5141.                 int remain = tiles & 3;

  5142. #else

  5143.                 int remain = tiles;

  5144. #endif // __ARM_NEON

  5145. 

  5146. #if __ARM_NEON

  5147. #if __aarch64__

  5148.                 for (; nn>0; nn--)

  5149.                 {

  5150.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  5151.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  5152. 

  5153.                     float32x4_t _r0 = vld1q_f32(r0);

  5154.                     float32x4_t _r0n = vld1q_f32(r0+4);

  5155.                     float32x4_t _r1 = vld1q_f32(r1);

  5156.                     float32x4_t _r1n = vld1q_f32(r1+4);

  5157. 

  5158.                     r0 += 8;

  5159.                     r1 += 8;

  5160. 

  5161.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5162.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5163.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  5164.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  5165. 

  5166.                     vst1q_f32(output0_tm, _output0_tm);

  5167.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5168. 

  5169.                     output0_tm += 8;

  5170. 

  5171.                     _output0_tm = vld1q_f32(output0_tm);

  5172.                     _output0_tmn = vld1q_f32(output0_tm+4);

  5173. 

  5174.                     _r0 = vld1q_f32(r0);

  5175.                     _r0n = vld1q_f32(r0+4);

  5176.                     _r1 = vld1q_f32(r1);

  5177.                     _r1n = vld1q_f32(r1+4);

  5178. 

  5179.                     r0 += 8;

  5180.                     r1 += 8;

  5181. 

  5182.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5183.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5184.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  5185.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  5186. 

  5187.                     vst1q_f32(output0_tm, _output0_tm);

  5188.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5189. 

  5190.                     output0_tm += 8;

  5191. 

  5192.                     _output0_tm = vld1q_f32(output0_tm);

  5193.                     _output0_tmn = vld1q_f32(output0_tm+4);

  5194. 

  5195.                     _r0 = vld1q_f32(r0);

  5196.                     _r0n = vld1q_f32(r0+4);

  5197.                     _r1 = vld1q_f32(r1);

  5198.                     _r1n = vld1q_f32(r1+4);

  5199. 

  5200.                     r0 += 8;

  5201.                     r1 += 8;

  5202. 

  5203.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5204.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5205.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  5206.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  5207. 

  5208.                     vst1q_f32(output0_tm, _output0_tm);

  5209.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5210. 

  5211.                     output0_tm += 8;

  5212. 

  5213.                     _output0_tm = vld1q_f32(output0_tm);

  5214.                     _output0_tmn = vld1q_f32(output0_tm+4);

  5215. 

  5216.                     _r0 = vld1q_f32(r0);

  5217.                     _r0n = vld1q_f32(r0+4);

  5218.                     _r1 = vld1q_f32(r1);

  5219.                     _r1n = vld1q_f32(r1+4);

  5220. 

  5221.                     r0 += 8;

  5222.                     r1 += 8;

  5223. 

  5224.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5225.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5226.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  5227.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  5228. 

  5229.                     vst1q_f32(output0_tm, _output0_tm);

  5230.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5231. 

  5232.                     output0_tm += 8;

  5233.                 }

  5234. #else

  5235.                 if (nn > 0)

  5236.                 {

  5237.                     asm volatile(

  5238.                         "0:                                 \n"

  5239. 

  5240.                         "pld        [%2, #256]              \n"

  5241.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  5242. 

  5243.                         "pld        [%1, #256]              \n"

  5244.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  5245. 

  5246.                         "vmla.f32   q8, q12, %q8            \n"

  5247.                         "vmla.f32   q9, q13, %q9            \n"

  5248. 

  5249.                         "pld        [%3, #256]              \n"

  5250.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  5251. 

  5252.                         "vmla.f32   q8, q14, %q10           \n"

  5253.                         "vmla.f32   q9, q15, %q11           \n"

  5254. 

  5255.                         "pld        [%2, #256]              \n"

  5256.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  5257. 

  5258.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  5259. 

  5260.                         "pld        [%1, #256]              \n"

  5261.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  5262. 

  5263.                         "vmla.f32   q8, q12, %q8            \n"

  5264.                         "vmla.f32   q9, q13, %q9            \n"

  5265. 

  5266.                         "pld        [%3, #256]              \n"

  5267.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  5268. 

  5269.                         "vmla.f32   q8, q14, %q10           \n"

  5270.                         "vmla.f32   q9, q15, %q11           \n"

  5271. 

  5272.                         "pld        [%2, #256]              \n"

  5273.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  5274. 

  5275.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  5276. 

  5277.                         "pld        [%1, #256]              \n"

  5278.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  5279. 

  5280.                         "vmla.f32   q8, q12, %q8            \n"

  5281.                         "vmla.f32   q9, q13, %q9            \n"

  5282. 

  5283.                         "pld        [%3, #256]              \n"

  5284.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  5285. 

  5286.                         "vmla.f32   q8, q14, %q10           \n"

  5287.                         "vmla.f32   q9, q15, %q11           \n"

  5288. 

  5289.                         "pld        [%2, #256]              \n"

  5290.                         "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0

  5291. 

  5292.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  5293. 

  5294.                         "pld        [%1, #256]              \n"

  5295.                         "vld1.f32   {d16-d19}, [%1 :128]    \n"// q8 q9 = _output0_tm

  5296. 

  5297.                         "vmla.f32   q8, q12, %q8            \n"

  5298.                         "vmla.f32   q9, q13, %q9            \n"

  5299. 

  5300.                         "pld        [%3, #256]              \n"

  5301.                         "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1

  5302. 

  5303.                         "vmla.f32   q8, q14, %q10           \n"

  5304.                         "vmla.f32   q9, q15, %q11           \n"

  5305. 

  5306.                         "vst1.f32   {d16-d19}, [%1 :128]!   \n"

  5307. 

  5308.                         "subs       %0, #1                  \n"

  5309. 

  5310.                         "bne        0b                      \n"

  5311. 

  5312.                         : "=r"(nn),         // %0

  5313.                           "=r"(output0_tm), // %1

  5314.                           "=r"(r0),         // %2

  5315.                           "=r"(r1)          // %3

  5316.                         : "0"(nn),

  5317.                           "1"(output0_tm),

  5318.                           "2"(r0),

  5319.                           "3"(r1),

  5320.                           "w"(_k00),        // %8

  5321.                           "w"(_k00n),       // %9

  5322.                           "w"(_k01),        // %10

  5323.                           "w"(_k01n)        // %11

  5324.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  5325.                     );

  5326.                 }

  5327. #endif // __aarch64__

  5328. #endif // __ARM_NEON

  5329.                 for (; remain>0; remain--)

  5330.                 {

  5331. #if __ARM_NEON

  5332. #if __aarch64__

  5333.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  5334.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  5335. 

  5336.                     float32x4_t _r0 = vld1q_f32(r0);

  5337.                     float32x4_t _r0n = vld1q_f32(r0+4);

  5338.                     float32x4_t _r1 = vld1q_f32(r1);

  5339.                     float32x4_t _r1n = vld1q_f32(r1+4);

  5340. 

  5341.                     r0 += 8;

  5342.                     r1 += 8;

  5343. 

  5344.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5345.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5346.                     _output0_tm = vmlaq_f32(_output0_tm, _r1, _k01);

  5347.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k01n);

  5348. 

  5349.                     vst1q_f32(output0_tm, _output0_tm);

  5350.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5351. 

  5352.                     output0_tm += 8;

  5353. #else

  5354.                     asm volatile(

  5355.                         "pld        [%1, #256]              \n"

  5356.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  5357. 

  5358.                         "pld        [%0, #256]              \n"

  5359.                         "vld1.f32   {d16-d19}, [%0 :128]    \n"// q8 q9 = _output0_tm

  5360. 

  5361.                         "vmla.f32   q8, q12, %q6            \n"

  5362.                         "vmla.f32   q9, q13, %q7            \n"

  5363. 

  5364.                         "pld        [%2, #256]              \n"

  5365.                         "vld1.f32   {d28-d31}, [%2 :128]!   \n"// q14 q15 = _r1

  5366. 

  5367.                         "vmla.f32   q8, q14, %q8            \n"

  5368.                         "vmla.f32   q9, q15, %q9            \n"

  5369. 

  5370.                         "vst1.f32   {d16-d19}, [%0 :128]!   \n"

  5371. 

  5372.                         : "=r"(output0_tm), // %0

  5373.                           "=r"(r0),         // %1

  5374.                           "=r"(r1)          // %2

  5375.                         : "0"(output0_tm),

  5376.                           "1"(r0),

  5377.                           "2"(r1),

  5378.                           "w"(_k00),        // %6

  5379.                           "w"(_k00n),       // %7

  5380.                           "w"(_k01),        // %8

  5381.                           "w"(_k01n)        // %9

  5382.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  5383.                     );

  5384. #endif // __aarch64__

  5385. #else

  5386.                     for (int m=0; m<8; m++)

  5387.                     {

  5388.                         output0_tm[m] += r0[m] * k00[m];

  5389.                         output0_tm[m] += r1[m] * k01[m];

  5390.                     }

  5391. 

  5392.                     r0 += 8;

  5393.                     r1 += 8;

  5394.                     output0_tm += 8;

  5395. #endif // __ARM_NEON

  5396.                 }

  5397. 

  5398. #if __ARM_NEON

  5399. #if __aarch64__

  5400.                 k00 += 8;

  5401.                 k01 += 8;

  5402. #endif // __aarch64__

  5403. #else

  5404.                 k00 += 8;

  5405.                 k01 += 8;

  5406. #endif // __ARM_NEON

  5407.                 }

  5408.             }

  5409. 

  5410.             for (; q<inch; q++)

  5411.             {

  5412.                 const float* r0 = bottom_blob_tm.channel(q);

  5413. 

  5414.                 const float* k00 = kernel0_tm.row(q);

  5415. 

  5416.                 float* output0_tm = out0_tm;

  5417. 

  5418.                 for (int r=0; r<8; r++)

  5419.                 {

  5420. #if __ARM_NEON

  5421. #if __aarch64__

  5422.                 float32x4_t _k00 = vld1q_f32(k00);

  5423.                 float32x4_t _k00n = vld1q_f32(k00+4);

  5424. #else

  5425.                 float32x4_t _k00;

  5426.                 float32x4_t _k00n;

  5427. 

  5428.                 asm volatile(

  5429.                     "pld        [%0, #256]              \n"

  5430.                     "vld1.f32   {%e1-%f1}, [%0 :128]!   \n"

  5431.                     "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"

  5432.                     : "=r"(k00),    // %0

  5433.                       "=w"(_k00),   // %1

  5434.                       "=w"(_k00n)   // %2

  5435.                     : "0"(k00)

  5436.                     : "cc", "memory"

  5437.                 );

  5438. #endif // __aarch64__

  5439. #endif // __ARM_NEON

  5440. 

  5441.                 // tile

  5442.                 for (int i=0; i<tiles; i++)

  5443.                 {

  5444. #if __ARM_NEON

  5445. #if __aarch64__

  5446.                     float32x4_t _output0_tm = vld1q_f32(output0_tm);

  5447.                     float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);

  5448. 

  5449.                     float32x4_t _r0 = vld1q_f32(r0);

  5450.                     float32x4_t _r0n = vld1q_f32(r0+4);

  5451. 

  5452.                     r0 += 8;

  5453. 

  5454.                     _output0_tm = vmlaq_f32(_output0_tm, _r0, _k00);

  5455.                     _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k00n);

  5456. 

  5457.                     vst1q_f32(output0_tm, _output0_tm);

  5458.                     vst1q_f32(output0_tm+4, _output0_tmn);

  5459. 

  5460.                     output0_tm += 8;

  5461. #else

  5462.                     asm volatile(

  5463.                         "pld        [%1, #256]              \n"

  5464.                         "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0

  5465. 

  5466.                         "pld        [%0, #256]              \n"

  5467.                         "vld1.f32   {d16-d19}, [%0 :128]    \n"// q8 q9 = _output0_tm

  5468. 

  5469.                         "vmla.f32   q8, q12, %q4            \n"

  5470.                         "vmla.f32   q9, q13, %q5            \n"

  5471. 

  5472.                         "vst1.f32   {d16-d19}, [%0 :128]!   \n"

  5473.                         : "=r"(output0_tm), // %0

  5474.                           "=r"(r0)          // %1

  5475.                         : "0"(output0_tm),

  5476.                           "1"(r0),

  5477.                           "w"(_k00),        // %4

  5478.                           "w"(_k00n)        // %5

  5479.                         : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  5480.                     );

  5481. #endif // __aarch64__

  5482. #else

  5483.                     for (int m=0; m<8; m++)

  5484.                     {

  5485.                         output0_tm[m] += r0[m] * k00[m];

  5486.                     }

  5487. 

  5488.                     r0 += 8;

  5489.                     output0_tm += 8;

  5490. #endif // __ARM_NEON

  5491.                 }

  5492. 

  5493. #if __ARM_NEON

  5494. #if __aarch64__

  5495.                 k00 += 8;

  5496. #endif // __aarch64__

  5497. #else

  5498.                 k00 += 8;

  5499. #endif // __ARM_NEON

  5500.                 }

  5501.             }

  5502.         }

  5503.     }

  5504.     bottom_blob_tm = Mat();

  5505.     // END dot

  5506. 

  5507.     // BEGIN transform output

  5508.     Mat top_blob_bordered;

  5509.     top_blob_bordered.create(outw, outh, outch);

  5510.     {

  5511. //         const float otm[6][8] = {

  5512. //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  5513. //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  5514. //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  5515. //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  5516. //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  5517. //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  5518. //         };

  5519. 

  5520.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  5521.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  5522.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  5523.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  5524.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  5525.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  5526. 

  5527.         int w_tm = outw / 6 * 8;

  5528.         int h_tm = outh / 6 * 8;

  5529.         const int tiles = w_tm/8 * h_tm/8;

  5530. 

  5531.         #pragma omp parallel for

  5532.         for (int p = 0; p<outch; p++)

  5533.         {

  5534.             const Mat out0_tm = top_blob_tm.channel(p);

  5535.             Mat out0 = top_blob_bordered.channel(p);

  5536. 

  5537.             const float bias0 = bias ? bias[p] : 0.f;

  5538. 

  5539.             float tmp[6][8];

  5540. 

  5541.             // tile

  5542.             for (int i=0; i<outh/6; i++)

  5543.             {

  5544.                 for (int j=0; j<outw/6; j++)

  5545.                 {

  5546.                     const float* output0_tm0 = out0_tm.row(i * w_tm/8 + j);

  5547.                     const float* output0_tm1 = out0_tm.row(i * w_tm/8 + j + tiles);

  5548.                     const float* output0_tm2 = out0_tm.row(i * w_tm/8 + j + tiles * 2);

  5549.                     const float* output0_tm3 = out0_tm.row(i * w_tm/8 + j + tiles * 3);

  5550.                     const float* output0_tm4 = out0_tm.row(i * w_tm/8 + j + tiles * 4);

  5551.                     const float* output0_tm5 = out0_tm.row(i * w_tm/8 + j + tiles * 5);

  5552.                     const float* output0_tm6 = out0_tm.row(i * w_tm/8 + j + tiles * 6);

  5553.                     const float* output0_tm7 = out0_tm.row(i * w_tm/8 + j + tiles * 7);

  5554.                     float* output0 = out0.row(i * 6) + j * 6;

  5555. 

  5556.                     const float* output0_tms[8] = { output0_tm0, output0_tm1, output0_tm2, output0_tm3, output0_tm4, output0_tm5, output0_tm6, output0_tm7 };

  5557. 

  5558.                     for (int m=0; m<8; m++)

  5559.                     {

  5560.                         const float* output0_tm = output0_tms[m];

  5561. 

  5562.                         float tmp024a = output0_tm[1] + output0_tm[2];

  5563.                         float tmp135a = output0_tm[1] - output0_tm[2];

  5564. 

  5565.                         float tmp024b = output0_tm[3] + output0_tm[4];

  5566.                         float tmp135b = output0_tm[3] - output0_tm[4];

  5567. 

  5568.                         float tmp024c = output0_tm[5] + output0_tm[6];

  5569.                         float tmp135c = output0_tm[5] - output0_tm[6];

  5570. 

  5571.                         tmp[0][m] = output0_tm[0] + tmp024a + tmp024b + tmp024c * 32;

  5572.                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  5573.                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  5574. 

  5575.                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  5576.                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  5577.                         tmp[5][m] = output0_tm[7] + tmp135a + tmp135b * 32 + tmp135c;

  5578.                     }

  5579. 

  5580.                     for (int m=0; m<6; m++)

  5581.                     {

  5582.                         const float* tmp0 = tmp[m];

  5583. 

  5584.                         float tmp024a = tmp0[1] + tmp0[2];

  5585.                         float tmp135a = tmp0[1] - tmp0[2];

  5586. 

  5587.                         float tmp024b = tmp0[3] + tmp0[4];

  5588.                         float tmp135b = tmp0[3] - tmp0[4];

  5589. 

  5590.                         float tmp024c = tmp0[5] + tmp0[6];

  5591.                         float tmp135c = tmp0[5] - tmp0[6];

  5592. 

  5593.                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  5594.                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  5595.                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  5596. 

  5597.                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  5598.                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  5599.                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  5600. 

  5601.                         output0 += outw;

  5602.                     }

  5603.                 }

  5604.             }

  5605.         }

  5606.     }

  5607.     // END transform output

  5608. 

  5609.     // cut result pad

  5610.     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w);

  5611. }

  5612. #endif

  5613. 

  5614. static void conv3x3s1_winograd64_neon4(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias, const Option& opt)

  5615. {

  5616.     int w = bottom_blob.w;

  5617.     int h = bottom_blob.h;

  5618.     int inch = bottom_blob.c;

  5619. 

  5620.     int outw = top_blob.w;

  5621.     int outh = top_blob.h;

  5622.     int outch = top_blob.c;

  5623. 

  5624.     // pad to 6n+2

  5625.     Mat bottom_blob_bordered = bottom_blob;

  5626. 

  5627.     outw = (outw + 5) / 6 * 6;

  5628.     outh = (outh + 5) / 6 * 6;

  5629. 

  5630.     w = outw + 2;

  5631.     h = outh + 2;

  5632.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt.workspace_allocator, opt.num_threads);

  5633. 

  5634.     const float* bias = _bias;

  5635. 

  5636.     // BEGIN transform input

  5637.     Mat bottom_blob_tm;

  5638.     {

  5639.         int w_tm = outw / 6 * 8;

  5640.         int h_tm = outh / 6 * 8;

  5641.         bottom_blob_tm.create(4, 16 * w_tm/8 * h_tm/8, inch, 4u, opt.workspace_allocator);

  5642.         const int tiles = w_tm/8 * h_tm/8;

  5643. 

  5644. //         const float itm[8][8] = {

  5645. //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  5646. //

  5647. //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  5648. //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  5649. //

  5650. //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  5651. //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  5652. //

  5653. //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  5654. //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  5655. //

  5656. //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  5657. //         };

  5658. 

  5659.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  5660.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  5661. 

  5662.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  5663.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  5664. 

  5665.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  5666.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  5667. 

  5668.         // reuse r04 * 1.25

  5669.         // reuse r03 * 2.5

  5670.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  5671.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  5672. 

  5673. #if __ARM_NEON

  5674.         const float coeff[8] = {

  5675.             0.25f, 0.5f, -1.25f,   2.f,

  5676.             -2.5f,  4.f,  4.25f, 5.25f

  5677.         };

  5678.         float32x4_t _coeff0 = vld1q_f32(coeff);

  5679.         float32x4_t _coeff1 = vld1q_f32(coeff+4);

  5680. #endif // __ARM_NEON

  5681. 

  5682.         #pragma omp parallel for num_threads(opt.num_threads)

  5683.         for (int q = 0; q<inch; q++)

  5684.         {

  5685.             const Mat img0 = bottom_blob_bordered.channel(q);

  5686.             Mat img0_tm = bottom_blob_tm.channel(q);

  5687. 

  5688.             float tmp[8][8];

  5689. 

  5690.             // tile

  5691.             for (int i=0; i<h_tm/8; i++)

  5692.             {

  5693.                 for (int j=0; j<w_tm/8; j++)

  5694.                 {

  5695. #if __ARM_NEON

  5696.                     const float* r0 = img0.row(i * 6) + j * 6;

  5697.                     const float* r1 = r0 + w;

  5698.                     const float* r2 = r0 + w*2;

  5699.                     const float* r3 = r0 + w*3;

  5700. 

  5701. #if __aarch64__

  5702.                     for (int m=0; m+3<8; m+=4)

  5703.                     {

  5704.                         float32x4_t _r0_0123 = vld1q_f32(r0);

  5705.                         float32x4_t _r0_4567 = vld1q_f32(r0+4);

  5706.                         float32x4_t _r1_0123 = vld1q_f32(r1);

  5707.                         float32x4_t _r1_4567 = vld1q_f32(r1+4);

  5708.                         float32x4_t _r2_0123 = vld1q_f32(r2);

  5709.                         float32x4_t _r2_4567 = vld1q_f32(r2+4);

  5710.                         float32x4_t _r3_0123 = vld1q_f32(r3);

  5711.                         float32x4_t _r3_4567 = vld1q_f32(r3+4);

  5712. 

  5713.                         float32x4x2_t _r01_00221133 = vtrnq_f32(_r0_0123, _r1_0123);

  5714.                         float32x4x2_t _r01_44665577 = vtrnq_f32(_r0_4567, _r1_4567);

  5715.                         float32x4x2_t _r23_00221133 = vtrnq_f32(_r2_0123, _r3_0123);

  5716.                         float32x4x2_t _r23_44665577 = vtrnq_f32(_r2_4567, _r3_4567);

  5717. 

  5718.                         // no vswp intrinsic  :(

  5719.                         float32x4_t _r_00 = vcombine_f32(vget_low_f32(_r01_00221133.val[0]), vget_low_f32(_r23_00221133.val[0]));

  5720.                         float32x4_t _r_11 = vcombine_f32(vget_low_f32(_r01_00221133.val[1]), vget_low_f32(_r23_00221133.val[1]));

  5721.                         float32x4_t _r_22 = vcombine_f32(vget_high_f32(_r01_00221133.val[0]), vget_high_f32(_r23_00221133.val[0]));

  5722.                         float32x4_t _r_33 = vcombine_f32(vget_high_f32(_r01_00221133.val[1]), vget_high_f32(_r23_00221133.val[1]));

  5723.                         float32x4_t _r_44 = vcombine_f32(vget_low_f32(_r01_44665577.val[0]), vget_low_f32(_r23_44665577.val[0]));

  5724.                         float32x4_t _r_55 = vcombine_f32(vget_low_f32(_r01_44665577.val[1]), vget_low_f32(_r23_44665577.val[1]));

  5725.                         float32x4_t _r_66 = vcombine_f32(vget_high_f32(_r01_44665577.val[0]), vget_high_f32(_r23_44665577.val[0]));

  5726.                         float32x4_t _r_77 = vcombine_f32(vget_high_f32(_r01_44665577.val[1]), vget_high_f32(_r23_44665577.val[1]));

  5727. 

  5728.                         float32x4_t _r_0_m_6 = vsubq_f32(_r_00, _r_66);

  5729.                         float32x4_t _r_7_m_1 = vsubq_f32(_r_77, _r_11);

  5730. 

  5731.                         float32x4_t _r_4_m_2 = vsubq_f32(_r_44, _r_22);

  5732.                         float32x4_t _r_3_m_5 = vsubq_f32(_r_33, _r_55);

  5733. 

  5734.                         float32x4_t _tmp0 = vmlaq_lane_f32(_r_0_m_6, _r_4_m_2, vget_high_f32(_coeff1), 1);

  5735.                         float32x4_t _tmp7 = vmlaq_lane_f32(_r_7_m_1, _r_3_m_5, vget_high_f32(_coeff1), 1);

  5736. 

  5737.                         vst1q_f32(&tmp[0][m], _tmp0);

  5738.                         vst1q_f32(&tmp[7][m], _tmp7);

  5739. 

  5740.                         float32x4_t _r_2_a_6 = vaddq_f32(_r_22, _r_66);

  5741.                         float32x4_t _r_1_a_5 = vaddq_f32(_r_11, _r_55);

  5742. 

  5743.                         float32x4_t _tmp12a = vmlsq_lane_f32(_r_2_a_6, _r_44, vget_high_f32(_coeff1), 0);

  5744.                         float32x4_t _tmp12b = vmlsq_lane_f32(_r_1_a_5, _r_33, vget_high_f32(_coeff1), 0);

  5745. 

  5746.                         float32x4_t _tmp1 = vaddq_f32(_tmp12a, _tmp12b);

  5747.                         float32x4_t _tmp2 = vsubq_f32(_tmp12a, _tmp12b);

  5748. 

  5749.                         vst1q_f32(&tmp[1][m], _tmp1);

  5750.                         vst1q_f32(&tmp[2][m], _tmp2);

  5751. 

  5752.                         float32x4_t _r_4_x_c = vmulq_lane_f32(_r_44, vget_high_f32(_coeff0), 0);

  5753.                         float32x4_t _r_3_x_c = vmulq_lane_f32(_r_33, vget_low_f32(_coeff1), 0);

  5754. 

  5755.                         float32x4_t _tmp34a = vaddq_f32(_r_66, _r_4_x_c);

  5756.                         _tmp34a = vmlaq_lane_f32(_tmp34a, _r_22, vget_low_f32(_coeff0), 0);

  5757. 

  5758.                         float32x4_t _tmp34b = vmlaq_lane_f32(_r_3_x_c, _r_11, vget_low_f32(_coeff0), 1);

  5759.                         _tmp34b = vmlaq_lane_f32(_tmp34b, _r_55, vget_high_f32(_coeff0), 1);

  5760. 

  5761.                         float32x4_t _tmp3 = vaddq_f32(_tmp34a, _tmp34b);

  5762.                         float32x4_t _tmp4 = vsubq_f32(_tmp34a, _tmp34b);

  5763. 

  5764.                         vst1q_f32(&tmp[3][m], _tmp3);

  5765.                         vst1q_f32(&tmp[4][m], _tmp4);

  5766. 

  5767.                         // reuse r04 * 1.25

  5768.                         // reuse r03 * 2.5

  5769.                         float32x4_t _r_2_a_4c = vaddq_f32(_r_22, _r_4_x_c);

  5770.                         float32x4_t _tmp56a = vmlaq_lane_f32(_r_66, _r_2_a_4c, vget_low_f32(_coeff1), 1);

  5771.                         float32x4_t _tmp56b = vmlaq_lane_f32(_r_3_x_c, _r_11, vget_high_f32(_coeff0), 1);

  5772.                         _tmp56b = vmlaq_lane_f32(_tmp56b, _r_55, vget_low_f32(_coeff0), 1);

  5773. 

  5774.                         float32x4_t _tmp5 = vaddq_f32(_tmp56a, _tmp56b);

  5775.                         float32x4_t _tmp6 = vsubq_f32(_tmp56a, _tmp56b);

  5776. 

  5777.                         vst1q_f32(&tmp[5][m], _tmp5);

  5778.                         vst1q_f32(&tmp[6][m], _tmp6);

  5779. 

  5780.                         r0 += w*4;

  5781.                         r1 += w*4;

  5782.                         r2 += w*4;

  5783.                         r3 += w*4;

  5784.                     }

  5785. 

  5786.                     const float* t0 = tmp[0];

  5787.                     const float* t1 = tmp[1];

  5788.                     const float* t2 = tmp[2];

  5789.                     const float* t3 = tmp[3];

  5790. 

  5791.                     float* r0_tm0_0 = img0_tm.row(i * w_tm/8 + j);

  5792.                     float* r0_tm0_4 = img0_tm.row(i * w_tm/8 + j + tiles);

  5793.                     float* r0_tm1_0 = img0_tm.row(i * w_tm/8 + j + tiles*2);

  5794.                     float* r0_tm1_4 = img0_tm.row(i * w_tm/8 + j + tiles*3);

  5795.                     float* r0_tm2_0 = img0_tm.row(i * w_tm/8 + j + tiles*4);

  5796.                     float* r0_tm2_4 = img0_tm.row(i * w_tm/8 + j + tiles*5);

  5797.                     float* r0_tm3_0 = img0_tm.row(i * w_tm/8 + j + tiles*6);

  5798.                     float* r0_tm3_4 = img0_tm.row(i * w_tm/8 + j + tiles*7);

  5799. 

  5800.                     for (int m=0; m+3<8; m+=4)

  5801.                     {

  5802.                         float32x4_t _t0_0123 = vld1q_f32(t0);

  5803.                         float32x4_t _t0_4567 = vld1q_f32(t0+4);

  5804.                         float32x4_t _t1_0123 = vld1q_f32(t1);

  5805.                         float32x4_t _t1_4567 = vld1q_f32(t1+4);

  5806.                         float32x4_t _t2_0123 = vld1q_f32(t2);

  5807.                         float32x4_t _t2_4567 = vld1q_f32(t2+4);

  5808.                         float32x4_t _t3_0123 = vld1q_f32(t3);

  5809.                         float32x4_t _t3_4567 = vld1q_f32(t3+4);

  5810. 

  5811.                         float32x4x2_t _t01_00221133 = vtrnq_f32(_t0_0123, _t1_0123);

  5812.                         float32x4x2_t _t01_44665577 = vtrnq_f32(_t0_4567, _t1_4567);

  5813.                         float32x4x2_t _t23_00221133 = vtrnq_f32(_t2_0123, _t3_0123);

  5814.                         float32x4x2_t _t23_44665577 = vtrnq_f32(_t2_4567, _t3_4567);

  5815. 

  5816.                         // no vswp intrinsic  :(

  5817.                         float32x4_t _t_00 = vcombine_f32(vget_low_f32(_t01_00221133.val[0]), vget_low_f32(_t23_00221133.val[0]));

  5818.                         float32x4_t _t_11 = vcombine_f32(vget_low_f32(_t01_00221133.val[1]), vget_low_f32(_t23_00221133.val[1]));

  5819.                         float32x4_t _t_22 = vcombine_f32(vget_high_f32(_t01_00221133.val[0]), vget_high_f32(_t23_00221133.val[0]));

  5820.                         float32x4_t _t_33 = vcombine_f32(vget_high_f32(_t01_00221133.val[1]), vget_high_f32(_t23_00221133.val[1]));

  5821.                         float32x4_t _t_44 = vcombine_f32(vget_low_f32(_t01_44665577.val[0]), vget_low_f32(_t23_44665577.val[0]));

  5822.                         float32x4_t _t_55 = vcombine_f32(vget_low_f32(_t01_44665577.val[1]), vget_low_f32(_t23_44665577.val[1]));

  5823.                         float32x4_t _t_66 = vcombine_f32(vget_high_f32(_t01_44665577.val[0]), vget_high_f32(_t23_44665577.val[0]));

  5824.                         float32x4_t _t_77 = vcombine_f32(vget_high_f32(_t01_44665577.val[1]), vget_high_f32(_t23_44665577.val[1]));

  5825. 

  5826.                         float32x4_t _t_0_m_6 = vsubq_f32(_t_00, _t_66);

  5827.                         float32x4_t _t_7_m_1 = vsubq_f32(_t_77, _t_11);

  5828. 

  5829.                         float32x4_t _t_4_m_2 = vsubq_f32(_t_44, _t_22);

  5830.                         float32x4_t _t_3_m_5 = vsubq_f32(_t_33, _t_55);

  5831. 

  5832.                         float32x4_t _r0_tm_0_0 = vmlaq_lane_f32(_t_0_m_6, _t_4_m_2, vget_high_f32(_coeff1), 1);

  5833.                         float32x4_t _r0_tm_4_3 = vmlaq_lane_f32(_t_7_m_1, _t_3_m_5, vget_high_f32(_coeff1), 1);

  5834. 

  5835.                         r0_tm0_0[0] = vgetq_lane_f32(_r0_tm_0_0, 0);

  5836.                         r0_tm1_0[0] = vgetq_lane_f32(_r0_tm_0_0, 1);

  5837.                         r0_tm2_0[0] = vgetq_lane_f32(_r0_tm_0_0, 2);

  5838.                         r0_tm3_0[0] = vgetq_lane_f32(_r0_tm_0_0, 3);

  5839. 

  5840.                         r0_tm0_4[3] = vgetq_lane_f32(_r0_tm_4_3, 0);

  5841.                         r0_tm1_4[3] = vgetq_lane_f32(_r0_tm_4_3, 1);

  5842.                         r0_tm2_4[3] = vgetq_lane_f32(_r0_tm_4_3, 2);

  5843.                         r0_tm3_4[3] = vgetq_lane_f32(_r0_tm_4_3, 3);

  5844. 

  5845.                         float32x4_t _t_2_m_6 = vaddq_f32(_t_22, _t_66);

  5846.                         float32x4_t _t_1_m_5 = vaddq_f32(_t_11, _t_55);

  5847. 

  5848.                         float32x4_t _tmp12a = vmlsq_lane_f32(_t_2_m_6, _t_44, vget_high_f32(_coeff1), 0);

  5849.                         float32x4_t _tmp12b = vmlsq_lane_f32(_t_1_m_5, _t_33, vget_high_f32(_coeff1), 0);

  5850. 

  5851.                         float32x4_t _r0_tm_0_1 = vaddq_f32(_tmp12a, _tmp12b);

  5852.                         float32x4_t _r0_tm_0_2 = vsubq_f32(_tmp12a, _tmp12b);

  5853. 

  5854.                         r0_tm0_0[1] = vgetq_lane_f32(_r0_tm_0_1, 0);

  5855.                         r0_tm1_0[1] = vgetq_lane_f32(_r0_tm_0_1, 1);

  5856.                         r0_tm2_0[1] = vgetq_lane_f32(_r0_tm_0_1, 2);

  5857.                         r0_tm3_0[1] = vgetq_lane_f32(_r0_tm_0_1, 3);

  5858. 

  5859.                         r0_tm0_0[2] = vgetq_lane_f32(_r0_tm_0_2, 0);

  5860.                         r0_tm1_0[2] = vgetq_lane_f32(_r0_tm_0_2, 1);

  5861.                         r0_tm2_0[2] = vgetq_lane_f32(_r0_tm_0_2, 2);

  5862.                         r0_tm3_0[2] = vgetq_lane_f32(_r0_tm_0_2, 3);

  5863. 

  5864.                         float32x4_t _t_4_x_c = vmulq_lane_f32(_t_44, vget_high_f32(_coeff0), 0);

  5865.                         float32x4_t _t_3_x_c = vmulq_lane_f32(_t_33, vget_low_f32(_coeff1), 0);

  5866. 

  5867.                         float32x4_t _tmp34a = vaddq_f32(_t_66, _t_4_x_c);

  5868.                         _tmp34a = vmlaq_lane_f32(_tmp34a, _t_22, vget_low_f32(_coeff0), 0);

  5869. 

  5870.                         float32x4_t _tmp34b = vmlaq_lane_f32(_t_3_x_c, _t_11, vget_low_f32(_coeff0), 1);

  5871.                         _tmp34b = vmlaq_lane_f32(_tmp34b, _t_55, vget_high_f32(_coeff0), 1);

  5872. 

  5873.                         float32x4_t _r0_tm_0_3 = vaddq_f32(_tmp34a, _tmp34b);

  5874.                         float32x4_t _r0_tm_4_0 = vsubq_f32(_tmp34a, _tmp34b);

  5875. 

  5876.                         r0_tm0_0[3] = vgetq_lane_f32(_r0_tm_0_3, 0);

  5877.                         r0_tm1_0[3] = vgetq_lane_f32(_r0_tm_0_3, 1);

  5878.                         r0_tm2_0[3] = vgetq_lane_f32(_r0_tm_0_3, 2);

  5879.                         r0_tm3_0[3] = vgetq_lane_f32(_r0_tm_0_3, 3);

  5880. 

  5881.                         r0_tm0_4[0] = vgetq_lane_f32(_r0_tm_4_0, 0);

  5882.                         r0_tm1_4[0] = vgetq_lane_f32(_r0_tm_4_0, 1);

  5883.                         r0_tm2_4[0] = vgetq_lane_f32(_r0_tm_4_0, 2);

  5884.                         r0_tm3_4[0] = vgetq_lane_f32(_r0_tm_4_0, 3);

  5885. 

  5886.                         float32x4_t _t_2_a_4c = vaddq_f32(_t_22, _t_4_x_c);

  5887.                         float32x4_t _tmp56a = vmlaq_lane_f32(_t_66, _t_2_a_4c, vget_low_f32(_coeff1), 1);

  5888.                         float32x4_t _tmp56b = vmlaq_lane_f32(_t_3_x_c, _t_11, vget_high_f32(_coeff0), 1);

  5889.                         _tmp56b = vmlaq_lane_f32(_tmp56b, _t_55, vget_low_f32(_coeff0), 1);

  5890. 

  5891.                         float32x4_t _r0_tm_4_1 = vaddq_f32(_tmp56a, _tmp56b);

  5892.                         float32x4_t _r0_tm_4_2 = vsubq_f32(_tmp56a, _tmp56b);

  5893. 

  5894.                         r0_tm0_4[1] = vgetq_lane_f32(_r0_tm_4_1, 0);

  5895.                         r0_tm1_4[1] = vgetq_lane_f32(_r0_tm_4_1, 1);

  5896.                         r0_tm2_4[1] = vgetq_lane_f32(_r0_tm_4_1, 2);

  5897.                         r0_tm3_4[1] = vgetq_lane_f32(_r0_tm_4_1, 3);

  5898. 

  5899.                         r0_tm0_4[2] = vgetq_lane_f32(_r0_tm_4_2, 0);

  5900.                         r0_tm1_4[2] = vgetq_lane_f32(_r0_tm_4_2, 1);

  5901.                         r0_tm2_4[2] = vgetq_lane_f32(_r0_tm_4_2, 2);

  5902.                         r0_tm3_4[2] = vgetq_lane_f32(_r0_tm_4_2, 3);

  5903. 

  5904.                         t0 += 8*4;

  5905.                         t1 += 8*4;

  5906.                         t2 += 8*4;

  5907.                         t3 += 8*4;

  5908. 

  5909.                         r0_tm0_0 += img0_tm.w*tiles*2*4;

  5910.                         r0_tm0_4 += img0_tm.w*tiles*2*4;

  5911.                         r0_tm1_0 += img0_tm.w*tiles*2*4;

  5912.                         r0_tm1_4 += img0_tm.w*tiles*2*4;

  5913.                         r0_tm2_0 += img0_tm.w*tiles*2*4;

  5914.                         r0_tm2_4 += img0_tm.w*tiles*2*4;

  5915.                         r0_tm3_0 += img0_tm.w*tiles*2*4;

  5916.                         r0_tm3_4 += img0_tm.w*tiles*2*4;

  5917.                     }

  5918. #else // __aarch64__

  5919.                     float* t0 = tmp[0];

  5920.                     float* t1 = tmp[1];

  5921.                     float* t2 = tmp[2];

  5922.                     float* t3 = tmp[3];

  5923.                     float* t4 = tmp[4];

  5924.                     float* t5 = tmp[5];

  5925.                     float* t6 = tmp[6];

  5926.                     float* t7 = tmp[7];

  5927. 

  5928.                     int stepw = w*4*4;

  5929. 

  5930.                     asm volatile(

  5931. 

  5932.                         // loop0

  5933.                         "vld1.f32   {d16-d19}, [%8], %26    \n"

  5934.                         "vld1.f32   {d20-d23}, [%9], %26    \n"

  5935.                         "vld1.f32   {d24-d27}, [%10], %26   \n"

  5936. 

  5937.                         "vtrn.32    q8, q10             \n"

  5938. 

  5939.                         "vld1.f32   {d28-d31}, [%11], %26   \n"

  5940. 

  5941.                         "vtrn.32    q9, q11             \n"

  5942.                         "vtrn.32    q12, q14            \n"

  5943.                         "vtrn.32    q13, q15            \n"

  5944. 

  5945.                         "vswp       d17, d24            \n"

  5946.                         "vswp       d19, d26            \n"

  5947.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  5948.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  5949. 

  5950.                         "vsub.f32   q2, q8, q13         \n"

  5951.                         "vsub.f32   q3, q9, q12         \n"

  5952. 

  5953.                         "vadd.f32   q4, q12, q13        \n"

  5954.                         "vadd.f32   q5, q10, q11        \n"

  5955. 

  5956.                         "vmla.f32   q2, q3, %f25[1]     \n"

  5957. 

  5958.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  5959.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  5960. 

  5961.                         "vmls.f32   q4, q9, %f25[0]     \n"

  5962.                         "vmls.f32   q5, q14, %f25[0]    \n"

  5963. 

  5964.                         "vst1.f32   {d4-d5}, [%0]!      \n"// tmp[0][m]

  5965. 

  5966.                         "vmov       q3, q7              \n"// use q7

  5967. 

  5968.                         "vadd.f32   q2, q13, q6         \n"// use q6

  5969.                         "vmla.f32   q3, q10, %e24[1]    \n"

  5970. 

  5971.                         "vadd.f32   q8, q4, q5          \n"

  5972.                         "vsub.f32   q9, q4, q5          \n"

  5973. 

  5974.                         "vmov       q5, q7              \n"// use q7

  5975. 

  5976.                         "vadd.f32   q6, q12, q6         \n"// use q6

  5977.                         "vmla.f32   q5, q10, %f24[1]    \n"

  5978. 

  5979.                         "vmov       q4, q13             \n"

  5980. 

  5981.                         "vmla.f32   q2, q12, %e24[0]    \n"

  5982.                         "vmla.f32   q3, q11, %f24[1]    \n"

  5983. 

  5984.                         "vst1.f32   {d16-d17}, [%1]!    \n"// tmp[1][m]

  5985. 

  5986.                         "vmla.f32   q4, q6, %e25[1]     \n"

  5987.                         "vmla.f32   q5, q11, %e24[1]    \n"

  5988. 

  5989.                         "vst1.f32   {d18-d19}, [%2]!    \n"// tmp[2][m]

  5990. 

  5991.                         "vadd.f32   q8, q2, q3          \n"

  5992.                         "vsub.f32   q9, q2, q3          \n"

  5993. 

  5994.                         "vsub.f32   q6, q15, q10        \n"

  5995.                         "vsub.f32   q7, q14, q11        \n"

  5996. 

  5997.                         "vadd.f32   q2, q4, q5          \n"

  5998.                         "vsub.f32   q3, q4, q5          \n"

  5999. 

  6000.                         "vst1.f32   {d16-d17}, [%3]!    \n"// tmp[3][m]

  6001.                         "vst1.f32   {d18-d19}, [%4]!    \n"// tmp[4][m]

  6002. 

  6003.                         "vmla.f32   q6, q7, %f25[1]     \n"

  6004. 

  6005.                         "vst1.f32   {d4-d5}, [%5]!      \n"// tmp[5][m]

  6006.                         "vst1.f32   {d6-d7}, [%6]!      \n"// tmp[6][m]

  6007. 

  6008.                         "vst1.f32   {d12-d13}, [%7]!    \n"// tmp[7][m]

  6009. 

  6010.                         // loop1

  6011.                         "vld1.f32   {d16-d19}, [%8]     \n"

  6012.                         "vld1.f32   {d20-d23}, [%9]     \n"

  6013.                         "vld1.f32   {d24-d27}, [%10]    \n"

  6014. 

  6015.                         "vtrn.32    q8, q10             \n"

  6016. 

  6017.                         "vld1.f32   {d28-d31}, [%11]    \n"

  6018. 

  6019.                         "vtrn.32    q9, q11             \n"

  6020.                         "vtrn.32    q12, q14            \n"

  6021.                         "vtrn.32    q13, q15            \n"

  6022. 

  6023.                         "vswp       d17, d24            \n"

  6024.                         "vswp       d19, d26            \n"

  6025.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  6026.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  6027. 

  6028.                         "vsub.f32   q2, q8, q13         \n"

  6029.                         "vsub.f32   q3, q9, q12         \n"

  6030. 

  6031.                         "vadd.f32   q4, q12, q13        \n"

  6032.                         "vadd.f32   q5, q10, q11        \n"

  6033. 

  6034.                         "vmla.f32   q2, q3, %f25[1]     \n"

  6035. 

  6036.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  6037.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  6038. 

  6039.                         "vmls.f32   q4, q9, %f25[0]     \n"

  6040.                         "vmls.f32   q5, q14, %f25[0]    \n"

  6041. 

  6042.                         "vst1.f32   {d4-d5}, [%0]!      \n"// tmp[0][m]

  6043. 

  6044.                         "vmov       q3, q7              \n"// use q7

  6045. 

  6046.                         "vadd.f32   q2, q13, q6         \n"// use q6

  6047.                         "vmla.f32   q3, q10, %e24[1]    \n"

  6048. 

  6049.                         "vadd.f32   q8, q4, q5          \n"

  6050.                         "vsub.f32   q9, q4, q5          \n"

  6051. 

  6052.                         "vmov       q5, q7              \n"// use q7

  6053. 

  6054.                         "vadd.f32   q6, q12, q6         \n"// use q6

  6055.                         "vmla.f32   q5, q10, %f24[1]    \n"

  6056. 

  6057.                         "vmov       q4, q13             \n"

  6058. 

  6059.                         "vmla.f32   q2, q12, %e24[0]    \n"

  6060.                         "vmla.f32   q3, q11, %f24[1]    \n"

  6061. 

  6062.                         "vst1.f32   {d16-d17}, [%1]!    \n"// tmp[1][m]

  6063. 

  6064.                         "vmla.f32   q4, q6, %e25[1]     \n"

  6065.                         "vmla.f32   q5, q11, %e24[1]    \n"

  6066. 

  6067.                         "vst1.f32   {d18-d19}, [%2]!    \n"// tmp[2][m]

  6068. 

  6069.                         "vadd.f32   q8, q2, q3          \n"

  6070.                         "vsub.f32   q9, q2, q3          \n"

  6071. 

  6072.                         "vsub.f32   q6, q15, q10        \n"

  6073.                         "vsub.f32   q7, q14, q11        \n"

  6074. 

  6075.                         "vadd.f32   q2, q4, q5          \n"

  6076.                         "vsub.f32   q3, q4, q5          \n"

  6077. 

  6078.                         "vst1.f32   {d16-d17}, [%3]!    \n"// tmp[3][m]

  6079.                         "vst1.f32   {d18-d19}, [%4]!    \n"// tmp[4][m]

  6080. 

  6081.                         "vmla.f32   q6, q7, %f25[1]     \n"

  6082. 

  6083.                         "vst1.f32   {d4-d5}, [%5]!      \n"// tmp[5][m]

  6084.                         "vst1.f32   {d6-d7}, [%6]!      \n"// tmp[6][m]

  6085. 

  6086.                         "vst1.f32   {d12-d13}, [%7]!    \n"// tmp[7][m]

  6087. 

  6088.                         : "=r"(t0),     // %0

  6089.                           "=r"(t1),     // %1

  6090.                           "=r"(t2),     // %2

  6091.                           "=r"(t3),     // %3

  6092.                           "=r"(t4),     // %4

  6093.                           "=r"(t5),     // %5

  6094.                           "=r"(t6),     // %6

  6095.                           "=r"(t7),     // %7

  6096.                           "=r"(r0),     // %8

  6097.                           "=r"(r1),     // %9

  6098.                           "=r"(r2),     // %10

  6099.                           "=r"(r3)      // %11

  6100.                         : "0"(t0),

  6101.                           "1"(t1),

  6102.                           "2"(t2),

  6103.                           "3"(t3),

  6104.                           "4"(t4),

  6105.                           "5"(t5),

  6106.                           "6"(t6),

  6107.                           "7"(t7),

  6108.                           "8"(r0),

  6109.                           "9"(r1),

  6110.                           "10"(r2),

  6111.                           "11"(r3),

  6112.                           "w"(_coeff0), // %24

  6113.                           "w"(_coeff1), // %25

  6114.                           "r"(stepw)        // %26

  6115.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  6116.                     );

  6117. 

  6118.                     t0 = tmp[0];

  6119.                     t1 = tmp[1];

  6120.                     t2 = tmp[2];

  6121.                     t3 = tmp[3];

  6122. 

  6123.                     float* r0_tm0_0 = img0_tm.row(i * w_tm/8 + j);

  6124.                     float* r0_tm0_4 = img0_tm.row(i * w_tm/8 + j + tiles);

  6125.                     float* r0_tm1_0 = img0_tm.row(i * w_tm/8 + j + tiles*2);

  6126.                     float* r0_tm1_4 = img0_tm.row(i * w_tm/8 + j + tiles*3);

  6127.                     float* r0_tm2_0 = img0_tm.row(i * w_tm/8 + j + tiles*4);

  6128.                     float* r0_tm2_4 = img0_tm.row(i * w_tm/8 + j + tiles*5);

  6129.                     float* r0_tm3_0 = img0_tm.row(i * w_tm/8 + j + tiles*6);

  6130.                     float* r0_tm3_4 = img0_tm.row(i * w_tm/8 + j + tiles*7);

  6131. 

  6132.                     int step = img0_tm.w*tiles*2*4*4;

  6133. 

  6134.                     asm volatile(

  6135. 

  6136.                         // loop0

  6137.                         "vld1.f32   {d16-d19}, [%8]     \n"

  6138.                         "add        %8, %8, #128        \n"

  6139.                         "vld1.f32   {d20-d23}, [%9]     \n"

  6140.                         "add        %9, %9, #128        \n"

  6141.                         "vld1.f32   {d24-d27}, [%10]    \n"

  6142.                         "add        %10, %10, #128      \n"

  6143. 

  6144.                         "vtrn.32    q8, q10             \n"

  6145. 

  6146.                         "vld1.f32   {d28-d31}, [%11]    \n"

  6147.                         "add        %11, %11, #128      \n"

  6148. 

  6149.                         "vtrn.32    q9, q11             \n"

  6150.                         "vtrn.32    q12, q14            \n"

  6151.                         "vtrn.32    q13, q15            \n"

  6152. 

  6153.                         "vswp       d17, d24            \n"

  6154.                         "vswp       d19, d26            \n"

  6155.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  6156.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  6157. 

  6158.                         "vsub.f32   q2, q8, q13         \n"

  6159.                         "vsub.f32   q3, q9, q12         \n"

  6160. 

  6161.                         "vadd.f32   q4, q12, q13        \n"

  6162.                         "vadd.f32   q5, q10, q11        \n"

  6163. 

  6164.                         "vmla.f32   q2, q3, %f25[1]     \n"

  6165. 

  6166.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  6167.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  6168. 

  6169.                         "vmls.f32   q4, q9, %f25[0]     \n"

  6170.                         "vmls.f32   q5, q14, %f25[0]    \n"

  6171. 

  6172.                         "vst1.f32   {d4[0]}, [%0]!      \n"

  6173.                         "vst1.f32   {d4[1]}, [%2]!      \n"

  6174. 

  6175.                         "vmov       q3, q7              \n"// use q7

  6176. 

  6177.                         "vst1.f32   {d5[0]}, [%4]!      \n"

  6178.                         "vst1.f32   {d5[1]}, [%6]!      \n"

  6179. 

  6180.                         "vadd.f32   q2, q13, q6         \n"// use q6

  6181.                         "vmla.f32   q3, q10, %e24[1]    \n"

  6182. 

  6183.                         "vadd.f32   q8, q4, q5          \n"

  6184.                         "vsub.f32   q9, q4, q5          \n"

  6185. 

  6186.                         "vmov       q5, q7              \n"// use q7

  6187. 

  6188.                         "vadd.f32   q6, q12, q6         \n"// use q6

  6189.                         "vmla.f32   q5, q10, %f24[1]    \n"

  6190. 

  6191.                         "vmov       q4, q13             \n"

  6192. 

  6193.                         "vmla.f32   q2, q12, %e24[0]    \n"

  6194.                         "vmla.f32   q3, q11, %f24[1]    \n"

  6195. 

  6196.                         "vst1.f32   {d16[0]}, [%0]!     \n"

  6197.                         "vst1.f32   {d16[1]}, [%2]!     \n"

  6198. 

  6199.                         "vmla.f32   q4, q6, %e25[1]     \n"

  6200. 

  6201.                         "vst1.f32   {d17[0]}, [%4]!     \n"

  6202.                         "vst1.f32   {d17[1]}, [%6]!     \n"

  6203. 

  6204.                         "vmla.f32   q5, q11, %e24[1]    \n"

  6205. 

  6206.                         "vst1.f32   {d18[0]}, [%0]!     \n"

  6207.                         "vst1.f32   {d18[1]}, [%2]!     \n"

  6208. 

  6209.                         "vadd.f32   q8, q2, q3          \n"

  6210. 

  6211.                         "vst1.f32   {d19[0]}, [%4]!     \n"

  6212.                         "vst1.f32   {d19[1]}, [%6]!     \n"

  6213. 

  6214.                         "vsub.f32   q9, q2, q3          \n"

  6215. 

  6216.                         "vsub.f32   q6, q15, q10        \n"

  6217.                         "vsub.f32   q7, q14, q11        \n"

  6218. 

  6219.                         "vadd.f32   q2, q4, q5          \n"

  6220.                         "vsub.f32   q3, q4, q5          \n"

  6221. 

  6222.                         "vst1.f32   {d16[0]}, [%0], %26 \n"

  6223.                         "vst1.f32   {d16[1]}, [%2], %26 \n"

  6224. 

  6225.                         "vmla.f32   q6, q7, %f25[1]     \n"

  6226. 

  6227.                         "vst1.f32   {d17[0]}, [%4], %26 \n"

  6228.                         "vst1.f32   {d17[1]}, [%6], %26 \n"

  6229. 

  6230.                         "vtrn.32    q9, q2              \n"

  6231.                         "vtrn.32    q3, q6              \n"

  6232. 

  6233.                         "sub        %0, %0, #12         \n"

  6234.                         "sub        %2, %2, #12         \n"

  6235.                         "sub        %4, %4, #12         \n"

  6236.                         "sub        %6, %6, #12         \n"

  6237. 

  6238.                         "vswp       d19, d6             \n"

  6239.                         "vswp       d5, d12             \n"

  6240. 

  6241.                         "vst1.f32   {d18-d19}, [%1], %26 \n"

  6242.                         "vst1.f32   {d4-d5}, [%3], %26  \n"

  6243.                         "vst1.f32   {d6-d7}, [%5], %26  \n"

  6244.                         "vst1.f32   {d12-d13}, [%7], %26 \n"

  6245. 

  6246.                         // loop1

  6247.                         "vld1.f32   {d16-d19}, [%8]     \n"

  6248.                         "vld1.f32   {d20-d23}, [%9]     \n"

  6249.                         "vld1.f32   {d24-d27}, [%10]    \n"

  6250. 

  6251.                         "vtrn.32    q8, q10             \n"

  6252. 

  6253.                         "vld1.f32   {d28-d31}, [%11]    \n"

  6254. 

  6255.                         "vtrn.32    q9, q11             \n"

  6256.                         "vtrn.32    q12, q14            \n"

  6257.                         "vtrn.32    q13, q15            \n"

  6258. 

  6259.                         "vswp       d17, d24            \n"

  6260.                         "vswp       d19, d26            \n"

  6261.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  6262.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  6263. 

  6264.                         "vsub.f32   q2, q8, q13         \n"

  6265.                         "vsub.f32   q3, q9, q12         \n"

  6266. 

  6267.                         "vadd.f32   q4, q12, q13        \n"

  6268.                         "vadd.f32   q5, q10, q11        \n"

  6269. 

  6270.                         "vmla.f32   q2, q3, %f25[1]     \n"

  6271. 

  6272.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  6273.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  6274. 

  6275.                         "vmls.f32   q4, q9, %f25[0]     \n"

  6276.                         "vmls.f32   q5, q14, %f25[0]    \n"

  6277. 

  6278.                         "vst1.f32   {d4[0]}, [%0]!      \n"

  6279.                         "vst1.f32   {d4[1]}, [%2]!      \n"

  6280. 

  6281.                         "vmov       q3, q7              \n"// use q7

  6282. 

  6283.                         "vst1.f32   {d5[0]}, [%4]!      \n"

  6284.                         "vst1.f32   {d5[1]}, [%6]!      \n"

  6285. 

  6286.                         "vadd.f32   q2, q13, q6         \n"// use q6

  6287.                         "vmla.f32   q3, q10, %e24[1]    \n"

  6288. 

  6289.                         "vadd.f32   q8, q4, q5          \n"

  6290.                         "vsub.f32   q9, q4, q5          \n"

  6291. 

  6292.                         "vmov       q5, q7              \n"// use q7

  6293. 

  6294.                         "vadd.f32   q6, q12, q6         \n"// use q6

  6295.                         "vmla.f32   q5, q10, %f24[1]    \n"

  6296. 

  6297.                         "vmov       q4, q13             \n"

  6298. 

  6299.                         "vmla.f32   q2, q12, %e24[0]    \n"

  6300.                         "vmla.f32   q3, q11, %f24[1]    \n"

  6301. 

  6302.                         "vst1.f32   {d16[0]}, [%0]!     \n"

  6303.                         "vst1.f32   {d16[1]}, [%2]!     \n"

  6304. 

  6305.                         "vmla.f32   q4, q6, %e25[1]     \n"

  6306. 

  6307.                         "vst1.f32   {d17[0]}, [%4]!     \n"

  6308.                         "vst1.f32   {d17[1]}, [%6]!     \n"

  6309. 

  6310.                         "vmla.f32   q5, q11, %e24[1]    \n"

  6311. 

  6312.                         "vst1.f32   {d18[0]}, [%0]!     \n"

  6313.                         "vst1.f32   {d18[1]}, [%2]!     \n"

  6314. 

  6315.                         "vadd.f32   q8, q2, q3          \n"

  6316. 

  6317.                         "vst1.f32   {d19[0]}, [%4]!     \n"

  6318.                         "vst1.f32   {d19[1]}, [%6]!     \n"

  6319. 

  6320.                         "vsub.f32   q9, q2, q3          \n"

  6321. 

  6322.                         "vsub.f32   q6, q15, q10        \n"

  6323.                         "vsub.f32   q7, q14, q11        \n"

  6324. 

  6325.                         "vadd.f32   q2, q4, q5          \n"

  6326.                         "vsub.f32   q3, q4, q5          \n"

  6327. 

  6328.                         "vst1.f32   {d16[0]}, [%0]      \n"

  6329.                         "vst1.f32   {d16[1]}, [%2]      \n"

  6330. 

  6331.                         "vmla.f32   q6, q7, %f25[1]     \n"

  6332. 

  6333.                         "vst1.f32   {d17[0]}, [%4]      \n"

  6334.                         "vst1.f32   {d17[1]}, [%6]      \n"

  6335. 

  6336.                         "vtrn.32    q9, q2              \n"

  6337.                         "vtrn.32    q3, q6              \n"

  6338. 

  6339.                         "vswp       d19, d6             \n"

  6340.                         "vswp       d5, d12             \n"

  6341. 

  6342.                         "vst1.f32   {d18-d19}, [%1]     \n"

  6343.                         "vst1.f32   {d4-d5}, [%3]       \n"

  6344.                         "vst1.f32   {d6-d7}, [%5]       \n"

  6345.                         "vst1.f32   {d12-d13}, [%7]     \n"

  6346. 

  6347.                         : "=r"(r0_tm0_0),     // %0

  6348.                           "=r"(r0_tm0_4),     // %1

  6349.                           "=r"(r0_tm1_0),     // %2

  6350.                           "=r"(r0_tm1_4),     // %3

  6351.                           "=r"(r0_tm2_0),     // %4

  6352.                           "=r"(r0_tm2_4),     // %5

  6353.                           "=r"(r0_tm3_0),     // %6

  6354.                           "=r"(r0_tm3_4),     // %7

  6355.                           "=r"(t0),     // %8

  6356.                           "=r"(t1),     // %9

  6357.                           "=r"(t2),     // %10

  6358.                           "=r"(t3)      // %11

  6359.                         : "0"(r0_tm0_0),

  6360.                           "1"(r0_tm0_4),

  6361.                           "2"(r0_tm1_0),

  6362.                           "3"(r0_tm1_4),

  6363.                           "4"(r0_tm2_0),

  6364.                           "5"(r0_tm2_4),

  6365.                           "6"(r0_tm3_0),

  6366.                           "7"(r0_tm3_4),

  6367.                           "8"(t0),

  6368.                           "9"(t1),

  6369.                           "10"(t2),

  6370.                           "11"(t3),

  6371.                           "w"(_coeff0), // %24

  6372.                           "w"(_coeff1), // %25

  6373.                           "r"(step)        // %26

  6374.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  6375.                     );

  6376. #endif // __aarch64__

  6377. #else

  6378.                     const float* r0 = img0.row(i * 6) + j * 6;

  6379. 

  6380.                     for (int m=0; m<8; m++)

  6381.                     {

  6382.                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25f;

  6383.                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25f;

  6384. 

  6385.                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25f);

  6386.                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25f);

  6387. 

  6388.                         tmp[1][m] = tmp12a + tmp12b;

  6389.                         tmp[2][m] = tmp12a - tmp12b;

  6390. 

  6391.                         float tmp34a = (r0[6] + r0[2] * 0.25f - r0[4] * 1.25f);

  6392.                         float tmp34b = (r0[1] * 0.5f - r0[3] * 2.5f + r0[5] * 2.f);

  6393. 

  6394.                         tmp[3][m] = tmp34a + tmp34b;

  6395.                         tmp[4][m] = tmp34a - tmp34b;

  6396. 

  6397.                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25f) * 4.f);

  6398.                         float tmp56b = (r0[1] * 2.f - r0[3] * 2.5f + r0[5] * 0.5f);

  6399. 

  6400.                         tmp[5][m] = tmp56a + tmp56b;

  6401.                         tmp[6][m] = tmp56a - tmp56b;

  6402. 

  6403.                         r0 += w;

  6404.                     }

  6405. 

  6406.                     float* r0_tm_0 = img0_tm.row(i * w_tm/8 + j);

  6407.                     float* r0_tm_4 = img0_tm.row(i * w_tm/8 + j + tiles);

  6408. 

  6409.                     for (int m=0; m<8; m++)

  6410.                     {

  6411.                         const float* tmp0 = tmp[m];

  6412. 

  6413.                         r0_tm_0[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25f;

  6414.                         r0_tm_4[3] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25f;

  6415. 

  6416.                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25f);

  6417.                         float tmp12b = (tmp0[1] - tmp0[3] * 4.25f + tmp0[5]);

  6418. 

  6419.                         r0_tm_0[1] = tmp12a + tmp12b;

  6420.                         r0_tm_0[2] = tmp12a - tmp12b;

  6421. 

  6422.                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25f - tmp0[4] * 1.25f);

  6423.                         float tmp34b = (tmp0[1] * 0.5f - tmp0[3] * 2.5f + tmp0[5] * 2.f);

  6424. 

  6425.                         r0_tm_0[3] = tmp34a + tmp34b;

  6426.                         r0_tm_4[0] = tmp34a - tmp34b;

  6427. 

  6428.                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25f) * 4.f);

  6429.                         float tmp56b = (tmp0[1] * 2.f - tmp0[3] * 2.5f + tmp0[5] * 0.5f);

  6430. 

  6431.                         r0_tm_4[1] = tmp56a + tmp56b;

  6432.                         r0_tm_4[2] = tmp56a - tmp56b;

  6433. 

  6434.                         r0_tm_0 += img0_tm.w * tiles * 2;

  6435.                         r0_tm_4 += img0_tm.w * tiles * 2;

  6436.                     }

  6437. #endif // __ARM_NEON

  6438.                 }

  6439.             }

  6440.         }

  6441.     }

  6442.     bottom_blob_bordered = Mat();

  6443.     // END transform input

  6444. 

  6445.     // BEGIN dot

  6446.     Mat top_blob_tm;

  6447.     {

  6448.         int w_tm = outw / 6 * 8;

  6449.         int h_tm = outh / 6 * 8;

  6450.         top_blob_tm.create(4, 16 * w_tm/8 * h_tm/8, outch, 4u, opt.workspace_allocator);

  6451. 

  6452.         const int tiles = h_tm/8 * w_tm/8;

  6453. 

  6454.         int nn_outch = outch >> 2;

  6455.         int remain_outch_start = nn_outch << 2;

  6456. 

  6457.         #pragma omp parallel for num_threads(opt.num_threads)

  6458.         for (int pp=0; pp<nn_outch; pp++)

  6459.         {

  6460.             int p = pp * 4;

  6461. 

  6462.             Mat out0_tm = top_blob_tm.channel(p);

  6463.             Mat out1_tm = top_blob_tm.channel(p+1);

  6464.             Mat out2_tm = top_blob_tm.channel(p+2);

  6465.             Mat out3_tm = top_blob_tm.channel(p+3);

  6466. 

  6467.             const float* ktm = kernel_tm.channel(pp);

  6468. 

  6469.             out0_tm.fill(0.f);

  6470.             out1_tm.fill(0.f);

  6471.             out2_tm.fill(0.f);

  6472.             out3_tm.fill(0.f);

  6473. 

  6474.             int q = 0;

  6475. 

  6476. #if __ARM_NEON && __aarch64__

  6477.             for (; q+3<inch; q+=4)

  6478.             {

  6479.                 const float* r0 = bottom_blob_tm.channel(q);

  6480.                 const float* r1 = bottom_blob_tm.channel(q+1);

  6481.                 const float* r2 = bottom_blob_tm.channel(q+2);

  6482.                 const float* r3 = bottom_blob_tm.channel(q+3);

  6483. 

  6484.                 float* output0_tm = out0_tm;

  6485.                 float* output1_tm = out1_tm;

  6486.                 float* output2_tm = out2_tm;

  6487.                 float* output3_tm = out3_tm;

  6488. 

  6489.                 asm volatile(

  6490.                     "mov    w0, #16                     \n"// w0 = r = 16

  6491.                     "0:                                 \n"

  6492. 

  6493.                     "prfm   pldl1keep, [%8, #512]                       \n"

  6494.                     "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%8], #64     \n"// v0  v1  v2  v3  = _k00 _k01 _k02 _k03

  6495. 

  6496.                     "prfm   pldl1keep, [%8, #512]                       \n"

  6497.                     "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%8], #64     \n"// v4  v5  v6  v7  = _k10 _k11 _k12 _k13

  6498. 

  6499.                     "prfm   pldl1keep, [%8, #512]                       \n"

  6500.                     "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64   \n"// v8  v9  v10 v11 = _k20 _k21 _k22 _k23

  6501. 

  6502.                     "prfm   pldl1keep, [%8, #512]                       \n"

  6503.                     "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%8], #64 \n"// v12 v13 v14 v15 = _k30 _k31 _k32 _k33

  6504. 

  6505.                     // tile loop

  6506.                     "lsr    w1, %w18, #2                \n"// w1 = nn = tiles >> 2

  6507.                     "cmp    w1, #0                      \n"

  6508.                     "beq    2f                          \n"

  6509. 

  6510.                     //BEGIN tile loop

  6511.                     "prfm   pldl1keep, [%4, #128]       \n"//

  6512.                     "ld1    {v16.4s}, [%4], #16         \n"

  6513. 

  6514.                     "1:                                 \n"

  6515. 

  6516.                     "prfm   pldl1keep, [%0, #128]       \n"

  6517.                     "ld1    {v20.4s}, [%0]              \n"

  6518.                     "add    x4, %0, #16                 \n"// x4 = %0 next

  6519. 

  6520.                     "fmla   v20.4s, v16.4s, v0.4s       \n"

  6521. 

  6522.                     "prfm   pldl1keep, [%1, #128]       \n"

  6523.                     "ld1    {v21.4s}, [%1]              \n"

  6524.                     "add    x5, %1, #16                 \n"// x5 = %1 next

  6525. 

  6526.                     "fmla   v21.4s, v16.4s, v4.4s       \n"

  6527. 

  6528.                     "prfm   pldl1keep, [%2, #128]       \n"

  6529.                     "ld1    {v22.4s}, [%2]              \n"

  6530.                     "add    x6, %2, #16                 \n"// x6 = %2 next

  6531. 

  6532.                     "fmla   v22.4s, v16.4s, v8.4s       \n"

  6533. 

  6534.                     "prfm   pldl1keep, [%3, #128]       \n"

  6535.                     "ld1    {v23.4s}, [%3]              \n"

  6536.                     "add    x7, %3, #16                 \n"// x7 = %3 next

  6537. 

  6538.                     "prfm   pldl1keep, [%5, #128]       \n"

  6539.                     "ld1    {v17.4s}, [%5], #16         \n"

  6540. 

  6541.                     "fmla   v23.4s, v16.4s, v12.4s      \n"

  6542. 

  6543.                     "prfm   pldl1keep, [x4, #128]       \n"

  6544.                     "ld1    {v24.4s}, [x4]              \n"

  6545. 

  6546.                     "fmla   v20.4s, v17.4s, v1.4s       \n"

  6547.                     "fmla   v21.4s, v17.4s, v5.4s       \n"

  6548. 

  6549.                     "prfm   pldl1keep, [%6, #128]       \n"

  6550.                     "ld1    {v18.4s}, [%6], #16         \n"

  6551. 

  6552.                     "fmla   v22.4s, v17.4s, v9.4s       \n"

  6553.                     "fmla   v23.4s, v17.4s, v13.4s      \n"

  6554. 

  6555.                     "prfm   pldl1keep, [x5, #128]       \n"

  6556.                     "ld1    {v25.4s}, [x5]              \n"

  6557. 

  6558.                     "fmla   v20.4s, v18.4s, v2.4s       \n"

  6559.                     "fmla   v21.4s, v18.4s, v6.4s       \n"

  6560. 

  6561.                     "prfm   pldl1keep, [%7, #128]       \n"

  6562.                     "ld1    {v19.4s}, [%7], #16         \n"

  6563. 

  6564.                     "fmla   v22.4s, v18.4s, v10.4s      \n"

  6565.                     "fmla   v23.4s, v18.4s, v14.4s      \n"

  6566. 

  6567.                     "prfm   pldl1keep, [x6, #128]       \n"

  6568.                     "ld1    {v26.4s}, [x6]              \n"

  6569. 

  6570.                     "fmla   v20.4s, v19.4s, v3.4s       \n"

  6571.                     "fmla   v21.4s, v19.4s, v7.4s       \n"

  6572. 

  6573.                     "prfm   pldl1keep, [%4, #128]       \n"

  6574.                     "ld1    {v16.4s}, [%4], #16         \n"

  6575. 

  6576.                     "fmla   v22.4s, v19.4s, v11.4s      \n"

  6577.                     "fmla   v23.4s, v19.4s, v15.4s      \n"

  6578. 

  6579.                     ///////

  6580. 

  6581.                     "prfm   pldl1keep, [x7, #128]       \n"

  6582.                     "ld1    {v27.4s}, [x7]              \n"

  6583. 

  6584.                     "st1    {v20.4s}, [%0]              \n"

  6585.                     "add    %0, %0, #32                 \n"

  6586. 

  6587.                     "fmla   v24.4s, v16.4s, v0.4s       \n"

  6588.                     "fmla   v25.4s, v16.4s, v4.4s       \n"

  6589. 

  6590.                     "prfm   pldl1keep, [%5, #128]       \n"

  6591.                     "ld1    {v17.4s}, [%5], #16         \n"

  6592. 

  6593.                     "fmla   v26.4s, v16.4s, v8.4s       \n"

  6594.                     "fmla   v27.4s, v16.4s, v12.4s      \n"

  6595. 

  6596.                     "prfm   pldl1keep, [%0, #128]       \n"

  6597.                     "ld1    {v20.4s}, [%0]              \n"

  6598. 

  6599.                     "st1    {v21.4s}, [%1]              \n"

  6600.                     "add    %1, %1, #32                 \n"

  6601. 

  6602.                     "fmla   v24.4s, v17.4s, v1.4s       \n"

  6603.                     "fmla   v25.4s, v17.4s, v5.4s       \n"

  6604. 

  6605.                     "prfm   pldl1keep, [%6, #128]       \n"

  6606.                     "ld1    {v18.4s}, [%6], #16         \n"

  6607. 

  6608.                     "fmla   v26.4s, v17.4s, v9.4s       \n"

  6609.                     "fmla   v27.4s, v17.4s, v13.4s      \n"

  6610. 

  6611.                     "prfm   pldl1keep, [%1, #128]       \n"

  6612.                     "ld1    {v21.4s}, [%1]              \n"

  6613. 

  6614.                     "st1    {v22.4s}, [%2]              \n"

  6615.                     "add    %2, %2, #32                 \n"

  6616. 

  6617.                     "fmla   v24.4s, v18.4s, v2.4s       \n"

  6618.                     "fmla   v25.4s, v18.4s, v6.4s       \n"

  6619. 

  6620.                     "prfm   pldl1keep, [%7, #128]       \n"

  6621.                     "ld1    {v19.4s}, [%7], #16         \n"

  6622. 

  6623.                     "fmla   v26.4s, v18.4s, v10.4s      \n"

  6624.                     "fmla   v27.4s, v18.4s, v14.4s      \n"

  6625. 

  6626.                     "prfm   pldl1keep, [%2, #128]       \n"

  6627.                     "ld1    {v22.4s}, [%2]              \n"

  6628. 

  6629.                     "st1    {v23.4s}, [%3]              \n"

  6630.                     "add    %3, %3, #32                 \n"

  6631. 

  6632.                     "fmla   v24.4s, v19.4s, v3.4s       \n"

  6633.                     "fmla   v25.4s, v19.4s, v7.4s       \n"

  6634. 

  6635.                     "prfm   pldl1keep, [%4, #128]       \n"

  6636.                     "ld1    {v16.4s}, [%4], #16         \n"

  6637. 

  6638.                     "fmla   v26.4s, v19.4s, v11.4s      \n"

  6639.                     "fmla   v27.4s, v19.4s, v15.4s      \n"

  6640. 

  6641.                     ///////

  6642. 

  6643.                     "prfm   pldl1keep, [%3, #128]       \n"

  6644.                     "ld1    {v23.4s}, [%3]              \n"

  6645. 

  6646.                     "st1    {v24.4s}, [x4]              \n"

  6647.                     "add    x4, x4, #32                 \n"

  6648. 

  6649.                     "fmla   v20.4s, v16.4s, v0.4s       \n"

  6650.                     "fmla   v21.4s, v16.4s, v4.4s       \n"

  6651. 

  6652.                     "prfm   pldl1keep, [%5, #128]       \n"

  6653.                     "ld1    {v17.4s}, [%5], #16         \n"

  6654. 

  6655.                     "fmla   v22.4s, v16.4s, v8.4s       \n"

  6656.                     "fmla   v23.4s, v16.4s, v12.4s      \n"

  6657. 

  6658.                     "prfm   pldl1keep, [x4, #128]       \n"

  6659.                     "ld1    {v24.4s}, [x4]              \n"

  6660. 

  6661.                     "st1    {v25.4s}, [x5]              \n"

  6662.                     "add    x5, x5, #32                 \n"

  6663. 

  6664.                     "fmla   v20.4s, v17.4s, v1.4s       \n"

  6665.                     "fmla   v21.4s, v17.4s, v5.4s       \n"

  6666. 

  6667.                     "prfm   pldl1keep, [%6, #128]       \n"

  6668.                     "ld1    {v18.4s}, [%6], #16         \n"

  6669. 

  6670.                     "fmla   v22.4s, v17.4s, v9.4s       \n"

  6671.                     "fmla   v23.4s, v17.4s, v13.4s      \n"

  6672. 

  6673.                     "prfm   pldl1keep, [x5, #128]       \n"

  6674.                     "ld1    {v25.4s}, [x5]              \n"

  6675. 

  6676.                     "st1    {v26.4s}, [x6]              \n"

  6677.                     "add    x6, x6, #32                 \n"

  6678. 

  6679.                     "fmla   v20.4s, v18.4s, v2.4s       \n"

  6680.                     "fmla   v21.4s, v18.4s, v6.4s       \n"

  6681. 

  6682.                     "prfm   pldl1keep, [%7, #128]       \n"

  6683.                     "ld1    {v19.4s}, [%7], #16         \n"

  6684. 

  6685.                     "fmla   v22.4s, v18.4s, v10.4s      \n"

  6686.                     "fmla   v23.4s, v18.4s, v14.4s      \n"

  6687. 

  6688.                     "prfm   pldl1keep, [x6, #128]       \n"

  6689.                     "ld1    {v26.4s}, [x6]              \n"

  6690. 

  6691.                     "st1    {v27.4s}, [x7]              \n"

  6692.                     "add    x7, x7, #32                 \n"

  6693. 

  6694.                     "fmla   v20.4s, v19.4s, v3.4s       \n"

  6695.                     "fmla   v21.4s, v19.4s, v7.4s       \n"

  6696. 

  6697.                     "prfm   pldl1keep, [%4, #128]       \n"

  6698.                     "ld1    {v16.4s}, [%4], #16         \n"

  6699. 

  6700.                     "fmla   v22.4s, v19.4s, v11.4s      \n"

  6701.                     "fmla   v23.4s, v19.4s, v15.4s      \n"

  6702. 

  6703.                     ///////

  6704. 

  6705.                     "prfm   pldl1keep, [x7, #128]       \n"

  6706.                     "ld1    {v27.4s}, [x7]              \n"

  6707. 

  6708.                     "st1    {v20.4s}, [%0]              \n"

  6709. 

  6710.                     "fmla   v24.4s, v16.4s, v0.4s       \n"

  6711.                     "fmla   v25.4s, v16.4s, v4.4s       \n"

  6712. 

  6713.                     "prfm   pldl1keep, [%5, #128]       \n"

  6714.                     "ld1    {v17.4s}, [%5], #16         \n"

  6715. 

  6716.                     "fmla   v26.4s, v16.4s, v8.4s       \n"

  6717.                     "fmla   v27.4s, v16.4s, v12.4s      \n"

  6718. 

  6719.                     "st1    {v21.4s}, [%1]              \n"

  6720. 

  6721.                     "fmla   v24.4s, v17.4s, v1.4s       \n"

  6722.                     "fmla   v25.4s, v17.4s, v5.4s       \n"

  6723. 

  6724.                     "prfm   pldl1keep, [%6, #128]       \n"

  6725.                     "ld1    {v18.4s}, [%6], #16         \n"

  6726. 

  6727.                     "fmla   v26.4s, v17.4s, v9.4s       \n"

  6728.                     "fmla   v27.4s, v17.4s, v13.4s      \n"

  6729. 

  6730.                     "st1    {v22.4s}, [%2]              \n"

  6731. 

  6732.                     "fmla   v24.4s, v18.4s, v2.4s       \n"

  6733.                     "fmla   v25.4s, v18.4s, v6.4s       \n"

  6734. 

  6735.                     "prfm   pldl1keep, [%7, #128]       \n"

  6736.                     "ld1    {v19.4s}, [%7], #16         \n"

  6737. 

  6738.                     "fmla   v26.4s, v18.4s, v10.4s      \n"

  6739.                     "fmla   v27.4s, v18.4s, v14.4s      \n"

  6740. 

  6741.                     "st1    {v23.4s}, [%3]              \n"

  6742. 

  6743.                     "fmla   v24.4s, v19.4s, v3.4s       \n"

  6744.                     "fmla   v25.4s, v19.4s, v7.4s       \n"

  6745. 

  6746.                     "prfm   pldl1keep, [%4, #128]       \n"

  6747.                     "ld1    {v16.4s}, [%4], #16         \n"

  6748. 

  6749.                     "fmla   v26.4s, v19.4s, v11.4s      \n"

  6750.                     "fmla   v27.4s, v19.4s, v15.4s      \n"

  6751. 

  6752.                     "st1    {v24.4s}, [x4], #16         \n"

  6753.                     "mov    %0, x4                      \n"

  6754. 

  6755.                     "st1    {v25.4s}, [x5], #16         \n"

  6756.                     "mov    %1, x5                      \n"

  6757. 

  6758.                     "subs   w1, w1, #1                  \n"

  6759. 

  6760.                     "st1    {v26.4s}, [x6], #16         \n"

  6761.                     "mov    %2, x6                      \n"

  6762. 

  6763.                     "st1    {v27.4s}, [x7], #16         \n"

  6764.                     "mov    %3, x7                      \n"

  6765. 

  6766.                     "bne    1b                          \n"

  6767.                     "sub    %4, %4, #16                 \n"

  6768.                     //END tile loop

  6769. 

  6770.                     "2:                                 \n"

  6771. 

  6772.                     // remain loop

  6773.                     "and    w1, %w18, #3                \n"// w1 = remain = tiles & 3;

  6774.                     "cmp    w1, #0                      \n"

  6775.                     "beq    4f                          \n"

  6776. 

  6777.                     //BEGIN remain loop

  6778.                     "3:                                 \n"

  6779. 

  6780.                     "prfm   pldl1keep, [%4, #128]       \n"

  6781.                     "ld1    {v16.4s}, [%4], #16         \n"

  6782. 

  6783.                     "prfm   pldl1keep, [%0, #128]       \n"

  6784.                     "ld1    {v20.4s}, [%0]              \n"

  6785. 

  6786.                     "fmla   v20.4s, v16.4s, v0.4s       \n"

  6787. 

  6788.                     "prfm   pldl1keep, [%1, #128]       \n"

  6789.                     "ld1    {v21.4s}, [%1]              \n"

  6790. 

  6791.                     "fmla   v21.4s, v16.4s, v4.4s       \n"

  6792. 

  6793.                     "prfm   pldl1keep, [%2, #128]       \n"

  6794.                     "ld1    {v22.4s}, [%2]              \n"

  6795. 

  6796.                     "fmla   v22.4s, v16.4s, v8.4s       \n"

  6797. 

  6798.                     "prfm   pldl1keep, [%3, #128]       \n"

  6799.                     "ld1    {v23.4s}, [%3]              \n"

  6800. 

  6801.                     "fmla   v23.4s, v16.4s, v12.4s      \n"

  6802. 

  6803.                     "prfm   pldl1keep, [%5, #128]       \n"

  6804.                     "ld1    {v17.4s}, [%5], #16         \n"

  6805. 

  6806.                     "fmla   v20.4s, v17.4s, v1.4s       \n"

  6807.                     "fmla   v21.4s, v17.4s, v5.4s       \n"

  6808. 

  6809.                     "fmla   v22.4s, v17.4s, v9.4s       \n"

  6810.                     "fmla   v23.4s, v17.4s, v13.4s      \n"

  6811. 

  6812.                     "prfm   pldl1keep, [%6, #128]       \n"

  6813.                     "ld1    {v18.4s}, [%6], #16         \n"

  6814. 

  6815.                     "fmla   v20.4s, v18.4s, v2.4s       \n"

  6816.                     "fmla   v21.4s, v18.4s, v6.4s       \n"

  6817. 

  6818.                     "fmla   v22.4s, v18.4s, v10.4s      \n"

  6819.                     "fmla   v23.4s, v18.4s, v14.4s      \n"

  6820. 

  6821.                     "prfm   pldl1keep, [%7, #128]       \n"

  6822.                     "ld1    {v19.4s}, [%7], #16         \n"

  6823. 

  6824.                     "fmla   v20.4s, v19.4s, v3.4s       \n"

  6825.                     "fmla   v21.4s, v19.4s, v7.4s       \n"

  6826.                     "fmla   v22.4s, v19.4s, v11.4s      \n"

  6827.                     "fmla   v23.4s, v19.4s, v15.4s      \n"

  6828. 

  6829.                     "st1    {v20.4s}, [%0], #16         \n"

  6830.                     "st1    {v21.4s}, [%1], #16         \n"

  6831. 

  6832.                     "subs   w1, w1, #1                  \n"

  6833. 

  6834.                     "st1    {v22.4s}, [%2], #16         \n"

  6835.                     "st1    {v23.4s}, [%3], #16         \n"

  6836. 

  6837.                     "bne    3b                          \n"

  6838.                     //END remain loop

  6839. 

  6840.                     "4:                                 \n"

  6841. 

  6842.                     "subs   w0, w0, #1                  \n"

  6843.                     "bne    0b                          \n"

  6844. 

  6845.                     : "=r"(output0_tm), // %0

  6846.                       "=r"(output1_tm), // %1

  6847.                       "=r"(output2_tm), // %2

  6848.                       "=r"(output3_tm), // %3

  6849.                       "=r"(r0),         // %4

  6850.                       "=r"(r1),         // %5

  6851.                       "=r"(r2),         // %6

  6852.                       "=r"(r3),         // %7

  6853.                       "=r"(ktm)         // %8

  6854.                     : "0"(output0_tm),

  6855.                       "1"(output1_tm),

  6856.                       "2"(output2_tm),

  6857.                       "3"(output3_tm),

  6858.                       "4"(r0),

  6859.                       "5"(r1),

  6860.                       "6"(r2),

  6861.                       "7"(r3),

  6862.                       "8"(ktm),

  6863.                       "r"(tiles)        // %18

  6864.                     : "cc", "memory", "x0", "x1", "x4", "x5", "x6", "x7", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"

  6865.                 );

  6866.             }

  6867. #endif // __ARM_NEON && __aarch64__

  6868. 

  6869.             for (; q+1<inch; q+=2)

  6870.             {

  6871.                 const float* r0 = bottom_blob_tm.channel(q);

  6872.                 const float* r1 = bottom_blob_tm.channel(q+1);

  6873. 

  6874.                 float* output0_tm = out0_tm;

  6875.                 float* output1_tm = out1_tm;

  6876.                 float* output2_tm = out2_tm;

  6877.                 float* output3_tm = out3_tm;

  6878. 

  6879. #if __ARM_NEON

  6880. #if __aarch64__

  6881.                 asm volatile(

  6882.                     "mov    w0, #16                     \n"// w0 = r = 16

  6883.                     "0:                                 \n"

  6884. 

  6885.                     "prfm   pldl1keep, [%6, #256]       \n"

  6886.                     "ld1    {v0.4s, v1.4s}, [%6], #32   \n"// v0 v1 = _k00 _k01

  6887. 

  6888.                     "prfm   pldl1keep, [%6, #256]       \n"

  6889.                     "ld1    {v2.4s, v3.4s}, [%6], #32   \n"// v2 v3 = _k10 _k11

  6890. 

  6891.                     "prfm   pldl1keep, [%6, #256]       \n"

  6892.                     "ld1    {v4.4s, v5.4s}, [%6], #32   \n"// v4 v5 = _k20 _k21

  6893. 

  6894.                     "prfm   pldl1keep, [%6, #256]       \n"

  6895.                     "ld1    {v6.4s, v7.4s}, [%6], #32   \n"// v6 v7 = _k30 _k31

  6896. 

  6897.                     // tile loop

  6898.                     "lsr    w1, %w14, #2                \n"// w1 = nn = tiles >> 2

  6899.                     "cmp    w1, #0                      \n"

  6900.                     "beq    2f                          \n"

  6901. 

  6902.                     //BEGIN tile loop

  6903.                     "prfm   pldl1keep, [%4, #128]       \n"

  6904.                     "ld1    {v20.4s}, [%4], #16         \n"

  6905. 

  6906.                     "1:                                 \n"

  6907. 

  6908.                     "prfm   pldl1keep, [%0, #128]       \n"

  6909.                     "ld1    {v16.4s}, [%0]              \n"

  6910. 

  6911.                     "fmla   v16.4s, v20.4s, v0.4s       \n"

  6912. 

  6913.                     "prfm   pldl1keep, [%1, #128]       \n"

  6914.                     "ld1    {v17.4s}, [%1]              \n"

  6915. 

  6916.                     "fmla   v17.4s, v20.4s, v2.4s       \n"

  6917. 

  6918.                     "prfm   pldl1keep, [%2, #128]       \n"

  6919.                     "ld1    {v18.4s}, [%2]              \n"

  6920. 

  6921.                     "fmla   v18.4s, v20.4s, v4.4s       \n"

  6922. 

  6923.                     "prfm   pldl1keep, [%3, #128]       \n"

  6924.                     "ld1    {v19.4s}, [%3]              \n"

  6925. 

  6926.                     "fmla   v19.4s, v20.4s, v6.4s       \n"

  6927. 

  6928.                     "prfm   pldl1keep, [%5, #128]       \n"

  6929.                     "ld1    {v21.4s}, [%5], #16         \n"

  6930. 

  6931.                     "fmla   v16.4s, v21.4s, v1.4s       \n"

  6932.                     "fmla   v17.4s, v21.4s, v3.4s       \n"

  6933. 

  6934.                     "prfm   pldl1keep, [%4, #128]       \n"

  6935.                     "ld1    {v20.4s}, [%4], #16         \n"

  6936. 

  6937.                     "fmla   v18.4s, v21.4s, v5.4s       \n"

  6938.                     "fmla   v19.4s, v21.4s, v7.4s       \n"

  6939. 

  6940.                     "st1    {v16.4s}, [%0], #16         \n"

  6941.                     "st1    {v17.4s}, [%1], #16         \n"

  6942. 

  6943.                     ////

  6944. 

  6945.                     "prfm   pldl1keep, [%0, #128]       \n"

  6946.                     "ld1    {v16.4s}, [%0]              \n"

  6947. 

  6948.                     "fmla   v16.4s, v20.4s, v0.4s       \n"

  6949. 

  6950.                     "prfm   pldl1keep, [%1, #128]       \n"

  6951.                     "ld1    {v17.4s}, [%1]              \n"

  6952. 

  6953.                     "fmla   v17.4s, v20.4s, v2.4s       \n"

  6954. 

  6955.                     "st1    {v18.4s}, [%2], #16         \n"

  6956.                     "st1    {v19.4s}, [%3], #16         \n"

  6957. 

  6958.                     "prfm   pldl1keep, [%2, #128]       \n"

  6959.                     "ld1    {v18.4s}, [%2]              \n"

  6960. 

  6961.                     "fmla   v18.4s, v20.4s, v4.4s       \n"

  6962. 

  6963.                     "prfm   pldl1keep, [%3, #128]       \n"

  6964.                     "ld1    {v19.4s}, [%3]              \n"

  6965. 

  6966.                     "fmla   v19.4s, v20.4s, v6.4s       \n"

  6967. 

  6968.                     "prfm   pldl1keep, [%5, #128]       \n"

  6969.                     "ld1    {v21.4s}, [%5], #16         \n"

  6970. 

  6971.                     "fmla   v16.4s, v21.4s, v1.4s       \n"

  6972.                     "fmla   v17.4s, v21.4s, v3.4s       \n"

  6973. 

  6974.                     "prfm   pldl1keep, [%4, #128]       \n"

  6975.                     "ld1    {v20.4s}, [%4], #16         \n"

  6976. 

  6977.                     "fmla   v18.4s, v21.4s, v5.4s       \n"

  6978.                     "fmla   v19.4s, v21.4s, v7.4s       \n"

  6979. 

  6980.                     "st1    {v16.4s}, [%0], #16         \n"

  6981.                     "st1    {v17.4s}, [%1], #16         \n"

  6982. 

  6983.                     ////

  6984. 

  6985.                     "prfm   pldl1keep, [%0, #128]       \n"

  6986.                     "ld1    {v16.4s}, [%0]              \n"

  6987. 

  6988.                     "fmla   v16.4s, v20.4s, v0.4s       \n"

  6989. 

  6990.                     "prfm   pldl1keep, [%1, #128]       \n"

  6991.                     "ld1    {v17.4s}, [%1]              \n"

  6992. 

  6993.                     "fmla   v17.4s, v20.4s, v2.4s       \n"

  6994. 

  6995.                     "st1    {v18.4s}, [%2], #16         \n"

  6996.                     "st1    {v19.4s}, [%3], #16         \n"

  6997. 

  6998.                     "prfm   pldl1keep, [%2, #128]       \n"

  6999.                     "ld1    {v18.4s}, [%2]              \n"

  7000. 

  7001.                     "fmla   v18.4s, v20.4s, v4.4s       \n"

  7002. 

  7003.                     "prfm   pldl1keep, [%3, #128]       \n"

  7004.                     "ld1    {v19.4s}, [%3]              \n"

  7005. 

  7006.                     "fmla   v19.4s, v20.4s, v6.4s       \n"

  7007. 

  7008.                     "prfm   pldl1keep, [%5, #128]       \n"

  7009.                     "ld1    {v21.4s}, [%5], #16         \n"

  7010. 

  7011.                     "fmla   v16.4s, v21.4s, v1.4s       \n"

  7012.                     "fmla   v17.4s, v21.4s, v3.4s       \n"

  7013. 

  7014.                     "prfm   pldl1keep, [%4, #128]       \n"

  7015.                     "ld1    {v20.4s}, [%4], #16         \n"

  7016. 

  7017.                     "fmla   v18.4s, v21.4s, v5.4s       \n"

  7018.                     "fmla   v19.4s, v21.4s, v7.4s       \n"

  7019. 

  7020.                     "st1    {v16.4s}, [%0], #16         \n"

  7021.                     "st1    {v17.4s}, [%1], #16         \n"

  7022. 

  7023.                     ////

  7024. 

  7025.                     "prfm   pldl1keep, [%0, #128]       \n"

  7026.                     "ld1    {v16.4s}, [%0]              \n"

  7027. 

  7028.                     "fmla   v16.4s, v20.4s, v0.4s       \n"

  7029. 

  7030.                     "prfm   pldl1keep, [%1, #128]       \n"

  7031.                     "ld1    {v17.4s}, [%1]              \n"

  7032. 

  7033.                     "fmla   v17.4s, v20.4s, v2.4s       \n"

  7034. 

  7035.                     "st1    {v18.4s}, [%2], #16         \n"

  7036.                     "st1    {v19.4s}, [%3], #16         \n"

  7037. 

  7038.                     "prfm   pldl1keep, [%2, #128]       \n"

  7039.                     "ld1    {v18.4s}, [%2]              \n"

  7040. 

  7041.                     "fmla   v18.4s, v20.4s, v4.4s       \n"

  7042. 

  7043.                     "prfm   pldl1keep, [%3, #128]       \n"

  7044.                     "ld1    {v19.4s}, [%3]              \n"

  7045. 

  7046.                     "fmla   v19.4s, v20.4s, v6.4s       \n"

  7047. 

  7048.                     "prfm   pldl1keep, [%5, #128]       \n"

  7049.                     "ld1    {v21.4s}, [%5], #16         \n"

  7050. 

  7051.                     "fmla   v16.4s, v21.4s, v1.4s       \n"

  7052.                     "fmla   v17.4s, v21.4s, v3.4s       \n"

  7053. 

  7054.                     "prfm   pldl1keep, [%4, #128]       \n"

  7055.                     "ld1    {v20.4s}, [%4], #16         \n"

  7056. 

  7057.                     "fmla   v18.4s, v21.4s, v5.4s       \n"

  7058.                     "fmla   v19.4s, v21.4s, v7.4s       \n"

  7059. 

  7060.                     "st1    {v16.4s}, [%0], #16         \n"

  7061.                     "st1    {v17.4s}, [%1], #16         \n"

  7062. 

  7063.                     "subs   w1, w1, #1                  \n"

  7064. 

  7065.                     "st1    {v18.4s}, [%2], #16         \n"

  7066.                     "st1    {v19.4s}, [%3], #16         \n"

  7067. 

  7068.                     "bne    1b                          \n"

  7069.                     "sub    %4, %4, #16                 \n"

  7070.                     //END tile loop

  7071. 

  7072.                     "2:                                 \n"

  7073. 

  7074.                     // remain loop

  7075.                     "and    w1, %w14, #3                \n"// w1 = remain = tiles & 3;

  7076.                     "cmp    w1, #0                      \n"

  7077.                     "beq    4f                          \n"

  7078. 

  7079.                     //BEGIN remain loop

  7080.                     "3:                                 \n"

  7081. 

  7082.                     "prfm   pldl1keep, [%4, #128]       \n"

  7083.                     "ld1    {v20.4s}, [%4], #16         \n"

  7084. 

  7085.                     "prfm   pldl1keep, [%0, #128]       \n"

  7086.                     "ld1    {v16.4s}, [%0]              \n"

  7087. 

  7088.                     "fmla   v16.4s, v20.4s, v0.4s       \n"

  7089. 

  7090.                     "prfm   pldl1keep, [%1, #128]       \n"

  7091.                     "ld1    {v17.4s}, [%1]              \n"

  7092. 

  7093.                     "fmla   v17.4s, v20.4s, v2.4s       \n"

  7094. 

  7095.                     "prfm   pldl1keep, [%2, #128]       \n"

  7096.                     "ld1    {v18.4s}, [%2]              \n"

  7097. 

  7098.                     "fmla   v18.4s, v20.4s, v4.4s       \n"

  7099. 

  7100.                     "prfm   pldl1keep, [%3, #128]       \n"

  7101.                     "ld1    {v19.4s}, [%3]              \n"

  7102. 

  7103.                     "fmla   v19.4s, v20.4s, v6.4s       \n"

  7104. 

  7105.                     "prfm   pldl1keep, [%5, #128]       \n"

  7106.                     "ld1    {v21.4s}, [%5], #16         \n"

  7107. 

  7108.                     "fmla   v16.4s, v21.4s, v1.4s       \n"

  7109.                     "fmla   v17.4s, v21.4s, v3.4s       \n"

  7110.                     "fmla   v18.4s, v21.4s, v5.4s       \n"

  7111.                     "fmla   v19.4s, v21.4s, v7.4s       \n"

  7112. 

  7113.                     "st1    {v16.4s}, [%0], #16         \n"

  7114.                     "st1    {v17.4s}, [%1], #16         \n"

  7115. 

  7116.                     "subs   w1, w1, #1                  \n"

  7117. 

  7118.                     "st1    {v18.4s}, [%2], #16         \n"

  7119.                     "st1    {v19.4s}, [%3], #16         \n"

  7120. 

  7121.                     "bne    3b                          \n"

  7122.                     //END remain loop

  7123. 

  7124.                     "4:                                 \n"

  7125. 

  7126.                     "subs   w0, w0, #1                  \n"

  7127.                     "bne    0b                          \n"

  7128. 

  7129.                     : "=r"(output0_tm), // %0

  7130.                       "=r"(output1_tm), // %1

  7131.                       "=r"(output2_tm), // %2

  7132.                       "=r"(output3_tm), // %3

  7133.                       "=r"(r0),         // %4

  7134.                       "=r"(r1),         // %5

  7135.                       "=r"(ktm)         // %6

  7136.                     : "0"(output0_tm),

  7137.                       "1"(output1_tm),

  7138.                       "2"(output2_tm),

  7139.                       "3"(output3_tm),

  7140.                       "4"(r0),

  7141.                       "5"(r1),

  7142.                       "6"(ktm),

  7143.                       "r"(tiles)        // %14

  7144.                     : "cc", "memory", "x0", "x1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21"

  7145.                 );

  7146. #else

  7147.                 asm volatile(

  7148.                     "mov        r0, #16                 \n"// r0 = r = 16

  7149.                     "0:                                 \n"

  7150. 

  7151.                     "pld        [%6, #256]              \n"

  7152.                     "vld1.f32   {d0-d3}, [%6 :128]!     \n"// q0 q1 = _k00 _k01

  7153. 

  7154.                     "pld        [%6, #256]              \n"

  7155.                     "vld1.f32   {d4-d7}, [%6 :128]!     \n"// q2 q3 = _k10 _k11

  7156. 

  7157.                     "pld        [%6, #256]              \n"

  7158.                     "vld1.f32   {d8-d11}, [%6 :128]!    \n"// q4 q5 = _k20 _k21

  7159. 

  7160.                     "pld        [%6, #256]              \n"

  7161.                     "vld1.f32   {d12-d15}, [%6 :128]!   \n"// q6 q7 = _k30 _k31

  7162. 

  7163.                     // tile loop

  7164.                     "lsr        r1, %14, #2             \n"// r1 = nn = tiles >> 2

  7165.                     "cmp        r1, #0                  \n"

  7166.                     "beq        2f                      \n"

  7167. 

  7168.                     //BEGIN tile loop

  7169.                     "pld        [%4, #128]              \n"

  7170.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7171. 

  7172.                     "1:                                 \n"

  7173. 

  7174.                     "pld        [%0, #128]              \n"

  7175.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7176. 

  7177.                     "vmla.f32   q8, q12, q0             \n"

  7178. 

  7179.                     "pld        [%1, #128]              \n"

  7180.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7181. 

  7182.                     "vmla.f32   q9, q12, q2             \n"

  7183. 

  7184.                     "pld        [%2, #128]              \n"

  7185.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7186. 

  7187.                     "vmla.f32   q10, q12, q4            \n"

  7188. 

  7189.                     "pld        [%3, #128]              \n"

  7190.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7191. 

  7192.                     "vmla.f32   q11, q12, q6            \n"

  7193. 

  7194.                     "pld        [%5, #128]              \n"

  7195.                     "vld1.f32   {d26-d27}, [%5 :128]!   \n"// q13 = _r1

  7196. 

  7197.                     "vmla.f32   q8, q13, q1             \n"

  7198.                     "vmla.f32   q9, q13, q3             \n"

  7199. 

  7200.                     "pld        [%4, #128]              \n"

  7201.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7202. 

  7203.                     "vmla.f32   q10, q13, q5            \n"

  7204.                     "vmla.f32   q11, q13, q7            \n"

  7205. 

  7206.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7207.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7208. 

  7209.                     ////

  7210. 

  7211.                     "pld        [%0, #128]              \n"

  7212.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7213. 

  7214.                     "vmla.f32   q8, q12, q0             \n"

  7215. 

  7216.                     "pld        [%1, #128]              \n"

  7217.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7218. 

  7219.                     "vmla.f32   q9, q12, q2             \n"

  7220. 

  7221.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7222.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7223. 

  7224.                     "pld        [%2, #128]              \n"

  7225.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7226. 

  7227.                     "vmla.f32   q10, q12, q4            \n"

  7228. 

  7229.                     "pld        [%3, #128]              \n"

  7230.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7231. 

  7232.                     "vmla.f32   q11, q12, q6            \n"

  7233. 

  7234.                     "pld        [%5, #128]              \n"

  7235.                     "vld1.f32   {d26-d27}, [%5 :128]!   \n"// q13 = _r1

  7236. 

  7237.                     "vmla.f32   q8, q13, q1             \n"

  7238.                     "vmla.f32   q9, q13, q3             \n"

  7239. 

  7240.                     "pld        [%4, #128]              \n"

  7241.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7242. 

  7243.                     "vmla.f32   q10, q13, q5            \n"

  7244.                     "vmla.f32   q11, q13, q7            \n"

  7245. 

  7246.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7247.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7248. 

  7249.                     ////

  7250. 

  7251.                     "pld        [%0, #128]              \n"

  7252.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7253. 

  7254.                     "vmla.f32   q8, q12, q0             \n"

  7255. 

  7256.                     "pld        [%1, #128]              \n"

  7257.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7258. 

  7259.                     "vmla.f32   q9, q12, q2             \n"

  7260. 

  7261.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7262.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7263. 

  7264.                     "pld        [%2, #128]              \n"

  7265.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7266. 

  7267.                     "vmla.f32   q10, q12, q4            \n"

  7268. 

  7269.                     "pld        [%3, #128]              \n"

  7270.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7271. 

  7272.                     "vmla.f32   q11, q12, q6            \n"

  7273. 

  7274.                     "pld        [%5, #128]              \n"

  7275.                     "vld1.f32   {d26-d27}, [%5 :128]!   \n"// q13 = _r1

  7276. 

  7277.                     "vmla.f32   q8, q13, q1             \n"

  7278.                     "vmla.f32   q9, q13, q3             \n"

  7279. 

  7280.                     "pld        [%4, #128]              \n"

  7281.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7282. 

  7283.                     "vmla.f32   q10, q13, q5            \n"

  7284.                     "vmla.f32   q11, q13, q7            \n"

  7285. 

  7286.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7287.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7288. 

  7289.                     ////

  7290. 

  7291.                     "pld        [%0, #128]              \n"

  7292.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7293. 

  7294.                     "vmla.f32   q8, q12, q0             \n"

  7295. 

  7296.                     "pld        [%1, #128]              \n"

  7297.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7298. 

  7299.                     "vmla.f32   q9, q12, q2             \n"

  7300. 

  7301.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7302.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7303. 

  7304.                     "pld        [%2, #128]              \n"

  7305.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7306. 

  7307.                     "vmla.f32   q10, q12, q4            \n"

  7308. 

  7309.                     "pld        [%3, #128]              \n"

  7310.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7311. 

  7312.                     "vmla.f32   q11, q12, q6            \n"

  7313. 

  7314.                     "pld        [%5, #128]              \n"

  7315.                     "vld1.f32   {d26-d27}, [%5 :128]!   \n"// q13 = _r1

  7316. 

  7317.                     "vmla.f32   q8, q13, q1             \n"

  7318.                     "vmla.f32   q9, q13, q3             \n"

  7319. 

  7320.                     "pld        [%4, #128]              \n"

  7321.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7322. 

  7323.                     "vmla.f32   q10, q13, q5            \n"

  7324.                     "vmla.f32   q11, q13, q7            \n"

  7325. 

  7326.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7327.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7328. 

  7329.                     "subs       r1, #1                  \n"

  7330. 

  7331.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7332.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7333. 

  7334.                     "bne        1b                      \n"

  7335.                     "sub        %4, %4, #16             \n"

  7336.                     //END tile loop

  7337. 

  7338.                     "2:                                 \n"

  7339. 

  7340.                     // remain loop

  7341.                     "and        r1, %14, #3             \n"// r1 = remain = tiles & 3;

  7342.                     "cmp        r1, #0                  \n"

  7343.                     "beq        4f                      \n"

  7344. 

  7345.                     //BEGIN remain loop

  7346.                     "3:                                 \n"

  7347. 

  7348.                     "pld        [%4, #128]              \n"

  7349.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7350. 

  7351.                     "pld        [%0, #128]              \n"

  7352.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7353. 

  7354.                     "vmla.f32   q8, q12, q0             \n"

  7355. 

  7356.                     "pld        [%1, #128]              \n"

  7357.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7358. 

  7359.                     "vmla.f32   q9, q12, q2             \n"

  7360. 

  7361.                     "pld        [%2, #128]              \n"

  7362.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7363. 

  7364.                     "vmla.f32   q10, q12, q4            \n"

  7365. 

  7366.                     "pld        [%3, #128]              \n"

  7367.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7368. 

  7369.                     "vmla.f32   q11, q12, q6            \n"

  7370. 

  7371.                     "pld        [%5, #128]              \n"

  7372.                     "vld1.f32   {d26-d27}, [%5 :128]!   \n"// q13 = _r1

  7373. 

  7374.                     "vmla.f32   q8, q13, q1             \n"

  7375.                     "vmla.f32   q9, q13, q3             \n"

  7376.                     "vmla.f32   q10, q13, q5            \n"

  7377.                     "vmla.f32   q11, q13, q7            \n"

  7378. 

  7379.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7380.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7381. 

  7382.                     "subs       r1, #1                  \n"

  7383. 

  7384.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7385.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7386. 

  7387.                     "bne        3b                      \n"

  7388.                     //END remain loop

  7389. 

  7390.                     "4:                                 \n"

  7391. 

  7392.                     "subs       r0, #1                  \n"

  7393.                     "bne        0b                      \n"

  7394. 

  7395.                     : "=r"(output0_tm), // %0

  7396.                       "=r"(output1_tm), // %1

  7397.                       "=r"(output2_tm), // %2

  7398.                       "=r"(output3_tm), // %3

  7399.                       "=r"(r0),         // %4

  7400.                       "=r"(r1),         // %5

  7401.                       "=r"(ktm)         // %6

  7402.                     : "0"(output0_tm),

  7403.                       "1"(output1_tm),

  7404.                       "2"(output2_tm),

  7405.                       "3"(output3_tm),

  7406.                       "4"(r0),

  7407.                       "5"(r1),

  7408.                       "6"(ktm),

  7409.                       "r"(tiles)        // %14

  7410.                     : "cc", "memory", "r0", "r1", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13"

  7411.                 );

  7412. #endif // __aarch64__

  7413. #else

  7414.                 for (int r=0; r<16; r++)

  7415.                 {

  7416.                     for (int t=0; t<tiles; t++)

  7417.                     {

  7418.                         for (int m=0; m<4; m++)

  7419.                         {

  7420.                             output0_tm[m] += r0[m] * ktm[0 +m];

  7421.                             output0_tm[m] += r1[m] * ktm[4 +m];

  7422.                             output1_tm[m] += r0[m] * ktm[8 +m];

  7423.                             output1_tm[m] += r1[m] * ktm[12+m];

  7424.                             output2_tm[m] += r0[m] * ktm[16+m];

  7425.                             output2_tm[m] += r1[m] * ktm[20+m];

  7426.                             output3_tm[m] += r0[m] * ktm[24+m];

  7427.                             output3_tm[m] += r1[m] * ktm[28+m];

  7428.                         }

  7429. 

  7430.                         r0 += 4;

  7431.                         r1 += 4;

  7432.                         output0_tm += 4;

  7433.                         output1_tm += 4;

  7434.                         output2_tm += 4;

  7435.                         output3_tm += 4;

  7436.                     }

  7437. 

  7438.                     ktm += 32;

  7439.                 }

  7440. #endif // __ARM_NEON

  7441.             }

  7442. 

  7443.             for (; q<inch; q++)

  7444.             {

  7445.                 const float* r0 = bottom_blob_tm.channel(q);

  7446. 

  7447.                 float* output0_tm = out0_tm;

  7448.                 float* output1_tm = out1_tm;

  7449.                 float* output2_tm = out2_tm;

  7450.                 float* output3_tm = out3_tm;

  7451. 

  7452. #if __ARM_NEON

  7453. #if __aarch64__

  7454.                 asm volatile(

  7455.                     "mov    w0, #16                     \n"// w0 = r = 16

  7456.                     "0:                                 \n"

  7457. 

  7458.                     "prfm   pldl1keep, [%5, #256]       \n"

  7459.                     "ld1    {v0.4s, v1.4s}, [%5], #32   \n"// v0 v1 = _k00 _k10

  7460. 

  7461.                     "prfm   pldl1keep, [%5, #256]       \n"

  7462.                     "ld1    {v2.4s, v3.4s}, [%5], #32   \n"// v2 v3 = _k20 _k30

  7463. 

  7464.                     // tile loop

  7465.                     "mov    w1, %w12                    \n"// w1 = tiles

  7466.                     "cmp    w1, #0                      \n"

  7467.                     "beq    2f                          \n"

  7468. 

  7469.                     //BEGIN tile loop

  7470.                     "1:                                 \n"

  7471. 

  7472.                     "prfm   pldl1keep, [%4, #128]       \n"

  7473.                     "ld1    {v16.4s}, [%4], #16         \n"

  7474. 

  7475.                     "prfm   pldl1keep, [%0, #128]       \n"

  7476.                     "ld1    {v17.4s}, [%0]              \n"

  7477. 

  7478.                     "fmla   v17.4s, v16.4s, v0.4s       \n"

  7479. 

  7480.                     "prfm   pldl1keep, [%1, #128]       \n"

  7481.                     "ld1    {v18.4s}, [%1]              \n"

  7482. 

  7483.                     "fmla   v18.4s, v16.4s, v1.4s       \n"

  7484. 

  7485.                     "prfm   pldl1keep, [%2, #128]       \n"

  7486.                     "ld1    {v19.4s}, [%2]              \n"

  7487. 

  7488.                     "fmla   v19.4s, v16.4s, v2.4s       \n"

  7489. 

  7490.                     "prfm   pldl1keep, [%3, #128]       \n"

  7491.                     "ld1    {v20.4s}, [%3]              \n"

  7492. 

  7493.                     "fmla   v20.4s, v16.4s, v3.4s       \n"

  7494. 

  7495.                     "st1    {v17.4s}, [%0], #16         \n"

  7496.                     "st1    {v18.4s}, [%1], #16         \n"

  7497. 

  7498.                     "subs   w1, w1, #1                  \n"

  7499. 

  7500.                     "st1    {v19.4s}, [%2], #16         \n"

  7501.                     "st1    {v20.4s}, [%3], #16         \n"

  7502. 

  7503.                     "bne    1b                          \n"

  7504.                     //END tile loop

  7505. 

  7506.                     "2:                                 \n"

  7507. 

  7508.                     "subs   w0, w0, #1                  \n"

  7509.                     "bne    0b                          \n"

  7510. 

  7511.                     : "=r"(output0_tm), // %0

  7512.                       "=r"(output1_tm), // %1

  7513.                       "=r"(output2_tm), // %2

  7514.                       "=r"(output3_tm), // %3

  7515.                       "=r"(r0),         // %4

  7516.                       "=r"(ktm)         // %5

  7517.                     : "0"(output0_tm),

  7518.                       "1"(output1_tm),

  7519.                       "2"(output2_tm),

  7520.                       "3"(output3_tm),

  7521.                       "4"(r0),

  7522.                       "5"(ktm),

  7523.                       "r"(tiles)        // %12

  7524.                     : "cc", "memory", "x0", "x1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20"

  7525.                 );

  7526. #else

  7527.                 asm volatile(

  7528.                     "mov        r0, #16                 \n"// r0 = r = 16

  7529.                     "0:                                 \n"

  7530. 

  7531.                     "pld        [%5, #256]              \n"

  7532.                     "vld1.f32   {d0-d3}, [%5 :128]!     \n"// q0 q1 = _k00 _k10

  7533. 

  7534.                     "pld        [%5, #256]              \n"

  7535.                     "vld1.f32   {d4-d7}, [%5 :128]!     \n"// q2 q3 = _k20 _k30

  7536. 

  7537.                     // tile loop

  7538.                     "mov        r1, %12                 \n"// r1 = tiles

  7539.                     "cmp        r1, #0                  \n"

  7540.                     "beq        2f                      \n"

  7541. 

  7542.                     //BEGIN tile loop

  7543.                     "1:                                 \n"

  7544. 

  7545.                     "pld        [%4, #128]              \n"

  7546.                     "vld1.f32   {d24-d25}, [%4 :128]!   \n"// q12 = _r0

  7547. 

  7548.                     "pld        [%0, #128]              \n"

  7549.                     "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7550. 

  7551.                     "vmla.f32   q8, q12, q0             \n"

  7552. 

  7553.                     "pld        [%1, #128]              \n"

  7554.                     "vld1.f32   {d18-d19}, [%1 :128]    \n"// q9 = _output1_tm

  7555. 

  7556.                     "vmla.f32   q9, q12, q1             \n"

  7557. 

  7558.                     "pld        [%2, #128]              \n"

  7559.                     "vld1.f32   {d20-d21}, [%2 :128]    \n"// q10 = _output2_tm

  7560. 

  7561.                     "vmla.f32   q10, q12, q2            \n"

  7562. 

  7563.                     "pld        [%3, #128]              \n"

  7564.                     "vld1.f32   {d22-d23}, [%3 :128]    \n"// q11 = _output3_tm

  7565. 

  7566.                     "vmla.f32   q11, q12, q3            \n"

  7567. 

  7568.                     "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7569.                     "vst1.f32   {d18-d19}, [%1 :128]!   \n"

  7570. 

  7571.                     "subs       r1, #1                  \n"

  7572. 

  7573.                     "vst1.f32   {d20-d21}, [%2 :128]!   \n"

  7574.                     "vst1.f32   {d22-d23}, [%3 :128]!   \n"

  7575. 

  7576.                     "bne        1b                      \n"

  7577.                     //END tile loop

  7578. 

  7579.                     "2:                                 \n"

  7580. 

  7581.                     "subs       r0, #1                  \n"

  7582.                     "bne        0b                      \n"

  7583. 

  7584.                     : "=r"(output0_tm), // %0

  7585.                       "=r"(output1_tm), // %1

  7586.                       "=r"(output2_tm), // %2

  7587.                       "=r"(output3_tm), // %3

  7588.                       "=r"(r0),         // %4

  7589.                       "=r"(ktm)         // %5

  7590.                     : "0"(output0_tm),

  7591.                       "1"(output1_tm),

  7592.                       "2"(output2_tm),

  7593.                       "3"(output3_tm),

  7594.                       "4"(r0),

  7595.                       "5"(ktm),

  7596.                       "r"(tiles)        // %12

  7597.                     : "cc", "memory", "r0", "r1", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13"

  7598.                 );

  7599. #endif // __aarch64__

  7600. #else

  7601.                 for (int r=0; r<16; r++)

  7602.                 {

  7603.                     for (int t=0; t<tiles; t++)

  7604.                     {

  7605.                         for (int m=0; m<4; m++)

  7606.                         {

  7607.                             output0_tm[m] += r0[m] * ktm[0 +m];

  7608.                             output1_tm[m] += r0[m] * ktm[4 +m];

  7609.                             output2_tm[m] += r0[m] * ktm[8 +m];

  7610.                             output3_tm[m] += r0[m] * ktm[12+m];

  7611.                         }

  7612. 

  7613.                         r0 += 4;

  7614.                         output0_tm += 4;

  7615.                         output1_tm += 4;

  7616.                         output2_tm += 4;

  7617.                         output3_tm += 4;

  7618.                     }

  7619. 

  7620.                     ktm += 16;

  7621.                 }

  7622. #endif // __ARM_NEON

  7623.             }

  7624.         }

  7625. 

  7626.         #pragma omp parallel for num_threads(opt.num_threads)

  7627.         for (int p = remain_outch_start; p<outch; p++)

  7628.         {

  7629.             Mat out0_tm = top_blob_tm.channel(p);

  7630. 

  7631.             const float* ktm = (const float*)kernel_tm.channel(nn_outch) + 8*8 * inch * (p-remain_outch_start);

  7632. 

  7633.             out0_tm.fill(0.f);

  7634. 

  7635.             int q = 0;

  7636. 

  7637.             for (; q<inch; q++)

  7638.             {

  7639.                 const float* r0 = bottom_blob_tm.channel(q);

  7640. 

  7641.                 float* output0_tm = out0_tm;

  7642. 

  7643.                 for (int r=0; r<16; r++)

  7644.                 {

  7645. #if __ARM_NEON

  7646.                 float32x4_t _k00 = vld1q_f32(ktm); ktm += 4;

  7647. #endif // __ARM_NEON

  7648. 

  7649.                 // tile

  7650.                 for (int i=0; i<tiles; i++)

  7651.                 {

  7652. #if __ARM_NEON

  7653. #if __aarch64__

  7654.                     asm volatile(

  7655.                         "prfm   pldl1keep, [%1, #128]   \n"

  7656.                         "ld1    {v17.4s}, [%1], #16     \n"

  7657. 

  7658.                         "prfm   pldl1keep, [%0, #128]   \n"

  7659.                         "ld1    {v16.4s}, [%0]          \n"

  7660. 

  7661.                         "fmla   v16.4s, v17.4s, %4.4s   \n"

  7662. 

  7663.                         "st1    {v16.4s}, [%0], #16     \n"

  7664.                         : "=r"(output0_tm), // %0

  7665.                           "=r"(r0)          // %1

  7666.                         : "0"(output0_tm),

  7667.                           "1"(r0),

  7668.                           "w"(_k00)         // %4

  7669.                         : "cc", "memory", "v16", "v17"

  7670.                     );

  7671. #else

  7672.                     asm volatile(

  7673.                         "pld        [%1, #128]              \n"

  7674.                         "vld1.f32   {d18-d19}, [%1 :128]!   \n"// q9 = _r0

  7675. 

  7676.                         "pld        [%0, #128]              \n"

  7677.                         "vld1.f32   {d16-d17}, [%0 :128]    \n"// q8 = _output0_tm

  7678. 

  7679.                         "vmla.f32   q8, q9, %q4             \n"

  7680. 

  7681.                         "vst1.f32   {d16-d17}, [%0 :128]!   \n"

  7682.                         : "=r"(output0_tm), // %0

  7683.                           "=r"(r0)          // %1

  7684.                         : "0"(output0_tm),

  7685.                           "1"(r0),

  7686.                           "w"(_k00)         // %4

  7687.                         : "cc", "memory", "q8", "q9"

  7688.                     );

  7689. #endif // __aarch64__

  7690. #else

  7691.                     for (int m=0; m<4; m++)

  7692.                     {

  7693.                         output0_tm[m] += r0[m] * ktm[m];

  7694.                     }

  7695. 

  7696.                     r0 += 4;

  7697.                     output0_tm += 4;

  7698. #endif // __ARM_NEON

  7699.                 }

  7700. 

  7701. #if !__ARM_NEON

  7702.                 ktm += 4;

  7703. #endif // __ARM_NEON

  7704.                 }

  7705.             }

  7706.         }

  7707.     }

  7708.     bottom_blob_tm = Mat();

  7709.     // END dot

  7710. 

  7711.     // BEGIN transform output

  7712.     Mat top_blob_bordered;

  7713.     top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator);

  7714.     {

  7715. //         const float otm[6][8] = {

  7716. //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  7717. //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  7718. //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  7719. //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  7720. //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  7721. //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  7722. //         };

  7723. 

  7724.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  7725.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  7726.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  7727.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  7728.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  7729.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  7730. 

  7731. #if __ARM_NEON

  7732.         const float coeff[4] = { 4.f, 8.f, 16.f, 32.f };

  7733.         float32x4_t _coeff = vld1q_f32(coeff);

  7734. #endif // __ARM_NEON

  7735. 

  7736.         int w_tm = outw / 6 * 8;

  7737.         int h_tm = outh / 6 * 8;

  7738.         const int tiles = w_tm/8 * h_tm/8;

  7739. 

  7740.         #pragma omp parallel for num_threads(opt.num_threads)

  7741.         for (int p = 0; p<outch; p++)

  7742.         {

  7743.             const Mat out0_tm = top_blob_tm.channel(p);

  7744.             Mat out0 = top_blob_bordered.channel(p);

  7745. 

  7746.             const float bias0 = bias ? bias[p] : 0.f;

  7747. #if __ARM_NEON

  7748.             float32x2_t _bias0 = vdup_n_f32(bias0);

  7749. #endif // __ARM_NEON

  7750. 

  7751.             float tmp[6][8];

  7752. 

  7753.             // tile

  7754.             for (int i=0; i<outh/6; i++)

  7755.             {

  7756.                 for (int j=0; j<outw/6; j++)

  7757.                 {

  7758. #if __ARM_NEON

  7759.                     const float* output0_tm0_0 = out0_tm.row(i * w_tm/8 + j);

  7760.                     const float* output0_tm0_4 = out0_tm.row(i * w_tm/8 + j + tiles);

  7761.                     const float* output0_tm1_0 = out0_tm.row(i * w_tm/8 + j + tiles*2);

  7762.                     const float* output0_tm1_4 = out0_tm.row(i * w_tm/8 + j + tiles*3);

  7763.                     const float* output0_tm2_0 = out0_tm.row(i * w_tm/8 + j + tiles*4);

  7764.                     const float* output0_tm2_4 = out0_tm.row(i * w_tm/8 + j + tiles*5);

  7765.                     const float* output0_tm3_0 = out0_tm.row(i * w_tm/8 + j + tiles*6);

  7766.                     const float* output0_tm3_4 = out0_tm.row(i * w_tm/8 + j + tiles*7);

  7767. 

  7768. #if __aarch64__

  7769.                     for (int m=0; m+3<8; m+=4)

  7770.                     {

  7771.                         float32x4_t _output0_tm0_0123 = vld1q_f32(output0_tm0_0);

  7772.                         float32x4_t _output0_tm0_4567 = vld1q_f32(output0_tm0_4);

  7773.                         float32x4_t _output0_tm1_0123 = vld1q_f32(output0_tm1_0);

  7774.                         float32x4_t _output0_tm1_4567 = vld1q_f32(output0_tm1_4);

  7775.                         float32x4_t _output0_tm2_0123 = vld1q_f32(output0_tm2_0);

  7776.                         float32x4_t _output0_tm2_4567 = vld1q_f32(output0_tm2_4);

  7777.                         float32x4_t _output0_tm3_0123 = vld1q_f32(output0_tm3_0);

  7778.                         float32x4_t _output0_tm3_4567 = vld1q_f32(output0_tm3_4);

  7779. 

  7780.                         float32x4x2_t _output0_tm01_00221133 = vtrnq_f32(_output0_tm0_0123, _output0_tm1_0123);

  7781.                         float32x4x2_t _output0_tm01_44665577 = vtrnq_f32(_output0_tm0_4567, _output0_tm1_4567);

  7782.                         float32x4x2_t _output0_tm23_00221133 = vtrnq_f32(_output0_tm2_0123, _output0_tm3_0123);

  7783.                         float32x4x2_t _output0_tm23_44665577 = vtrnq_f32(_output0_tm2_4567, _output0_tm3_4567);

  7784. 

  7785.                         // no vswp intrinsic  :(

  7786.                         float32x4_t _output0_tm_00 = vcombine_f32(vget_low_f32(_output0_tm01_00221133.val[0]), vget_low_f32(_output0_tm23_00221133.val[0]));

  7787.                         float32x4_t _output0_tm_11 = vcombine_f32(vget_low_f32(_output0_tm01_00221133.val[1]), vget_low_f32(_output0_tm23_00221133.val[1]));

  7788.                         float32x4_t _output0_tm_22 = vcombine_f32(vget_high_f32(_output0_tm01_00221133.val[0]), vget_high_f32(_output0_tm23_00221133.val[0]));

  7789.                         float32x4_t _output0_tm_33 = vcombine_f32(vget_high_f32(_output0_tm01_00221133.val[1]), vget_high_f32(_output0_tm23_00221133.val[1]));

  7790.                         float32x4_t _output0_tm_44 = vcombine_f32(vget_low_f32(_output0_tm01_44665577.val[0]), vget_low_f32(_output0_tm23_44665577.val[0]));

  7791.                         float32x4_t _output0_tm_55 = vcombine_f32(vget_low_f32(_output0_tm01_44665577.val[1]), vget_low_f32(_output0_tm23_44665577.val[1]));

  7792.                         float32x4_t _output0_tm_66 = vcombine_f32(vget_high_f32(_output0_tm01_44665577.val[0]), vget_high_f32(_output0_tm23_44665577.val[0]));

  7793.                         float32x4_t _output0_tm_77 = vcombine_f32(vget_high_f32(_output0_tm01_44665577.val[1]), vget_high_f32(_output0_tm23_44665577.val[1]));

  7794. 

  7795.                         float32x4_t _tmp024a = vaddq_f32(_output0_tm_11, _output0_tm_22);

  7796.                         float32x4_t _tmp135a = vsubq_f32(_output0_tm_11, _output0_tm_22);

  7797. 

  7798.                         float32x4_t _tmp024b = vaddq_f32(_output0_tm_33, _output0_tm_44);

  7799.                         float32x4_t _tmp135b = vsubq_f32(_output0_tm_33, _output0_tm_44);

  7800. 

  7801.                         float32x4_t _tmp024c = vaddq_f32(_output0_tm_55, _output0_tm_66);

  7802.                         float32x4_t _tmp135c = vsubq_f32(_output0_tm_55, _output0_tm_66);

  7803. 

  7804.                         float32x4_t _tmp0 = vaddq_f32(_output0_tm_00, _tmp024a);

  7805.                         _tmp0 = vmlaq_lane_f32(_tmp0, _tmp024c, vget_high_f32(_coeff), 1);

  7806.                         _tmp0 = vaddq_f32(_tmp0, _tmp024b);

  7807. 

  7808.                         float32x4_t _tmp2 = vmlaq_lane_f32(_tmp024a, _tmp024b, vget_low_f32(_coeff), 0);

  7809.                         _tmp2 = vmlaq_lane_f32(_tmp2, _tmp024c, vget_low_f32(_coeff), 1);

  7810. 

  7811.                         float32x4_t _tmp4 = vmlaq_lane_f32(_tmp024a, _tmp024b, vget_high_f32(_coeff), 0);

  7812.                         _tmp4 = vaddq_f32(_tmp4, _tmp024c);

  7813.                         _tmp4 = vaddq_f32(_tmp4, _tmp024c);

  7814. 

  7815.                         vst1q_f32(&tmp[0][m], _tmp0);

  7816.                         vst1q_f32(&tmp[2][m], _tmp2);

  7817.                         vst1q_f32(&tmp[4][m], _tmp4);

  7818. 

  7819.                         float32x4_t _tmp1 = vmlaq_lane_f32(_tmp135a, _tmp135c, vget_high_f32(_coeff), 0);

  7820.                         _tmp1 = vaddq_f32(_tmp1, _tmp135b);

  7821.                         _tmp1 = vaddq_f32(_tmp1, _tmp135b);

  7822. 

  7823.                         float32x4_t _tmp3 = vmlaq_lane_f32(_tmp135a, _tmp135b, vget_low_f32(_coeff), 1);

  7824.                         _tmp3 = vmlaq_lane_f32(_tmp3, _tmp135c, vget_low_f32(_coeff), 0);

  7825. 

  7826.                         float32x4_t _tmp5 = vaddq_f32(_output0_tm_77, _tmp135a);

  7827.                         _tmp5 = vmlaq_lane_f32(_tmp5, _tmp135b, vget_high_f32(_coeff), 1);

  7828.                         _tmp5 = vaddq_f32(_tmp5, _tmp135c);

  7829. 

  7830.                         vst1q_f32(&tmp[1][m], _tmp1);

  7831.                         vst1q_f32(&tmp[3][m], _tmp3);

  7832.                         vst1q_f32(&tmp[5][m], _tmp5);

  7833. 

  7834.                         output0_tm0_0 += out0_tm.w * tiles * 2*4;

  7835.                         output0_tm0_4 += out0_tm.w * tiles * 2*4;

  7836.                         output0_tm1_0 += out0_tm.w * tiles * 2*4;

  7837.                         output0_tm1_4 += out0_tm.w * tiles * 2*4;

  7838.                         output0_tm2_0 += out0_tm.w * tiles * 2*4;

  7839.                         output0_tm2_4 += out0_tm.w * tiles * 2*4;

  7840.                         output0_tm3_0 += out0_tm.w * tiles * 2*4;

  7841.                         output0_tm3_4 += out0_tm.w * tiles * 2*4;

  7842.                     }

  7843. 

  7844.                     const float* t0 = tmp[0];

  7845.                     const float* t1 = tmp[1];

  7846. 

  7847.                     float* output0 = out0.row(i * 6) + j * 6;

  7848.                     float* output1 = output0 + outw;

  7849. 

  7850.                     for (int m=0; m+1<6; m+=2)

  7851.                     {

  7852.                         float32x4_t _t0_0123 = vld1q_f32(t0);

  7853.                         float32x4_t _t0_4567 = vld1q_f32(t0+4);

  7854.                         float32x4_t _t1_0123 = vld1q_f32(t1);

  7855.                         float32x4_t _t1_4567 = vld1q_f32(t1+4);

  7856. 

  7857.                         float32x4x2_t _t01_00221133 = vtrnq_f32(_t0_0123, _t1_0123);

  7858.                         float32x4x2_t _t01_44665577 = vtrnq_f32(_t0_4567, _t1_4567);

  7859. 

  7860.                         float32x2_t _t_00 = vget_low_f32(_t01_00221133.val[0]);

  7861.                         float32x2_t _t_11 = vget_low_f32(_t01_00221133.val[1]);

  7862.                         float32x2_t _t_22 = vget_high_f32(_t01_00221133.val[0]);

  7863.                         float32x2_t _t_33 = vget_high_f32(_t01_00221133.val[1]);

  7864.                         float32x2_t _t_44 = vget_low_f32(_t01_44665577.val[0]);

  7865.                         float32x2_t _t_55 = vget_low_f32(_t01_44665577.val[1]);

  7866.                         float32x2_t _t_66 = vget_high_f32(_t01_44665577.val[0]);

  7867.                         float32x2_t _t_77 = vget_high_f32(_t01_44665577.val[1]);

  7868. 

  7869.                         float32x2_t _tmp024a = vadd_f32(_t_11, _t_22);

  7870.                         float32x2_t _tmp135a = vsub_f32(_t_11, _t_22);

  7871. 

  7872.                         float32x2_t _tmp024b = vadd_f32(_t_33, _t_44);

  7873.                         float32x2_t _tmp135b = vsub_f32(_t_33, _t_44);

  7874. 

  7875.                         float32x2_t _tmp024c = vadd_f32(_t_55, _t_66);

  7876.                         float32x2_t _tmp135c = vsub_f32(_t_55, _t_66);

  7877. 

  7878.                         float32x2_t _output_0 = vadd_f32(_t_00, _tmp024a);

  7879.                         _output_0 = vmla_lane_f32(_output_0, _tmp024c, vget_high_f32(_coeff), 1);

  7880.                         _output_0 = vadd_f32(_output_0, _tmp024b);

  7881.                         _output_0 = vadd_f32(_output_0, _bias0);

  7882. 

  7883.                         float32x2_t _output_2 = vmla_lane_f32(_tmp024a, _tmp024b, vget_low_f32(_coeff), 0);

  7884.                         _output_2 = vmla_lane_f32(_output_2, _tmp024c, vget_low_f32(_coeff), 1);

  7885.                         _output_2 = vadd_f32(_output_2, _bias0);

  7886. 

  7887.                         float32x2_t _output_4 = vmla_lane_f32(_tmp024a, _tmp024b, vget_high_f32(_coeff), 0);

  7888.                         _output_4 = vadd_f32(_output_4, _tmp024c);

  7889.                         _output_4 = vadd_f32(_output_4, _tmp024c);

  7890.                         _output_4 = vadd_f32(_output_4, _bias0);

  7891. 

  7892.                         output0[0] = vget_lane_f32(_output_0, 0);

  7893.                         output1[0] = vget_lane_f32(_output_0, 1);

  7894.                         output0[2] = vget_lane_f32(_output_2, 0);

  7895.                         output1[2] = vget_lane_f32(_output_2, 1);

  7896.                         output0[4] = vget_lane_f32(_output_4, 0);

  7897.                         output1[4] = vget_lane_f32(_output_4, 1);

  7898. 

  7899.                         float32x2_t _output_1 = vmla_lane_f32(_tmp135a, _tmp135c, vget_high_f32(_coeff), 0);

  7900.                         _output_1 = vadd_f32(_output_1, _tmp135b);

  7901.                         _output_1 = vadd_f32(_output_1, _tmp135b);

  7902.                         _output_1 = vadd_f32(_output_1, _bias0);

  7903. 

  7904.                         float32x2_t _output_3 = vmla_lane_f32(_tmp135a, _tmp135b, vget_low_f32(_coeff), 1);

  7905.                         _output_3 = vmla_lane_f32(_output_3, _tmp135c, vget_low_f32(_coeff), 0);

  7906.                         _output_3 = vadd_f32(_output_3, _bias0);

  7907. 

  7908.                         float32x2_t _output_5 = vadd_f32(_t_77, _tmp135a);

  7909.                         _output_5 = vmla_lane_f32(_output_5, _tmp135b, vget_high_f32(_coeff), 1);

  7910.                         _output_5 = vadd_f32(_output_5, _tmp135c);

  7911.                         _output_5 = vadd_f32(_output_5, _bias0);

  7912. 

  7913.                         output0[1] = vget_lane_f32(_output_1, 0);

  7914.                         output1[1] = vget_lane_f32(_output_1, 1);

  7915.                         output0[3] = vget_lane_f32(_output_3, 0);

  7916.                         output1[3] = vget_lane_f32(_output_3, 1);

  7917.                         output0[5] = vget_lane_f32(_output_5, 0);

  7918.                         output1[5] = vget_lane_f32(_output_5, 1);

  7919. 

  7920.                         t0 += 8*2;

  7921.                         t1 += 8*2;

  7922.                         output0 += outw*2;

  7923.                         output1 += outw*2;

  7924.                     }

  7925. #else // __aarch64__

  7926.                     float* t0 = tmp[0];

  7927.                     float* t1 = tmp[1];

  7928. 

  7929.                     int step = out0_tm.w * tiles * 2*4 *4;

  7930. 

  7931.                     asm volatile(

  7932. 

  7933.                         // loop0

  7934.                         "vld1.f32   {d16-d17}, [%2], %21 \n"

  7935.                         "vld1.f32   {d18-d19}, [%3], %21 \n"

  7936.                         "vld1.f32   {d20-d21}, [%4], %21 \n"

  7937.                         "vld1.f32   {d22-d23}, [%5], %21 \n"

  7938.                         "vld1.f32   {d24-d25}, [%6], %21 \n"

  7939.                         "vld1.f32   {d26-d27}, [%7], %21 \n"

  7940.                         "vld1.f32   {d28-d29}, [%8], %21 \n"

  7941.                         "vld1.f32   {d30-d31}, [%9], %21 \n"

  7942. 

  7943.                         "vtrn.32    q8, q10             \n"

  7944.                         "vtrn.32    q9, q11             \n"

  7945.                         "vtrn.32    q12, q14            \n"

  7946.                         "vtrn.32    q13, q15            \n"

  7947. 

  7948.                         "vswp       d17, d24            \n"

  7949.                         "vswp       d19, d26            \n"

  7950.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  7951.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  7952. 

  7953.                         "vadd.f32   q2, q10, q12        \n"

  7954.                         "vsub.f32   q3, q10, q12        \n"

  7955. 

  7956.                         "vadd.f32   q4, q14, q9         \n"

  7957.                         "vsub.f32   q5, q14, q9         \n"

  7958. 

  7959.                         "vadd.f32   q6, q11, q13        \n"

  7960.                         "vsub.f32   q7, q11, q13        \n"// spare q9 q10 q11 q12 q13 q14

  7961. 

  7962.                         "vmov       q9, q3              \n"

  7963.                         "vadd.f32   q8, q8, q2          \n"

  7964.                         "vmla.f32   q9, q7, %f20[0]     \n"

  7965.                         "vmov       q12, q2             \n"

  7966.                         "vmov       q10, q2             \n"

  7967.                         "vmov       q11, q3             \n"

  7968.                         "vmla.f32   q12, q4, %f20[0]    \n"

  7969.                         "vadd.f32   q15, q15, q3        \n"

  7970.                         "vmla.f32   q8, q6, %f20[1]     \n"

  7971.                         "vadd.f32   q9, q9, q5          \n"

  7972.                         "vmla.f32   q10, q4, %e20[0]    \n"

  7973.                         "vmla.f32   q11, q5, %e20[1]    \n"

  7974.                         "vadd.f32   q12, q12, q6        \n"

  7975.                         "vmla.f32   q15, q5, %f20[1]    \n"

  7976.                         "vadd.f32   q8, q8, q4          \n"

  7977.                         "vadd.f32   q9, q9, q5          \n"

  7978.                         "vmla.f32   q10, q6, %e20[1]    \n"

  7979.                         "vmla.f32   q11, q7, %e20[0]    \n"

  7980.                         "vadd.f32   q12, q12, q6        \n"

  7981.                         "vadd.f32   q15, q15, q7        \n"

  7982. 

  7983.                         "vst1.f32   {d16-d17}, [%0]     \n"

  7984.                         "add        %0, %0, #64         \n"

  7985. 

  7986.                         "vst1.f32   {d18-d19}, [%1]     \n"

  7987.                         "add        %1, %1, #64         \n"

  7988. 

  7989.                         "vst1.f32   {d20-d21}, [%0]     \n"

  7990.                         "add        %0, %0, #64         \n"

  7991. 

  7992.                         "vst1.f32   {d22-d23}, [%1]     \n"

  7993.                         "add        %1, %1, #64         \n"

  7994. 

  7995.                         "vst1.f32   {d24-d25}, [%0]     \n"

  7996.                         "sub        %0, %0, #112        \n"

  7997. 

  7998.                         "vst1.f32   {d30-d31}, [%1]     \n"

  7999.                         "sub        %1, %1, #112        \n"

  8000. 

  8001.                         // loop1

  8002.                         "vld1.f32   {d16-d17}, [%2]     \n"

  8003.                         "vld1.f32   {d18-d19}, [%3]     \n"

  8004.                         "vld1.f32   {d20-d21}, [%4]     \n"

  8005.                         "vld1.f32   {d22-d23}, [%5]     \n"

  8006.                         "vld1.f32   {d24-d25}, [%6]     \n"

  8007.                         "vld1.f32   {d26-d27}, [%7]     \n"

  8008.                         "vld1.f32   {d28-d29}, [%8]     \n"

  8009.                         "vld1.f32   {d30-d31}, [%9]     \n"

  8010. 

  8011.                         "vtrn.32    q8, q10             \n"

  8012.                         "vtrn.32    q9, q11             \n"

  8013.                         "vtrn.32    q12, q14            \n"

  8014.                         "vtrn.32    q13, q15            \n"

  8015. 

  8016.                         "vswp       d17, d24            \n"

  8017.                         "vswp       d19, d26            \n"

  8018.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  8019.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  8020. 

  8021.                         "vadd.f32   q2, q10, q12        \n"

  8022.                         "vsub.f32   q3, q10, q12        \n"

  8023. 

  8024.                         "vadd.f32   q4, q14, q9         \n"

  8025.                         "vsub.f32   q5, q14, q9         \n"

  8026. 

  8027.                         "vadd.f32   q6, q11, q13        \n"

  8028.                         "vsub.f32   q7, q11, q13        \n"// spare q9 q10 q11 q12 q13 q14

  8029. 

  8030.                         "vmov       q9, q3              \n"

  8031.                         "vadd.f32   q8, q8, q2          \n"

  8032.                         "vmla.f32   q9, q7, %f20[0]     \n"

  8033.                         "vmov       q12, q2             \n"

  8034.                         "vmov       q10, q2             \n"

  8035.                         "vmov       q11, q3             \n"

  8036.                         "vmla.f32   q12, q4, %f20[0]    \n"

  8037.                         "vadd.f32   q15, q15, q3        \n"

  8038.                         "vmla.f32   q8, q6, %f20[1]     \n"

  8039.                         "vadd.f32   q9, q9, q5          \n"

  8040.                         "vmla.f32   q10, q4, %e20[0]    \n"

  8041.                         "vmla.f32   q11, q5, %e20[1]    \n"

  8042.                         "vadd.f32   q12, q12, q6        \n"

  8043.                         "vmla.f32   q15, q5, %f20[1]    \n"

  8044.                         "vadd.f32   q8, q8, q4          \n"

  8045.                         "vadd.f32   q9, q9, q5          \n"

  8046.                         "vmla.f32   q10, q6, %e20[1]    \n"

  8047.                         "vmla.f32   q11, q7, %e20[0]    \n"

  8048.                         "vadd.f32   q12, q12, q6        \n"

  8049.                         "vadd.f32   q15, q15, q7        \n"

  8050. 

  8051.                         "vst1.f32   {d16-d17}, [%0]     \n"

  8052.                         "add        %0, %0, #64         \n"

  8053. 

  8054.                         "vst1.f32   {d18-d19}, [%1]     \n"

  8055.                         "add        %1, %1, #64         \n"

  8056. 

  8057.                         "vst1.f32   {d20-d21}, [%0]     \n"

  8058.                         "add        %0, %0, #64         \n"

  8059. 

  8060.                         "vst1.f32   {d22-d23}, [%1]     \n"

  8061.                         "add        %1, %1, #64         \n"

  8062. 

  8063.                         "vst1.f32   {d24-d25}, [%0]     \n"

  8064. 

  8065.                         "vst1.f32   {d30-d31}, [%1]     \n"

  8066. 

  8067.                         : "=r"(t0),             // %0

  8068.                           "=r"(t1),             // %1

  8069.                           "=r"(output0_tm0_0),  // %2

  8070.                           "=r"(output0_tm0_4),  // %3

  8071.                           "=r"(output0_tm1_0),  // %4

  8072.                           "=r"(output0_tm1_4),  // %5

  8073.                           "=r"(output0_tm2_0),  // %6

  8074.                           "=r"(output0_tm2_4),  // %7

  8075.                           "=r"(output0_tm3_0),  // %8

  8076.                           "=r"(output0_tm3_4)   // %9

  8077.                         : "0"(t0),

  8078.                           "1"(t1),

  8079.                           "2"(output0_tm0_0),

  8080.                           "3"(output0_tm0_4),

  8081.                           "4"(output0_tm1_0),

  8082.                           "5"(output0_tm1_4),

  8083.                           "6"(output0_tm2_0),

  8084.                           "7"(output0_tm2_4),

  8085.                           "8"(output0_tm3_0),

  8086.                           "9"(output0_tm3_4),

  8087.                           "w"(_coeff),          // %20

  8088.                           "r"(step)             // %21

  8089.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  8090.                     );

  8091. 

  8092.                     t0 = tmp[0];

  8093.                     t1 = tmp[1];

  8094. 

  8095.                     float* output0 = out0.row(i * 6) + j * 6;

  8096.                     float* output1 = output0 + outw;

  8097. 

  8098.                     int stepw = outw*2 * 4;

  8099. 

  8100.                     asm volatile(

  8101. 

  8102.                         // loop0

  8103.                         "vld1.f32   {d16-d19}, [%2]     \n"

  8104.                         "vld1.f32   {d20-d23}, [%3]     \n"

  8105. 

  8106.                         "add        %2, %2, #64         \n"

  8107.                         "add        %3, %3, #64         \n"

  8108. 

  8109.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  8110.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  8111. 

  8112.                         "vadd.f32   d4, d20, d17        \n"

  8113.                         "vsub.f32   d5, d20, d17        \n"

  8114. 

  8115.                         "vadd.f32   d6, d21, d18        \n"

  8116.                         "vsub.f32   d7, d21, d18        \n"

  8117. 

  8118.                         "vadd.f32   d8, d22, d19        \n"

  8119.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  8120. 

  8121.                         "vmov       d20, d5             \n"

  8122.                         "vmov       d18, d4             \n"

  8123. 

  8124.                         "vadd.f32   d16, d16, d4        \n"

  8125.                         "vmla.f32   d20, d9, %f8[0]     \n"

  8126.                         "vmov       d17, d4             \n"

  8127.                         "vmov       d21, d5             \n"

  8128.                         "vmla.f32   d18, d6, %f8[0]     \n"

  8129.                         "vadd.f32   d22, d23, d5        \n"

  8130. 

  8131.                         "vmla.f32   d16, d8, %f8[1]     \n"

  8132.                         "vadd.f32   d20, d20, d7        \n"

  8133.                         "vmla.f32   d17, d6, %e8[0]     \n"

  8134.                         "vmla.f32   d21, d7, %e8[1]     \n"

  8135.                         "vadd.f32   d18, d18, d8        \n"

  8136.                         "vmla.f32   d22, d7, %f8[1]     \n"

  8137. 

  8138.                         "vadd.f32   d16, d16, d6        \n"

  8139.                         "vadd.f32   d20, d20, d7        \n"

  8140.                         "vmla.f32   d17, d8, %e8[1]     \n"

  8141.                         "vmla.f32   d21, d9, %e8[0]     \n"

  8142.                         "vadd.f32   d18, d18, d8        \n"

  8143.                         "vadd.f32   d22, d22, d9        \n"

  8144. 

  8145.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  8146.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  8147.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  8148.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  8149.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  8150.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  8151. 

  8152.                         "vtrn.f32   q8, q10             \n"

  8153.                         "vtrn.f32   d18, d22            \n"

  8154. 

  8155.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  8156.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  8157. 

  8158.                         // loop1

  8159.                         "vld1.f32   {d16-d19}, [%2]     \n"

  8160.                         "vld1.f32   {d20-d23}, [%3]     \n"

  8161. 

  8162.                         "add        %2, %2, #64         \n"

  8163.                         "add        %3, %3, #64         \n"

  8164. 

  8165.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  8166.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  8167. 

  8168.                         "vadd.f32   d4, d20, d17        \n"

  8169.                         "vsub.f32   d5, d20, d17        \n"

  8170. 

  8171.                         "vadd.f32   d6, d21, d18        \n"

  8172.                         "vsub.f32   d7, d21, d18        \n"

  8173. 

  8174.                         "vadd.f32   d8, d22, d19        \n"

  8175.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  8176. 

  8177.                         "vmov       d20, d5             \n"

  8178.                         "vmov       d18, d4             \n"

  8179. 

  8180.                         "vadd.f32   d16, d16, d4        \n"

  8181.                         "vmla.f32   d20, d9, %f8[0]     \n"

  8182.                         "vmov       d17, d4             \n"

  8183.                         "vmov       d21, d5             \n"

  8184.                         "vmla.f32   d18, d6, %f8[0]     \n"

  8185.                         "vadd.f32   d22, d23, d5        \n"

  8186. 

  8187.                         "vmla.f32   d16, d8, %f8[1]     \n"

  8188.                         "vadd.f32   d20, d20, d7        \n"

  8189.                         "vmla.f32   d17, d6, %e8[0]     \n"

  8190.                         "vmla.f32   d21, d7, %e8[1]     \n"

  8191.                         "vadd.f32   d18, d18, d8        \n"

  8192.                         "vmla.f32   d22, d7, %f8[1]     \n"

  8193. 

  8194.                         "vadd.f32   d16, d16, d6        \n"

  8195.                         "vadd.f32   d20, d20, d7        \n"

  8196.                         "vmla.f32   d17, d8, %e8[1]     \n"

  8197.                         "vmla.f32   d21, d9, %e8[0]     \n"

  8198.                         "vadd.f32   d18, d18, d8        \n"

  8199.                         "vadd.f32   d22, d22, d9        \n"

  8200. 

  8201.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  8202.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  8203.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  8204.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  8205.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  8206.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  8207. 

  8208.                         "vtrn.f32   q8, q10             \n"

  8209.                         "vtrn.f32   d18, d22            \n"

  8210. 

  8211.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  8212.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  8213. 

  8214.                         // loop2

  8215.                         "vld1.f32   {d16-d19}, [%2]     \n"

  8216.                         "vld1.f32   {d20-d23}, [%3]     \n"

  8217. 

  8218.                         "add        %2, %2, #64         \n"

  8219.                         "add        %3, %3, #64         \n"

  8220. 

  8221.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  8222.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  8223. 

  8224.                         "vadd.f32   d4, d20, d17        \n"

  8225.                         "vsub.f32   d5, d20, d17        \n"

  8226. 

  8227.                         "vadd.f32   d6, d21, d18        \n"

  8228.                         "vsub.f32   d7, d21, d18        \n"

  8229. 

  8230.                         "vadd.f32   d8, d22, d19        \n"

  8231.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  8232. 

  8233.                         "vmov       d20, d5             \n"

  8234.                         "vmov       d18, d4             \n"

  8235. 

  8236.                         "vadd.f32   d16, d16, d4        \n"

  8237.                         "vmla.f32   d20, d9, %f8[0]     \n"

  8238.                         "vmov       d17, d4             \n"

  8239.                         "vmov       d21, d5             \n"

  8240.                         "vmla.f32   d18, d6, %f8[0]     \n"

  8241.                         "vadd.f32   d22, d23, d5        \n"

  8242. 

  8243.                         "vmla.f32   d16, d8, %f8[1]     \n"

  8244.                         "vadd.f32   d20, d20, d7        \n"

  8245.                         "vmla.f32   d17, d6, %e8[0]     \n"

  8246.                         "vmla.f32   d21, d7, %e8[1]     \n"

  8247.                         "vadd.f32   d18, d18, d8        \n"

  8248.                         "vmla.f32   d22, d7, %f8[1]     \n"

  8249. 

  8250.                         "vadd.f32   d16, d16, d6        \n"

  8251.                         "vadd.f32   d20, d20, d7        \n"

  8252.                         "vmla.f32   d17, d8, %e8[1]     \n"

  8253.                         "vmla.f32   d21, d9, %e8[0]     \n"

  8254.                         "vadd.f32   d18, d18, d8        \n"

  8255.                         "vadd.f32   d22, d22, d9        \n"

  8256. 

  8257.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  8258.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  8259.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  8260.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  8261.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  8262.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  8263. 

  8264.                         "vtrn.f32   q8, q10             \n"

  8265.                         "vtrn.f32   d18, d22            \n"

  8266. 

  8267.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  8268.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  8269. 

  8270.                         : "=r"(output0),    // %0

  8271.                           "=r"(output1),    // %1

  8272.                           "=r"(t0),         // %2

  8273.                           "=r"(t1)          // %3

  8274.                         : "0"(output0),

  8275.                           "1"(output1),

  8276.                           "2"(t0),

  8277.                           "3"(t1),

  8278.                           "w"(_coeff),      // %8

  8279.                           "w"(_bias0),      // %9

  8280.                           "r"(stepw)        // %10

  8281.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  8282.                     );

  8283. #endif // __aarch64__

  8284. #else

  8285.                     const float* output0_tm_0 = out0_tm.row(i * w_tm/8 + j);

  8286.                     const float* output0_tm_4 = out0_tm.row(i * w_tm/8 + j + tiles);

  8287. 

  8288.                     for (int m=0; m<8; m++)

  8289.                     {

  8290.                         float tmp024a = output0_tm_0[1] + output0_tm_0[2];

  8291.                         float tmp135a = output0_tm_0[1] - output0_tm_0[2];

  8292. 

  8293.                         float tmp024b = output0_tm_0[3] + output0_tm_4[0];

  8294.                         float tmp135b = output0_tm_0[3] - output0_tm_4[0];

  8295. 

  8296.                         float tmp024c = output0_tm_4[1] + output0_tm_4[2];

  8297.                         float tmp135c = output0_tm_4[1] - output0_tm_4[2];

  8298. 

  8299.                         tmp[0][m] = output0_tm_0[0] + tmp024a + tmp024b + tmp024c * 32;

  8300.                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  8301.                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  8302. 

  8303.                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  8304.                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  8305.                         tmp[5][m] = output0_tm_4[3] + tmp135a + tmp135b * 32 + tmp135c;

  8306. 

  8307.                         output0_tm_0 += out0_tm.w * tiles * 2;

  8308.                         output0_tm_4 += out0_tm.w * tiles * 2;

  8309.                     }

  8310. 

  8311.                     float* output0 = out0.row(i * 6) + j * 6;

  8312. 

  8313.                     for (int m=0; m<6; m++)

  8314.                     {

  8315.                         const float* tmp0 = tmp[m];

  8316. 

  8317.                         float tmp024a = tmp0[1] + tmp0[2];

  8318.                         float tmp135a = tmp0[1] - tmp0[2];

  8319. 

  8320.                         float tmp024b = tmp0[3] + tmp0[4];

  8321.                         float tmp135b = tmp0[3] - tmp0[4];

  8322. 

  8323.                         float tmp024c = tmp0[5] + tmp0[6];

  8324.                         float tmp135c = tmp0[5] - tmp0[6];

  8325. 

  8326.                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  8327.                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  8328.                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  8329. 

  8330.                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  8331.                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  8332.                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  8333. 

  8334.                         output0 += outw;

  8335.                     }

  8336. #endif // __ARM_NEON

  8337.                 }

  8338.             }

  8339.         }

  8340.     }

  8341.     // END transform output

  8342. 

  8343.     // cut result pad

  8344.     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt.blob_allocator, opt.num_threads);

  8345. }

  8346. 

  8347. static void conv3x3s1_winograd64_neon5(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias, const Option& opt)

  8348. {

  8349.     int w = bottom_blob.w;

  8350.     int h = bottom_blob.h;

  8351.     int inch = bottom_blob.c;

  8352. 

  8353.     int outw = top_blob.w;

  8354.     int outh = top_blob.h;

  8355.     int outch = top_blob.c;

  8356. 

  8357.     // pad to 6n+2

  8358.     Mat bottom_blob_bordered = bottom_blob;

  8359. 

  8360.     outw = (outw + 5) / 6 * 6;

  8361.     outh = (outh + 5) / 6 * 6;

  8362. 

  8363.     w = outw + 2;

  8364.     h = outh + 2;

  8365.     copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt.workspace_allocator, opt.num_threads);

  8366. 

  8367.     const float* bias = _bias;

  8368. 

  8369.     // BEGIN transform input

  8370.     Mat bottom_blob_tm;

  8371.     {

  8372.         int w_tm = outw / 6 * 8;

  8373.         int h_tm = outh / 6 * 8;

  8374.         const int tiles = w_tm/8 * h_tm/8;

  8375.         bottom_blob_tm.create(1, 64 * tiles, inch, 4u, opt.workspace_allocator);

  8376. //         bottom_blob_tm.create(inch, tiles, 64);

  8377. 

  8378. //         const float itm[8][8] = {

  8379. //             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},

  8380. //

  8381. //             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},

  8382. //             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},

  8383. //

  8384. //             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},

  8385. //             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},

  8386. //

  8387. //             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},

  8388. //             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},

  8389. //

  8390. //             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}

  8391. //         };

  8392. 

  8393.         // 0 = r00 - r06 + (r04 - r02) * 5.25

  8394.         // 7 = r07 - r01 + (r03 - r05) * 5.25

  8395. 

  8396.         // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)

  8397.         // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)

  8398. 

  8399.         // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  8400.         // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)

  8401. 

  8402.         // reuse r04 * 1.25

  8403.         // reuse r03 * 2.5

  8404.         // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  8405.         // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)

  8406. 

  8407. #if __ARM_NEON

  8408.         const float coeff[8] = {

  8409.             0.25f, 0.5f, -1.25f,   2.f,

  8410.             -2.5f,  4.f,  4.25f, 5.25f

  8411.         };

  8412.         float32x4_t _coeff0 = vld1q_f32(coeff);

  8413.         float32x4_t _coeff1 = vld1q_f32(coeff+4);

  8414. #endif // __ARM_NEON

  8415. 

  8416.         #pragma omp parallel for num_threads(opt.num_threads)

  8417.         for (int q = 0; q<inch; q++)

  8418.         {

  8419.             const Mat img0 = bottom_blob_bordered.channel(q);

  8420.             Mat img0_tm = bottom_blob_tm.channel(q);

  8421. 

  8422.             float tmp[8][8];

  8423. 

  8424.             // tile

  8425.             for (int i=0; i<h_tm/8; i++)

  8426.             {

  8427.                 for (int j=0; j<w_tm/8; j++)

  8428.                 {

  8429. #if __ARM_NEON

  8430.                     const float* r0 = img0.row(i * 6) + j * 6;

  8431.                     const float* r1 = r0 + w;

  8432.                     const float* r2 = r0 + w*2;

  8433.                     const float* r3 = r0 + w*3;

  8434. 

  8435. #if __aarch64__

  8436.                     for (int m=0; m+3<8; m+=4)

  8437.                     {

  8438.                         float32x4_t _r0_0123 = vld1q_f32(r0);

  8439.                         float32x4_t _r0_4567 = vld1q_f32(r0+4);

  8440.                         float32x4_t _r1_0123 = vld1q_f32(r1);

  8441.                         float32x4_t _r1_4567 = vld1q_f32(r1+4);

  8442.                         float32x4_t _r2_0123 = vld1q_f32(r2);

  8443.                         float32x4_t _r2_4567 = vld1q_f32(r2+4);

  8444.                         float32x4_t _r3_0123 = vld1q_f32(r3);

  8445.                         float32x4_t _r3_4567 = vld1q_f32(r3+4);

  8446. 

  8447.                         float32x4x2_t _r01_00221133 = vtrnq_f32(_r0_0123, _r1_0123);

  8448.                         float32x4x2_t _r01_44665577 = vtrnq_f32(_r0_4567, _r1_4567);

  8449.                         float32x4x2_t _r23_00221133 = vtrnq_f32(_r2_0123, _r3_0123);

  8450.                         float32x4x2_t _r23_44665577 = vtrnq_f32(_r2_4567, _r3_4567);

  8451. 

  8452.                         // no vswp intrinsic  :(

  8453.                         float32x4_t _r_00 = vcombine_f32(vget_low_f32(_r01_00221133.val[0]), vget_low_f32(_r23_00221133.val[0]));

  8454.                         float32x4_t _r_11 = vcombine_f32(vget_low_f32(_r01_00221133.val[1]), vget_low_f32(_r23_00221133.val[1]));

  8455.                         float32x4_t _r_22 = vcombine_f32(vget_high_f32(_r01_00221133.val[0]), vget_high_f32(_r23_00221133.val[0]));

  8456.                         float32x4_t _r_33 = vcombine_f32(vget_high_f32(_r01_00221133.val[1]), vget_high_f32(_r23_00221133.val[1]));

  8457.                         float32x4_t _r_44 = vcombine_f32(vget_low_f32(_r01_44665577.val[0]), vget_low_f32(_r23_44665577.val[0]));

  8458.                         float32x4_t _r_55 = vcombine_f32(vget_low_f32(_r01_44665577.val[1]), vget_low_f32(_r23_44665577.val[1]));

  8459.                         float32x4_t _r_66 = vcombine_f32(vget_high_f32(_r01_44665577.val[0]), vget_high_f32(_r23_44665577.val[0]));

  8460.                         float32x4_t _r_77 = vcombine_f32(vget_high_f32(_r01_44665577.val[1]), vget_high_f32(_r23_44665577.val[1]));

  8461. 

  8462.                         float32x4_t _r_0_m_6 = vsubq_f32(_r_00, _r_66);

  8463.                         float32x4_t _r_7_m_1 = vsubq_f32(_r_77, _r_11);

  8464. 

  8465.                         float32x4_t _r_4_m_2 = vsubq_f32(_r_44, _r_22);

  8466.                         float32x4_t _r_3_m_5 = vsubq_f32(_r_33, _r_55);

  8467. 

  8468.                         float32x4_t _tmp0 = vmlaq_lane_f32(_r_0_m_6, _r_4_m_2, vget_high_f32(_coeff1), 1);

  8469.                         float32x4_t _tmp7 = vmlaq_lane_f32(_r_7_m_1, _r_3_m_5, vget_high_f32(_coeff1), 1);

  8470. 

  8471.                         vst1q_f32(&tmp[0][m], _tmp0);

  8472.                         vst1q_f32(&tmp[7][m], _tmp7);

  8473. 

  8474.                         float32x4_t _r_2_a_6 = vaddq_f32(_r_22, _r_66);

  8475.                         float32x4_t _r_1_a_5 = vaddq_f32(_r_11, _r_55);

  8476. 

  8477.                         float32x4_t _tmp12a = vmlsq_lane_f32(_r_2_a_6, _r_44, vget_high_f32(_coeff1), 0);

  8478.                         float32x4_t _tmp12b = vmlsq_lane_f32(_r_1_a_5, _r_33, vget_high_f32(_coeff1), 0);

  8479. 

  8480.                         float32x4_t _tmp1 = vaddq_f32(_tmp12a, _tmp12b);

  8481.                         float32x4_t _tmp2 = vsubq_f32(_tmp12a, _tmp12b);

  8482. 

  8483.                         vst1q_f32(&tmp[1][m], _tmp1);

  8484.                         vst1q_f32(&tmp[2][m], _tmp2);

  8485. 

  8486.                         float32x4_t _r_4_x_c = vmulq_lane_f32(_r_44, vget_high_f32(_coeff0), 0);

  8487.                         float32x4_t _r_3_x_c = vmulq_lane_f32(_r_33, vget_low_f32(_coeff1), 0);

  8488. 

  8489.                         float32x4_t _tmp34a = vaddq_f32(_r_66, _r_4_x_c);

  8490.                         _tmp34a = vmlaq_lane_f32(_tmp34a, _r_22, vget_low_f32(_coeff0), 0);

  8491. 

  8492.                         float32x4_t _tmp34b = vmlaq_lane_f32(_r_3_x_c, _r_11, vget_low_f32(_coeff0), 1);

  8493.                         _tmp34b = vmlaq_lane_f32(_tmp34b, _r_55, vget_high_f32(_coeff0), 1);

  8494. 

  8495.                         float32x4_t _tmp3 = vaddq_f32(_tmp34a, _tmp34b);

  8496.                         float32x4_t _tmp4 = vsubq_f32(_tmp34a, _tmp34b);

  8497. 

  8498.                         vst1q_f32(&tmp[3][m], _tmp3);

  8499.                         vst1q_f32(&tmp[4][m], _tmp4);

  8500. 

  8501.                         // reuse r04 * 1.25

  8502.                         // reuse r03 * 2.5

  8503.                         float32x4_t _r_2_a_4c = vaddq_f32(_r_22, _r_4_x_c);

  8504.                         float32x4_t _tmp56a = vmlaq_lane_f32(_r_66, _r_2_a_4c, vget_low_f32(_coeff1), 1);

  8505.                         float32x4_t _tmp56b = vmlaq_lane_f32(_r_3_x_c, _r_11, vget_high_f32(_coeff0), 1);

  8506.                         _tmp56b = vmlaq_lane_f32(_tmp56b, _r_55, vget_low_f32(_coeff0), 1);

  8507. 

  8508.                         float32x4_t _tmp5 = vaddq_f32(_tmp56a, _tmp56b);

  8509.                         float32x4_t _tmp6 = vsubq_f32(_tmp56a, _tmp56b);

  8510. 

  8511.                         vst1q_f32(&tmp[5][m], _tmp5);

  8512.                         vst1q_f32(&tmp[6][m], _tmp6);

  8513. 

  8514.                         r0 += w*4;

  8515.                         r1 += w*4;

  8516.                         r2 += w*4;

  8517.                         r3 += w*4;

  8518.                     }

  8519. 

  8520.                     const float* t0 = tmp[0];

  8521.                     const float* t1 = tmp[1];

  8522.                     const float* t2 = tmp[2];

  8523.                     const float* t3 = tmp[3];

  8524. 

  8525.                     float* r0_tm0 = img0_tm.row(i * w_tm/8 + j);

  8526.                     float* r0_tm1 = img0_tm.row(i * w_tm/8 + j + tiles*8);

  8527.                     float* r0_tm2 = img0_tm.row(i * w_tm/8 + j + tiles*16);

  8528.                     float* r0_tm3 = img0_tm.row(i * w_tm/8 + j + tiles*24);

  8529. 

  8530.                     for (int m=0; m+3<8; m+=4)

  8531.                     {

  8532.                         float32x4_t _t0_0123 = vld1q_f32(t0);

  8533.                         float32x4_t _t0_4567 = vld1q_f32(t0+4);

  8534.                         float32x4_t _t1_0123 = vld1q_f32(t1);

  8535.                         float32x4_t _t1_4567 = vld1q_f32(t1+4);

  8536.                         float32x4_t _t2_0123 = vld1q_f32(t2);

  8537.                         float32x4_t _t2_4567 = vld1q_f32(t2+4);

  8538.                         float32x4_t _t3_0123 = vld1q_f32(t3);

  8539.                         float32x4_t _t3_4567 = vld1q_f32(t3+4);

  8540. 

  8541.                         float32x4x2_t _t01_00221133 = vtrnq_f32(_t0_0123, _t1_0123);

  8542.                         float32x4x2_t _t01_44665577 = vtrnq_f32(_t0_4567, _t1_4567);

  8543.                         float32x4x2_t _t23_00221133 = vtrnq_f32(_t2_0123, _t3_0123);

  8544.                         float32x4x2_t _t23_44665577 = vtrnq_f32(_t2_4567, _t3_4567);

  8545. 

  8546.                         // no vswp intrinsic  :(

  8547.                         float32x4_t _t_00 = vcombine_f32(vget_low_f32(_t01_00221133.val[0]), vget_low_f32(_t23_00221133.val[0]));

  8548.                         float32x4_t _t_11 = vcombine_f32(vget_low_f32(_t01_00221133.val[1]), vget_low_f32(_t23_00221133.val[1]));

  8549.                         float32x4_t _t_22 = vcombine_f32(vget_high_f32(_t01_00221133.val[0]), vget_high_f32(_t23_00221133.val[0]));

  8550.                         float32x4_t _t_33 = vcombine_f32(vget_high_f32(_t01_00221133.val[1]), vget_high_f32(_t23_00221133.val[1]));

  8551.                         float32x4_t _t_44 = vcombine_f32(vget_low_f32(_t01_44665577.val[0]), vget_low_f32(_t23_44665577.val[0]));

  8552.                         float32x4_t _t_55 = vcombine_f32(vget_low_f32(_t01_44665577.val[1]), vget_low_f32(_t23_44665577.val[1]));

  8553.                         float32x4_t _t_66 = vcombine_f32(vget_high_f32(_t01_44665577.val[0]), vget_high_f32(_t23_44665577.val[0]));

  8554.                         float32x4_t _t_77 = vcombine_f32(vget_high_f32(_t01_44665577.val[1]), vget_high_f32(_t23_44665577.val[1]));

  8555. 

  8556.                         float32x4_t _t_0_m_6 = vsubq_f32(_t_00, _t_66);

  8557.                         float32x4_t _t_7_m_1 = vsubq_f32(_t_77, _t_11);

  8558. 

  8559.                         float32x4_t _t_4_m_2 = vsubq_f32(_t_44, _t_22);

  8560.                         float32x4_t _t_3_m_5 = vsubq_f32(_t_33, _t_55);

  8561. 

  8562.                         float32x4_t _r0_tm_0_0 = vmlaq_lane_f32(_t_0_m_6, _t_4_m_2, vget_high_f32(_coeff1), 1);

  8563.                         float32x4_t _r0_tm_4_3 = vmlaq_lane_f32(_t_7_m_1, _t_3_m_5, vget_high_f32(_coeff1), 1);

  8564. 

  8565.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_0_0, 0);

  8566.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_0_0, 1);

  8567.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_0_0, 2);

  8568.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_0_0, 3);

  8569. 

  8570.                         r0_tm0 += img0_tm.w*tiles;

  8571.                         r0_tm1 += img0_tm.w*tiles;

  8572.                         r0_tm2 += img0_tm.w*tiles;

  8573.                         r0_tm3 += img0_tm.w*tiles;

  8574. 

  8575.                         float32x4_t _t_2_m_6 = vaddq_f32(_t_22, _t_66);

  8576.                         float32x4_t _t_1_m_5 = vaddq_f32(_t_11, _t_55);

  8577. 

  8578.                         float32x4_t _tmp12a = vmlsq_lane_f32(_t_2_m_6, _t_44, vget_high_f32(_coeff1), 0);

  8579.                         float32x4_t _tmp12b = vmlsq_lane_f32(_t_1_m_5, _t_33, vget_high_f32(_coeff1), 0);

  8580. 

  8581.                         float32x4_t _r0_tm_0_1 = vaddq_f32(_tmp12a, _tmp12b);

  8582.                         float32x4_t _r0_tm_0_2 = vsubq_f32(_tmp12a, _tmp12b);

  8583. 

  8584.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_0_1, 0);

  8585.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_0_1, 1);

  8586.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_0_1, 2);

  8587.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_0_1, 3);

  8588. 

  8589.                         r0_tm0 += img0_tm.w*tiles;

  8590.                         r0_tm1 += img0_tm.w*tiles;

  8591.                         r0_tm2 += img0_tm.w*tiles;

  8592.                         r0_tm3 += img0_tm.w*tiles;

  8593. 

  8594.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_0_2, 0);

  8595.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_0_2, 1);

  8596.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_0_2, 2);

  8597.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_0_2, 3);

  8598. 

  8599.                         r0_tm0 += img0_tm.w*tiles;

  8600.                         r0_tm1 += img0_tm.w*tiles;

  8601.                         r0_tm2 += img0_tm.w*tiles;

  8602.                         r0_tm3 += img0_tm.w*tiles;

  8603. 

  8604.                         float32x4_t _t_4_x_c = vmulq_lane_f32(_t_44, vget_high_f32(_coeff0), 0);

  8605.                         float32x4_t _t_3_x_c = vmulq_lane_f32(_t_33, vget_low_f32(_coeff1), 0);

  8606. 

  8607.                         float32x4_t _tmp34a = vaddq_f32(_t_66, _t_4_x_c);

  8608.                         _tmp34a = vmlaq_lane_f32(_tmp34a, _t_22, vget_low_f32(_coeff0), 0);

  8609. 

  8610.                         float32x4_t _tmp34b = vmlaq_lane_f32(_t_3_x_c, _t_11, vget_low_f32(_coeff0), 1);

  8611.                         _tmp34b = vmlaq_lane_f32(_tmp34b, _t_55, vget_high_f32(_coeff0), 1);

  8612. 

  8613.                         float32x4_t _r0_tm_0_3 = vaddq_f32(_tmp34a, _tmp34b);

  8614.                         float32x4_t _r0_tm_4_0 = vsubq_f32(_tmp34a, _tmp34b);

  8615. 

  8616.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_0_3, 0);

  8617.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_0_3, 1);

  8618.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_0_3, 2);

  8619.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_0_3, 3);

  8620. 

  8621.                         r0_tm0 += img0_tm.w*tiles;

  8622.                         r0_tm1 += img0_tm.w*tiles;

  8623.                         r0_tm2 += img0_tm.w*tiles;

  8624.                         r0_tm3 += img0_tm.w*tiles;

  8625. 

  8626.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_4_0, 0);

  8627.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_4_0, 1);

  8628.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_4_0, 2);

  8629.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_4_0, 3);

  8630. 

  8631.                         r0_tm0 += img0_tm.w*tiles;

  8632.                         r0_tm1 += img0_tm.w*tiles;

  8633.                         r0_tm2 += img0_tm.w*tiles;

  8634.                         r0_tm3 += img0_tm.w*tiles;

  8635. 

  8636.                         float32x4_t _t_2_a_4c = vaddq_f32(_t_22, _t_4_x_c);

  8637.                         float32x4_t _tmp56a = vmlaq_lane_f32(_t_66, _t_2_a_4c, vget_low_f32(_coeff1), 1);

  8638.                         float32x4_t _tmp56b = vmlaq_lane_f32(_t_3_x_c, _t_11, vget_high_f32(_coeff0), 1);

  8639.                         _tmp56b = vmlaq_lane_f32(_tmp56b, _t_55, vget_low_f32(_coeff0), 1);

  8640. 

  8641.                         float32x4_t _r0_tm_4_1 = vaddq_f32(_tmp56a, _tmp56b);

  8642.                         float32x4_t _r0_tm_4_2 = vsubq_f32(_tmp56a, _tmp56b);

  8643. 

  8644.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_4_1, 0);

  8645.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_4_1, 1);

  8646.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_4_1, 2);

  8647.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_4_1, 3);

  8648. 

  8649.                         r0_tm0 += img0_tm.w*tiles;

  8650.                         r0_tm1 += img0_tm.w*tiles;

  8651.                         r0_tm2 += img0_tm.w*tiles;

  8652.                         r0_tm3 += img0_tm.w*tiles;

  8653. 

  8654.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_4_2, 0);

  8655.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_4_2, 1);

  8656.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_4_2, 2);

  8657.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_4_2, 3);

  8658. 

  8659.                         r0_tm0 += img0_tm.w*tiles;

  8660.                         r0_tm1 += img0_tm.w*tiles;

  8661.                         r0_tm2 += img0_tm.w*tiles;

  8662.                         r0_tm3 += img0_tm.w*tiles;

  8663. 

  8664.                         r0_tm0[0] = vgetq_lane_f32(_r0_tm_4_3, 0);

  8665.                         r0_tm1[0] = vgetq_lane_f32(_r0_tm_4_3, 1);

  8666.                         r0_tm2[0] = vgetq_lane_f32(_r0_tm_4_3, 2);

  8667.                         r0_tm3[0] = vgetq_lane_f32(_r0_tm_4_3, 3);

  8668. 

  8669.                         t0 += 8*4;

  8670.                         t1 += 8*4;

  8671.                         t2 += 8*4;

  8672.                         t3 += 8*4;

  8673. 

  8674.                         r0_tm0 += img0_tm.w*tiles*25;

  8675.                         r0_tm1 += img0_tm.w*tiles*25;

  8676.                         r0_tm2 += img0_tm.w*tiles*25;

  8677.                         r0_tm3 += img0_tm.w*tiles*25;

  8678.                     }

  8679. #else // __aarch64__

  8680.                     float* t0 = tmp[0];

  8681.                     float* t1 = tmp[1];

  8682.                     float* t2 = tmp[2];

  8683.                     float* t3 = tmp[3];

  8684.                     float* t4 = tmp[4];

  8685.                     float* t5 = tmp[5];

  8686.                     float* t6 = tmp[6];

  8687.                     float* t7 = tmp[7];

  8688. 

  8689.                     int stepw = w*4*4;

  8690. 

  8691.                     asm volatile(

  8692. 

  8693.                         // loop0

  8694.                         "vld1.f32   {d16-d19}, [%8], %26    \n"

  8695.                         "vld1.f32   {d20-d23}, [%9], %26    \n"

  8696.                         "vld1.f32   {d24-d27}, [%10], %26   \n"

  8697. 

  8698.                         "vtrn.32    q8, q10             \n"

  8699. 

  8700.                         "vld1.f32   {d28-d31}, [%11], %26   \n"

  8701. 

  8702.                         "vtrn.32    q9, q11             \n"

  8703.                         "vtrn.32    q12, q14            \n"

  8704.                         "vtrn.32    q13, q15            \n"

  8705. 

  8706.                         "vswp       d17, d24            \n"

  8707.                         "vswp       d19, d26            \n"

  8708.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  8709.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  8710. 

  8711.                         "vsub.f32   q2, q8, q13         \n"

  8712.                         "vsub.f32   q3, q9, q12         \n"

  8713. 

  8714.                         "vadd.f32   q4, q12, q13        \n"

  8715.                         "vadd.f32   q5, q10, q11        \n"

  8716. 

  8717.                         "vmla.f32   q2, q3, %f25[1]     \n"

  8718. 

  8719.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  8720.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  8721. 

  8722.                         "vmls.f32   q4, q9, %f25[0]     \n"

  8723.                         "vmls.f32   q5, q14, %f25[0]    \n"

  8724. 

  8725.                         "vst1.f32   {d4-d5}, [%0]!      \n"// tmp[0][m]

  8726. 

  8727.                         "vmov       q3, q7              \n"// use q7

  8728. 

  8729.                         "vadd.f32   q2, q13, q6         \n"// use q6

  8730.                         "vmla.f32   q3, q10, %e24[1]    \n"

  8731. 

  8732.                         "vadd.f32   q8, q4, q5          \n"

  8733.                         "vsub.f32   q9, q4, q5          \n"

  8734. 

  8735.                         "vmov       q5, q7              \n"// use q7

  8736. 

  8737.                         "vadd.f32   q6, q12, q6         \n"// use q6

  8738.                         "vmla.f32   q5, q10, %f24[1]    \n"

  8739. 

  8740.                         "vmov       q4, q13             \n"

  8741. 

  8742.                         "vmla.f32   q2, q12, %e24[0]    \n"

  8743.                         "vmla.f32   q3, q11, %f24[1]    \n"

  8744. 

  8745.                         "vst1.f32   {d16-d17}, [%1]!    \n"// tmp[1][m]

  8746. 

  8747.                         "vmla.f32   q4, q6, %e25[1]     \n"

  8748.                         "vmla.f32   q5, q11, %e24[1]    \n"

  8749. 

  8750.                         "vst1.f32   {d18-d19}, [%2]!    \n"// tmp[2][m]

  8751. 

  8752.                         "vadd.f32   q8, q2, q3          \n"

  8753.                         "vsub.f32   q9, q2, q3          \n"

  8754. 

  8755.                         "vsub.f32   q6, q15, q10        \n"

  8756.                         "vsub.f32   q7, q14, q11        \n"

  8757. 

  8758.                         "vadd.f32   q2, q4, q5          \n"

  8759.                         "vsub.f32   q3, q4, q5          \n"

  8760. 

  8761.                         "vst1.f32   {d16-d17}, [%3]!    \n"// tmp[3][m]

  8762.                         "vst1.f32   {d18-d19}, [%4]!    \n"// tmp[4][m]

  8763. 

  8764.                         "vmla.f32   q6, q7, %f25[1]     \n"

  8765. 

  8766.                         "vst1.f32   {d4-d5}, [%5]!      \n"// tmp[5][m]

  8767.                         "vst1.f32   {d6-d7}, [%6]!      \n"// tmp[6][m]

  8768. 

  8769.                         "vst1.f32   {d12-d13}, [%7]!    \n"// tmp[7][m]

  8770. 

  8771.                         // loop1

  8772.                         "vld1.f32   {d16-d19}, [%8]     \n"

  8773.                         "vld1.f32   {d20-d23}, [%9]     \n"

  8774.                         "vld1.f32   {d24-d27}, [%10]    \n"

  8775. 

  8776.                         "vtrn.32    q8, q10             \n"

  8777. 

  8778.                         "vld1.f32   {d28-d31}, [%11]    \n"

  8779. 

  8780.                         "vtrn.32    q9, q11             \n"

  8781.                         "vtrn.32    q12, q14            \n"

  8782.                         "vtrn.32    q13, q15            \n"

  8783. 

  8784.                         "vswp       d17, d24            \n"

  8785.                         "vswp       d19, d26            \n"

  8786.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  8787.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  8788. 

  8789.                         "vsub.f32   q2, q8, q13         \n"

  8790.                         "vsub.f32   q3, q9, q12         \n"

  8791. 

  8792.                         "vadd.f32   q4, q12, q13        \n"

  8793.                         "vadd.f32   q5, q10, q11        \n"

  8794. 

  8795.                         "vmla.f32   q2, q3, %f25[1]     \n"

  8796. 

  8797.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  8798.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  8799. 

  8800.                         "vmls.f32   q4, q9, %f25[0]     \n"

  8801.                         "vmls.f32   q5, q14, %f25[0]    \n"

  8802. 

  8803.                         "vst1.f32   {d4-d5}, [%0]!      \n"// tmp[0][m]

  8804. 

  8805.                         "vmov       q3, q7              \n"// use q7

  8806. 

  8807.                         "vadd.f32   q2, q13, q6         \n"// use q6

  8808.                         "vmla.f32   q3, q10, %e24[1]    \n"

  8809. 

  8810.                         "vadd.f32   q8, q4, q5          \n"

  8811.                         "vsub.f32   q9, q4, q5          \n"

  8812. 

  8813.                         "vmov       q5, q7              \n"// use q7

  8814. 

  8815.                         "vadd.f32   q6, q12, q6         \n"// use q6

  8816.                         "vmla.f32   q5, q10, %f24[1]    \n"

  8817. 

  8818.                         "vmov       q4, q13             \n"

  8819. 

  8820.                         "vmla.f32   q2, q12, %e24[0]    \n"

  8821.                         "vmla.f32   q3, q11, %f24[1]    \n"

  8822. 

  8823.                         "vst1.f32   {d16-d17}, [%1]!    \n"// tmp[1][m]

  8824. 

  8825.                         "vmla.f32   q4, q6, %e25[1]     \n"

  8826.                         "vmla.f32   q5, q11, %e24[1]    \n"

  8827. 

  8828.                         "vst1.f32   {d18-d19}, [%2]!    \n"// tmp[2][m]

  8829. 

  8830.                         "vadd.f32   q8, q2, q3          \n"

  8831.                         "vsub.f32   q9, q2, q3          \n"

  8832. 

  8833.                         "vsub.f32   q6, q15, q10        \n"

  8834.                         "vsub.f32   q7, q14, q11        \n"

  8835. 

  8836.                         "vadd.f32   q2, q4, q5          \n"

  8837.                         "vsub.f32   q3, q4, q5          \n"

  8838. 

  8839.                         "vst1.f32   {d16-d17}, [%3]!    \n"// tmp[3][m]

  8840.                         "vst1.f32   {d18-d19}, [%4]!    \n"// tmp[4][m]

  8841. 

  8842.                         "vmla.f32   q6, q7, %f25[1]     \n"

  8843. 

  8844.                         "vst1.f32   {d4-d5}, [%5]!      \n"// tmp[5][m]

  8845.                         "vst1.f32   {d6-d7}, [%6]!      \n"// tmp[6][m]

  8846. 

  8847.                         "vst1.f32   {d12-d13}, [%7]!    \n"// tmp[7][m]

  8848. 

  8849.                         : "=r"(t0),     // %0

  8850.                           "=r"(t1),     // %1

  8851.                           "=r"(t2),     // %2

  8852.                           "=r"(t3),     // %3

  8853.                           "=r"(t4),     // %4

  8854.                           "=r"(t5),     // %5

  8855.                           "=r"(t6),     // %6

  8856.                           "=r"(t7),     // %7

  8857.                           "=r"(r0),     // %8

  8858.                           "=r"(r1),     // %9

  8859.                           "=r"(r2),     // %10

  8860.                           "=r"(r3)      // %11

  8861.                         : "0"(t0),

  8862.                           "1"(t1),

  8863.                           "2"(t2),

  8864.                           "3"(t3),

  8865.                           "4"(t4),

  8866.                           "5"(t5),

  8867.                           "6"(t6),

  8868.                           "7"(t7),

  8869.                           "8"(r0),

  8870.                           "9"(r1),

  8871.                           "10"(r2),

  8872.                           "11"(r3),

  8873.                           "w"(_coeff0), // %24

  8874.                           "w"(_coeff1), // %25

  8875.                           "r"(stepw)        // %26

  8876.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  8877.                     );

  8878. 

  8879.                     t0 = tmp[0];

  8880.                     t1 = tmp[1];

  8881.                     t2 = tmp[2];

  8882.                     t3 = tmp[3];

  8883. 

  8884.                     float* r0_tm0_0 = img0_tm.row(i * w_tm/8 + j);

  8885.                     float* r0_tm1_0 = img0_tm.row(i * w_tm/8 + j + tiles*8);

  8886.                     float* r0_tm2_0 = img0_tm.row(i * w_tm/8 + j + tiles*16);

  8887.                     float* r0_tm3_0 = img0_tm.row(i * w_tm/8 + j + tiles*24);

  8888.                     float* r0_tm0_4 = img0_tm.row(i * w_tm/8 + j + tiles*32);

  8889.                     float* r0_tm1_4 = img0_tm.row(i * w_tm/8 + j + tiles*40);

  8890.                     float* r0_tm2_4 = img0_tm.row(i * w_tm/8 + j + tiles*48);

  8891.                     float* r0_tm3_4 = img0_tm.row(i * w_tm/8 + j + tiles*56);

  8892. 

  8893.                     int step = img0_tm.w*tiles*4;

  8894. 

  8895.                     asm volatile(

  8896. 

  8897.                         // loop0

  8898.                         "vld1.f32   {d16-d19}, [%8]     \n"

  8899.                         "add        %8, %8, #128        \n"

  8900.                         "vld1.f32   {d20-d23}, [%9]     \n"

  8901.                         "add        %9, %9, #128        \n"

  8902.                         "vld1.f32   {d24-d27}, [%10]    \n"

  8903.                         "add        %10, %10, #128      \n"

  8904. 

  8905.                         "vtrn.32    q8, q10             \n"

  8906. 

  8907.                         "vld1.f32   {d28-d31}, [%11]    \n"

  8908.                         "add        %11, %11, #128      \n"

  8909. 

  8910.                         "vtrn.32    q9, q11             \n"

  8911.                         "vtrn.32    q12, q14            \n"

  8912.                         "vtrn.32    q13, q15            \n"

  8913. 

  8914.                         "vswp       d17, d24            \n"

  8915.                         "vswp       d19, d26            \n"

  8916.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  8917.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  8918. 

  8919.                         "vsub.f32   q2, q8, q13         \n"

  8920.                         "vsub.f32   q3, q9, q12         \n"

  8921. 

  8922.                         "vadd.f32   q4, q12, q13        \n"

  8923.                         "vadd.f32   q5, q10, q11        \n"

  8924. 

  8925.                         "vmla.f32   q2, q3, %f25[1]     \n"

  8926. 

  8927.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  8928.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  8929. 

  8930.                         "vmls.f32   q4, q9, %f25[0]     \n"

  8931.                         "vmls.f32   q5, q14, %f25[0]    \n"

  8932. 

  8933.                         "vst1.f32   {d4[0]}, [%0], %26  \n"

  8934.                         "vst1.f32   {d4[1]}, [%1], %26  \n"

  8935. 

  8936.                         "vmov       q3, q7              \n"// use q7

  8937. 

  8938.                         "vst1.f32   {d5[0]}, [%2], %26  \n"

  8939.                         "vst1.f32   {d5[1]}, [%3], %26  \n"

  8940. 

  8941.                         "vadd.f32   q2, q13, q6         \n"// use q6

  8942.                         "vmla.f32   q3, q10, %e24[1]    \n"

  8943. 

  8944.                         "vadd.f32   q8, q4, q5          \n"

  8945.                         "vsub.f32   q9, q4, q5          \n"

  8946. 

  8947.                         "vmov       q5, q7              \n"// use q7

  8948. 

  8949.                         "vadd.f32   q6, q12, q6         \n"// use q6

  8950.                         "vmla.f32   q5, q10, %f24[1]    \n"

  8951. 

  8952.                         "vmov       q4, q13             \n"

  8953. 

  8954.                         "vmla.f32   q2, q12, %e24[0]    \n"

  8955.                         "vmla.f32   q3, q11, %f24[1]    \n"

  8956. 

  8957.                         "vst1.f32   {d16[0]}, [%0], %26 \n"

  8958.                         "vst1.f32   {d16[1]}, [%1], %26 \n"

  8959. 

  8960.                         "vmla.f32   q4, q6, %e25[1]     \n"

  8961. 

  8962.                         "vst1.f32   {d17[0]}, [%2], %26 \n"

  8963.                         "vst1.f32   {d17[1]}, [%3], %26 \n"

  8964. 

  8965.                         "vmla.f32   q5, q11, %e24[1]    \n"

  8966. 

  8967.                         "vst1.f32   {d18[0]}, [%0], %26 \n"

  8968.                         "vst1.f32   {d18[1]}, [%1], %26 \n"

  8969. 

  8970.                         "vadd.f32   q8, q2, q3          \n"

  8971. 

  8972.                         "vst1.f32   {d19[0]}, [%2], %26 \n"

  8973.                         "vst1.f32   {d19[1]}, [%3], %26 \n"

  8974. 

  8975.                         "vsub.f32   q9, q2, q3          \n"

  8976. 

  8977.                         "vsub.f32   q6, q15, q10        \n"

  8978.                         "vsub.f32   q7, q14, q11        \n"

  8979. 

  8980.                         "vst1.f32   {d16[0]}, [%0], %26 \n"

  8981.                         "vst1.f32   {d16[1]}, [%1], %26 \n"

  8982.                         "vst1.f32   {d17[0]}, [%2], %26 \n"

  8983.                         "vst1.f32   {d17[1]}, [%3], %26 \n"

  8984. 

  8985.                         "vadd.f32   q2, q4, q5          \n"

  8986. 

  8987.                         "vst1.f32   {d18[0]}, [%0], %26 \n"

  8988.                         "vst1.f32   {d18[1]}, [%1], %26 \n"

  8989.                         "vst1.f32   {d19[0]}, [%2], %26 \n"

  8990.                         "vst1.f32   {d19[1]}, [%3], %26 \n"

  8991. 

  8992.                         "vsub.f32   q3, q4, q5          \n"

  8993. 

  8994.                         "vst1.f32   {d4[0]}, [%0], %26  \n"

  8995.                         "vst1.f32   {d4[1]}, [%1], %26  \n"

  8996.                         "vst1.f32   {d5[0]}, [%2], %26  \n"

  8997.                         "vst1.f32   {d5[1]}, [%3], %26  \n"

  8998. 

  8999.                         "vmla.f32   q6, q7, %f25[1]     \n"

  9000. 

  9001.                         "vst1.f32   {d6[0]}, [%0], %26  \n"

  9002.                         "vst1.f32   {d6[1]}, [%1], %26  \n"

  9003.                         "vst1.f32   {d7[0]}, [%2], %26  \n"

  9004.                         "vst1.f32   {d7[1]}, [%3], %26  \n"

  9005. 

  9006.                         "vst1.f32   {d12[0]}, [%0]      \n"

  9007.                         "vst1.f32   {d12[1]}, [%1]      \n"

  9008.                         "vst1.f32   {d13[0]}, [%2]      \n"

  9009.                         "vst1.f32   {d13[1]}, [%3]      \n"

  9010. 

  9011.                         // loop1

  9012.                         "vld1.f32   {d16-d19}, [%8]     \n"

  9013.                         "vld1.f32   {d20-d23}, [%9]     \n"

  9014.                         "vld1.f32   {d24-d27}, [%10]    \n"

  9015. 

  9016.                         "vtrn.32    q8, q10             \n"

  9017. 

  9018.                         "vld1.f32   {d28-d31}, [%11]    \n"

  9019. 

  9020.                         "vtrn.32    q9, q11             \n"

  9021.                         "vtrn.32    q12, q14            \n"

  9022.                         "vtrn.32    q13, q15            \n"

  9023. 

  9024.                         "vswp       d17, d24            \n"

  9025.                         "vswp       d19, d26            \n"

  9026.                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  9027.                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  9028. 

  9029.                         "vsub.f32   q2, q8, q13         \n"

  9030.                         "vsub.f32   q3, q9, q12         \n"

  9031. 

  9032.                         "vadd.f32   q4, q12, q13        \n"

  9033.                         "vadd.f32   q5, q10, q11        \n"

  9034. 

  9035.                         "vmla.f32   q2, q3, %f25[1]     \n"

  9036. 

  9037.                         "vmul.f32   q7, q14, %e25[0]    \n"// q7 = _r_3_x_c

  9038.                         "vmul.f32   q6, q9, %f24[0]     \n"// q6 = _r_4_x_c

  9039. 

  9040.                         "vmls.f32   q4, q9, %f25[0]     \n"

  9041.                         "vmls.f32   q5, q14, %f25[0]    \n"

  9042. 

  9043.                         "vst1.f32   {d4[0]}, [%4], %26  \n"

  9044.                         "vst1.f32   {d4[1]}, [%5], %26  \n"

  9045. 

  9046.                         "vmov       q3, q7              \n"// use q7

  9047. 

  9048.                         "vst1.f32   {d5[0]}, [%6], %26  \n"

  9049.                         "vst1.f32   {d5[1]}, [%7], %26  \n"

  9050. 

  9051.                         "vadd.f32   q2, q13, q6         \n"// use q6

  9052.                         "vmla.f32   q3, q10, %e24[1]    \n"

  9053. 

  9054.                         "vadd.f32   q8, q4, q5          \n"

  9055.                         "vsub.f32   q9, q4, q5          \n"

  9056. 

  9057.                         "vmov       q5, q7              \n"// use q7

  9058. 

  9059.                         "vadd.f32   q6, q12, q6         \n"// use q6

  9060.                         "vmla.f32   q5, q10, %f24[1]    \n"

  9061. 

  9062.                         "vmov       q4, q13             \n"

  9063. 

  9064.                         "vmla.f32   q2, q12, %e24[0]    \n"

  9065.                         "vmla.f32   q3, q11, %f24[1]    \n"

  9066. 

  9067.                         "vst1.f32   {d16[0]}, [%4], %26 \n"

  9068.                         "vst1.f32   {d16[1]}, [%5], %26 \n"

  9069. 

  9070.                         "vmla.f32   q4, q6, %e25[1]     \n"

  9071. 

  9072.                         "vst1.f32   {d17[0]}, [%6], %26 \n"

  9073.                         "vst1.f32   {d17[1]}, [%7], %26 \n"

  9074. 

  9075.                         "vmla.f32   q5, q11, %e24[1]    \n"

  9076. 

  9077.                         "vst1.f32   {d18[0]}, [%4], %26 \n"

  9078.                         "vst1.f32   {d18[1]}, [%5], %26 \n"

  9079. 

  9080.                         "vadd.f32   q8, q2, q3          \n"

  9081. 

  9082.                         "vst1.f32   {d19[0]}, [%6], %26 \n"

  9083.                         "vst1.f32   {d19[1]}, [%7], %26 \n"

  9084. 

  9085.                         "vsub.f32   q9, q2, q3          \n"

  9086. 

  9087.                         "vsub.f32   q6, q15, q10        \n"

  9088.                         "vsub.f32   q7, q14, q11        \n"

  9089. 

  9090.                         "vst1.f32   {d16[0]}, [%4], %26 \n"

  9091.                         "vst1.f32   {d16[1]}, [%5], %26 \n"

  9092.                         "vst1.f32   {d17[0]}, [%6], %26 \n"

  9093.                         "vst1.f32   {d17[1]}, [%7], %26 \n"

  9094. 

  9095.                         "vadd.f32   q2, q4, q5          \n"

  9096. 

  9097.                         "vst1.f32   {d18[0]}, [%4], %26 \n"

  9098.                         "vst1.f32   {d18[1]}, [%5], %26 \n"

  9099.                         "vst1.f32   {d19[0]}, [%6], %26 \n"

  9100.                         "vst1.f32   {d19[1]}, [%7], %26 \n"

  9101. 

  9102.                         "vsub.f32   q3, q4, q5          \n"

  9103. 

  9104.                         "vst1.f32   {d4[0]}, [%4], %26  \n"

  9105.                         "vst1.f32   {d4[1]}, [%5], %26  \n"

  9106.                         "vst1.f32   {d5[0]}, [%6], %26  \n"

  9107.                         "vst1.f32   {d5[1]}, [%7], %26  \n"

  9108. 

  9109.                         "vmla.f32   q6, q7, %f25[1]     \n"

  9110. 

  9111.                         "vst1.f32   {d6[0]}, [%4], %26  \n"

  9112.                         "vst1.f32   {d6[1]}, [%5], %26  \n"

  9113.                         "vst1.f32   {d7[0]}, [%6], %26  \n"

  9114.                         "vst1.f32   {d7[1]}, [%7], %26  \n"

  9115. 

  9116.                         "vst1.f32   {d12[0]}, [%4]      \n"

  9117.                         "vst1.f32   {d12[1]}, [%5]      \n"

  9118.                         "vst1.f32   {d13[0]}, [%6]      \n"

  9119.                         "vst1.f32   {d13[1]}, [%7]      \n"

  9120. 

  9121.                         : "=r"(r0_tm0_0),     // %0

  9122.                           "=r"(r0_tm1_0),     // %1

  9123.                           "=r"(r0_tm2_0),     // %2

  9124.                           "=r"(r0_tm3_0),     // %3

  9125.                           "=r"(r0_tm0_4),     // %4

  9126.                           "=r"(r0_tm1_4),     // %5

  9127.                           "=r"(r0_tm2_4),     // %6

  9128.                           "=r"(r0_tm3_4),     // %7

  9129.                           "=r"(t0),     // %8

  9130.                           "=r"(t1),     // %9

  9131.                           "=r"(t2),     // %10

  9132.                           "=r"(t3)      // %11

  9133.                         : "0"(r0_tm0_0),

  9134.                           "1"(r0_tm1_0),

  9135.                           "2"(r0_tm2_0),

  9136.                           "3"(r0_tm3_0),

  9137.                           "4"(r0_tm0_4),

  9138.                           "5"(r0_tm1_4),

  9139.                           "6"(r0_tm2_4),

  9140.                           "7"(r0_tm3_4),

  9141.                           "8"(t0),

  9142.                           "9"(t1),

  9143.                           "10"(t2),

  9144.                           "11"(t3),

  9145.                           "w"(_coeff0), // %24

  9146.                           "w"(_coeff1), // %25

  9147.                           "r"(step)        // %26

  9148.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  9149.                     );

  9150. #endif // __aarch64__

  9151. #else

  9152.                     const float* r0 = img0.row(i * 6) + j * 6;

  9153. 

  9154.                     for (int m=0; m<8; m++)

  9155.                     {

  9156.                         tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25f;

  9157.                         tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25f;

  9158. 

  9159.                         float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25f);

  9160.                         float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25f);

  9161. 

  9162.                         tmp[1][m] = tmp12a + tmp12b;

  9163.                         tmp[2][m] = tmp12a - tmp12b;

  9164. 

  9165.                         float tmp34a = (r0[6] + r0[2] * 0.25f - r0[4] * 1.25f);

  9166.                         float tmp34b = (r0[1] * 0.5f - r0[3] * 2.5f + r0[5] * 2.f);

  9167. 

  9168.                         tmp[3][m] = tmp34a + tmp34b;

  9169.                         tmp[4][m] = tmp34a - tmp34b;

  9170. 

  9171.                         float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25f) * 4.f);

  9172.                         float tmp56b = (r0[1] * 2.f - r0[3] * 2.5f + r0[5] * 0.5f);

  9173. 

  9174.                         tmp[5][m] = tmp56a + tmp56b;

  9175.                         tmp[6][m] = tmp56a - tmp56b;

  9176. 

  9177.                         r0 += w;

  9178.                     }

  9179. 

  9180.                     float* r0_tm_0 = img0_tm.row(i * w_tm/8 + j);

  9181.                     float* r0_tm_1 = img0_tm.row(i * w_tm/8 + j + tiles);

  9182.                     float* r0_tm_2 = img0_tm.row(i * w_tm/8 + j + tiles*2);

  9183.                     float* r0_tm_3 = img0_tm.row(i * w_tm/8 + j + tiles*3);

  9184.                     float* r0_tm_4 = img0_tm.row(i * w_tm/8 + j + tiles*4);

  9185.                     float* r0_tm_5 = img0_tm.row(i * w_tm/8 + j + tiles*5);

  9186.                     float* r0_tm_6 = img0_tm.row(i * w_tm/8 + j + tiles*6);

  9187.                     float* r0_tm_7 = img0_tm.row(i * w_tm/8 + j + tiles*7);

  9188. 

  9189.                     for (int m=0; m<8; m++)

  9190.                     {

  9191.                         const float* tmp0 = tmp[m];

  9192. 

  9193.                         r0_tm_0[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25f;

  9194.                         r0_tm_7[0] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25f;

  9195. 

  9196.                         float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25f);

  9197.                         float tmp12b = (tmp0[1] - tmp0[3] * 4.25f + tmp0[5]);

  9198. 

  9199.                         r0_tm_1[0] = tmp12a + tmp12b;

  9200.                         r0_tm_2[0] = tmp12a - tmp12b;

  9201. 

  9202.                         float tmp34a = (tmp0[6] + tmp0[2] * 0.25f - tmp0[4] * 1.25f);

  9203.                         float tmp34b = (tmp0[1] * 0.5f - tmp0[3] * 2.5f + tmp0[5] * 2.f);

  9204. 

  9205.                         r0_tm_3[0] = tmp34a + tmp34b;

  9206.                         r0_tm_4[0] = tmp34a - tmp34b;

  9207. 

  9208.                         float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25f) * 4.f);

  9209.                         float tmp56b = (tmp0[1] * 2.f - tmp0[3] * 2.5f + tmp0[5] * 0.5f);

  9210. 

  9211.                         r0_tm_5[0] = tmp56a + tmp56b;

  9212.                         r0_tm_6[0] = tmp56a - tmp56b;

  9213. 

  9214.                         r0_tm_0 += img0_tm.w * tiles * 8;

  9215.                         r0_tm_1 += img0_tm.w * tiles * 8;

  9216.                         r0_tm_2 += img0_tm.w * tiles * 8;

  9217.                         r0_tm_3 += img0_tm.w * tiles * 8;

  9218.                         r0_tm_4 += img0_tm.w * tiles * 8;

  9219.                         r0_tm_5 += img0_tm.w * tiles * 8;

  9220.                         r0_tm_6 += img0_tm.w * tiles * 8;

  9221.                         r0_tm_7 += img0_tm.w * tiles * 8;

  9222.                     }

  9223. #endif // __ARM_NEON

  9224.                 }

  9225.             }

  9226.         }

  9227.     }

  9228.     bottom_blob_bordered = Mat();

  9229.     // END transform input

  9230. 

  9231.     // BEGIN dot

  9232.     Mat top_blob_tm;

  9233.     {

  9234.         int w_tm = outw / 6 * 8;

  9235.         int h_tm = outh / 6 * 8;

  9236.         const int tiles = w_tm/8 * h_tm/8;

  9237. 

  9238.         // permute

  9239.         // bottom_blob_tm.create(1, 64 * tiles, inch);

  9240. //         Mat bottom_blob_tm2(inch, tiles, 64);

  9241.         Mat bottom_blob_tm2(8*inch, tiles/8 + (tiles%8)/4 + tiles%4, 64, 4u, opt.workspace_allocator);

  9242. 

  9243.         #pragma omp parallel for num_threads(opt.num_threads)

  9244.         for (int r=0; r<64; r++)

  9245.         {

  9246.             Mat tm2 = bottom_blob_tm2.channel(r);

  9247. 

  9248.             // tile

  9249.             int i=0;

  9250.             for (; i+7<tiles; i+=8)

  9251.             {

  9252.                 float* tm2p = tm2.row(i/8);

  9253. 

  9254.                 const float* r0 = bottom_blob_tm;

  9255. 

  9256.                 r0 += r*tiles + i;

  9257. 

  9258.                 for (int q=0; q<inch; q++)

  9259.                 {

  9260. #if __ARM_NEON

  9261.                     float32x4_t _r0 = vld1q_f32(r0);

  9262.                     float32x4_t _r0n = vld1q_f32(r0+4);

  9263.                     vst1q_f32(tm2p, _r0);

  9264.                     vst1q_f32(tm2p+4, _r0n);

  9265. #else

  9266.                     tm2p[0] = r0[0];

  9267.                     tm2p[1] = r0[1];

  9268.                     tm2p[2] = r0[2];

  9269.                     tm2p[3] = r0[3];

  9270.                     tm2p[4] = r0[4];

  9271.                     tm2p[5] = r0[5];

  9272.                     tm2p[6] = r0[6];

  9273.                     tm2p[7] = r0[7];

  9274. #endif // __ARM_NEON

  9275. 

  9276.                     r0 += bottom_blob_tm.cstep;

  9277.                     tm2p += 8;

  9278.                 }

  9279.             }

  9280.             for (; i+3<tiles; i+=4)

  9281.             {

  9282.                 float* tm2p = tm2.row(i/8+(i%8)/4);

  9283. 

  9284.                 const float* r0 = bottom_blob_tm;

  9285. 

  9286.                 r0 += r*tiles + i;

  9287. 

  9288.                 for (int q=0; q<inch; q++)

  9289.                 {

  9290. #if __ARM_NEON

  9291.                     float32x4_t _r0 = vld1q_f32(r0);

  9292.                     vst1q_f32(tm2p, _r0);

  9293. #else

  9294.                     tm2p[0] = r0[0];

  9295.                     tm2p[1] = r0[1];

  9296.                     tm2p[2] = r0[2];

  9297.                     tm2p[3] = r0[3];

  9298. #endif // __ARM_NEON

  9299. 

  9300.                     r0 += bottom_blob_tm.cstep;

  9301.                     tm2p += 4;

  9302.                 }

  9303.             }

  9304.             for (; i<tiles; i++)

  9305.             {

  9306.                 float* tm2p = tm2.row(i/8+(i%8)/4+i%4);

  9307. 

  9308.                 const float* r0 = bottom_blob_tm;

  9309. 

  9310.                 r0 += r*tiles + i;

  9311. 

  9312.                 for (int q=0; q<inch; q++)

  9313.                 {

  9314.                     tm2p[0] = r0[0];

  9315. 

  9316.                     r0 += bottom_blob_tm.cstep;

  9317.                     tm2p += 1;

  9318.                 }

  9319.             }

  9320.         }

  9321. 

  9322.         bottom_blob_tm = Mat();

  9323.         // permute end

  9324. 

  9325.         top_blob_tm.create(1, 64 * tiles, outch);

  9326. 

  9327.         int nn_outch = 0;

  9328.         int remain_outch_start = 0;

  9329. 

  9330. #if __ARM_NEON && __aarch64__

  9331.         nn_outch = outch >> 3;

  9332.         remain_outch_start = nn_outch << 3;

  9333. 

  9334.         #pragma omp parallel for num_threads(opt.num_threads)

  9335.         for (int pp=0; pp<nn_outch; pp++)

  9336.         {

  9337.             int p = pp * 8;

  9338. 

  9339.             const Mat kernel_tm0 = kernel_tm.channel(p/8);

  9340. 

  9341.             Mat out0_tm = top_blob_tm.channel(p);

  9342.             Mat out1_tm = top_blob_tm.channel(p+1);

  9343.             Mat out2_tm = top_blob_tm.channel(p+2);

  9344.             Mat out3_tm = top_blob_tm.channel(p+3);

  9345.             Mat out4_tm = top_blob_tm.channel(p+4);

  9346.             Mat out5_tm = top_blob_tm.channel(p+5);

  9347.             Mat out6_tm = top_blob_tm.channel(p+6);

  9348.             Mat out7_tm = top_blob_tm.channel(p+7);

  9349. 

  9350.             float* output0_tm = out0_tm;

  9351.             float* output1_tm = out1_tm;

  9352.             float* output2_tm = out2_tm;

  9353.             float* output3_tm = out3_tm;

  9354.             float* output4_tm = out4_tm;

  9355.             float* output5_tm = out5_tm;

  9356.             float* output6_tm = out6_tm;

  9357.             float* output7_tm = out7_tm;

  9358. 

  9359.             for (int r=0; r<64; r++)

  9360.             {

  9361.                 const Mat bb2 = bottom_blob_tm2.channel(r);

  9362. 

  9363.                 // tile

  9364.                 int i=0;

  9365.                 for (; i+7<tiles; i+=8)

  9366.                 {

  9367.                     const float* bb2p0 = bb2.row(i/8);

  9368. 

  9369.                     const float* ktm0 = kernel_tm0.row(r);

  9370. 

  9371.                     asm volatile(

  9372.                         "eor    v16.16b, v16.16b, v16.16b  \n"

  9373.                         "eor    v17.16b, v17.16b, v17.16b  \n"

  9374.                         "eor    v18.16b, v18.16b, v18.16b  \n"

  9375.                         "eor    v19.16b, v19.16b, v19.16b  \n"

  9376.                         "eor    v20.16b, v20.16b, v20.16b  \n"

  9377.                         "eor    v21.16b, v21.16b, v21.16b  \n"

  9378.                         "eor    v22.16b, v22.16b, v22.16b  \n"

  9379.                         "eor    v23.16b, v23.16b, v23.16b  \n"

  9380.                         "eor    v24.16b, v24.16b, v24.16b  \n"

  9381.                         "eor    v25.16b, v25.16b, v25.16b  \n"

  9382.                         "eor    v26.16b, v26.16b, v26.16b  \n"

  9383.                         "eor    v27.16b, v27.16b, v27.16b  \n"

  9384.                         "eor    v28.16b, v28.16b, v28.16b  \n"

  9385.                         "eor    v29.16b, v29.16b, v29.16b  \n"

  9386.                         "eor    v30.16b, v30.16b, v30.16b  \n"

  9387.                         "eor    v31.16b, v31.16b, v31.16b  \n"

  9388. 

  9389.                         // inch loop

  9390.                         "lsr    w4, %w20, #2            \n"// w4 = nn = inch >> 2

  9391.                         "cmp    w4, #0                  \n"

  9392.                         "beq    1f                      \n"

  9393. 

  9394.                         "0:                             \n"

  9395. 

  9396.                         "prfm   pldl1keep, [%8, #512]   \n"

  9397.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64   \n"

  9398. 

  9399.                         "prfm   pldl1keep, [%9, #512]   \n"

  9400.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64   \n"

  9401. 

  9402.                         "fmla   v16.4s, v8.4s, v0.s[0]  \n"

  9403.                         "fmla   v17.4s, v9.4s, v0.s[0]  \n"

  9404.                         "fmla   v18.4s, v8.4s, v0.s[1]  \n"

  9405.                         "fmla   v19.4s, v9.4s, v0.s[1]  \n"

  9406.                         "fmla   v20.4s, v8.4s, v0.s[2]  \n"

  9407.                         "fmla   v21.4s, v9.4s, v0.s[2]  \n"

  9408.                         "fmla   v22.4s, v8.4s, v0.s[3]  \n"

  9409.                         "fmla   v23.4s, v9.4s, v0.s[3]  \n"

  9410. 

  9411.                         "prfm   pldl1keep, [%9, #512]   \n"

  9412.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64   \n"

  9413. 

  9414.                         "fmla   v24.4s, v8.4s, v1.s[0]  \n"

  9415.                         "fmla   v25.4s, v9.4s, v1.s[0]  \n"

  9416.                         "fmla   v26.4s, v8.4s, v1.s[1]  \n"

  9417.                         "fmla   v27.4s, v9.4s, v1.s[1]  \n"

  9418.                         "fmla   v28.4s, v8.4s, v1.s[2]  \n"

  9419.                         "fmla   v29.4s, v9.4s, v1.s[2]  \n"

  9420.                         "fmla   v30.4s, v8.4s, v1.s[3]  \n"

  9421.                         "fmla   v31.4s, v9.4s, v1.s[3]  \n"

  9422. 

  9423.                         "fmla   v16.4s, v10.4s, v2.s[0] \n"

  9424.                         "fmla   v17.4s, v11.4s, v2.s[0] \n"

  9425.                         "fmla   v18.4s, v10.4s, v2.s[1] \n"

  9426.                         "fmla   v19.4s, v11.4s, v2.s[1] \n"

  9427.                         "fmla   v20.4s, v10.4s, v2.s[2] \n"

  9428.                         "fmla   v21.4s, v11.4s, v2.s[2] \n"

  9429.                         "fmla   v22.4s, v10.4s, v2.s[3] \n"

  9430.                         "fmla   v23.4s, v11.4s, v2.s[3] \n"

  9431. 

  9432.                         "prfm   pldl1keep, [%8, #512]   \n"

  9433.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%8], #64 \n"

  9434. 

  9435.                         "fmla   v24.4s, v10.4s, v3.s[0] \n"

  9436.                         "fmla   v25.4s, v11.4s, v3.s[0] \n"

  9437.                         "fmla   v26.4s, v10.4s, v3.s[1] \n"

  9438.                         "fmla   v27.4s, v11.4s, v3.s[1] \n"

  9439.                         "fmla   v28.4s, v10.4s, v3.s[2] \n"

  9440.                         "fmla   v29.4s, v11.4s, v3.s[2] \n"

  9441.                         "fmla   v30.4s, v10.4s, v3.s[3] \n"

  9442.                         "fmla   v31.4s, v11.4s, v3.s[3] \n"

  9443. 

  9444.                         "fmla   v16.4s, v12.4s, v4.s[0] \n"

  9445.                         "fmla   v17.4s, v13.4s, v4.s[0] \n"

  9446.                         "fmla   v18.4s, v12.4s, v4.s[1] \n"

  9447.                         "fmla   v19.4s, v13.4s, v4.s[1] \n"

  9448.                         "fmla   v20.4s, v12.4s, v4.s[2] \n"

  9449.                         "fmla   v21.4s, v13.4s, v4.s[2] \n"

  9450.                         "fmla   v22.4s, v12.4s, v4.s[3] \n"

  9451.                         "fmla   v23.4s, v13.4s, v4.s[3] \n"

  9452. 

  9453.                         "fmla   v24.4s, v12.4s, v5.s[0] \n"

  9454.                         "fmla   v25.4s, v13.4s, v5.s[0] \n"

  9455.                         "fmla   v26.4s, v12.4s, v5.s[1] \n"

  9456.                         "fmla   v27.4s, v13.4s, v5.s[1] \n"

  9457.                         "fmla   v28.4s, v12.4s, v5.s[2] \n"

  9458.                         "fmla   v29.4s, v13.4s, v5.s[2] \n"

  9459.                         "fmla   v30.4s, v12.4s, v5.s[3] \n"

  9460.                         "fmla   v31.4s, v13.4s, v5.s[3] \n"

  9461. 

  9462.                         "fmla   v16.4s, v14.4s, v6.s[0] \n"

  9463.                         "fmla   v17.4s, v15.4s, v6.s[0] \n"

  9464.                         "fmla   v18.4s, v14.4s, v6.s[1] \n"

  9465.                         "fmla   v19.4s, v15.4s, v6.s[1] \n"

  9466.                         "fmla   v20.4s, v14.4s, v6.s[2] \n"

  9467.                         "fmla   v21.4s, v15.4s, v6.s[2] \n"

  9468.                         "fmla   v22.4s, v14.4s, v6.s[3] \n"

  9469.                         "fmla   v23.4s, v15.4s, v6.s[3] \n"

  9470. 

  9471.                         "subs   w4, w4, #1              \n"

  9472. 

  9473.                         "fmla   v24.4s, v14.4s, v7.s[0] \n"

  9474.                         "fmla   v25.4s, v15.4s, v7.s[0] \n"

  9475.                         "fmla   v26.4s, v14.4s, v7.s[1] \n"

  9476.                         "fmla   v27.4s, v15.4s, v7.s[1] \n"

  9477.                         "fmla   v28.4s, v14.4s, v7.s[2] \n"

  9478.                         "fmla   v29.4s, v15.4s, v7.s[2] \n"

  9479.                         "fmla   v30.4s, v14.4s, v7.s[3] \n"

  9480.                         "fmla   v31.4s, v15.4s, v7.s[3] \n"

  9481. 

  9482.                         "bne    0b                      \n"

  9483. 

  9484.                         "1:                             \n"

  9485. 

  9486.                         // remain loop

  9487.                         "and    w4, %w20, #3            \n"// w4 = remain = tiles & 3;

  9488.                         "cmp    w4, #0                  \n"

  9489.                         "beq    3f                      \n"

  9490. 

  9491.                         "2:                             \n"

  9492. 

  9493.                         "prfm   pldl1keep, [%8, #256]   \n"

  9494.                         "ld1    {v8.4s, v9.4s}, [%8], #32   \n"

  9495. 

  9496.                         "prfm   pldl1keep, [%9, #256]   \n"

  9497.                         "ld1    {v0.4s, v1.4s}, [%9], #32   \n"

  9498. 

  9499.                         "fmla   v16.4s, v8.4s, v0.s[0]  \n"

  9500.                         "fmla   v17.4s, v9.4s, v0.s[0]  \n"

  9501.                         "fmla   v18.4s, v8.4s, v0.s[1]  \n"

  9502.                         "fmla   v19.4s, v9.4s, v0.s[1]  \n"

  9503.                         "fmla   v20.4s, v8.4s, v0.s[2]  \n"

  9504.                         "fmla   v21.4s, v9.4s, v0.s[2]  \n"

  9505.                         "fmla   v22.4s, v8.4s, v0.s[3]  \n"

  9506.                         "fmla   v23.4s, v9.4s, v0.s[3]  \n"

  9507. 

  9508.                         "subs   w4, w4, #1              \n"

  9509. 

  9510.                         "fmla   v24.4s, v8.4s, v1.s[0]  \n"

  9511.                         "fmla   v25.4s, v9.4s, v1.s[0]  \n"

  9512.                         "fmla   v26.4s, v8.4s, v1.s[1]  \n"

  9513.                         "fmla   v27.4s, v9.4s, v1.s[1]  \n"

  9514.                         "fmla   v28.4s, v8.4s, v1.s[2]  \n"

  9515.                         "fmla   v29.4s, v9.4s, v1.s[2]  \n"

  9516.                         "fmla   v30.4s, v8.4s, v1.s[3]  \n"

  9517.                         "fmla   v31.4s, v9.4s, v1.s[3]  \n"

  9518. 

  9519.                         "bne    2b                      \n"

  9520. 

  9521.                         "3:                             \n"

  9522. 

  9523.                         "st1    {v16.4s, v17.4s}, [%0], #32 \n"

  9524.                         "st1    {v18.4s, v19.4s}, [%1], #32 \n"

  9525.                         "st1    {v20.4s, v21.4s}, [%2], #32 \n"

  9526.                         "st1    {v22.4s, v23.4s}, [%3], #32 \n"

  9527.                         "st1    {v24.4s, v25.4s}, [%4], #32 \n"

  9528.                         "st1    {v26.4s, v27.4s}, [%5], #32 \n"

  9529.                         "st1    {v28.4s, v29.4s}, [%6], #32 \n"

  9530.                         "st1    {v30.4s, v31.4s}, [%7], #32 \n"

  9531. 

  9532.                         : "=r"(output0_tm), // %0

  9533.                           "=r"(output1_tm), // %1

  9534.                           "=r"(output2_tm), // %2

  9535.                           "=r"(output3_tm), // %3

  9536.                           "=r"(output4_tm), // %4

  9537.                           "=r"(output5_tm), // %5

  9538.                           "=r"(output6_tm), // %6

  9539.                           "=r"(output7_tm), // %7

  9540.                           "=r"(bb2p0),      // %8

  9541.                           "=r"(ktm0)        // %9

  9542.                         : "0"(output0_tm),

  9543.                           "1"(output1_tm),

  9544.                           "2"(output2_tm),

  9545.                           "3"(output3_tm),

  9546.                           "4"(output4_tm),

  9547.                           "5"(output5_tm),

  9548.                           "6"(output6_tm),

  9549.                           "7"(output7_tm),

  9550.                           "8"(bb2p0),

  9551.                           "9"(ktm0),

  9552.                           "r"(inch)         // %20

  9553.                         : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"

  9554.                     );

  9555.                 }

  9556.                 for (; i+3<tiles; i+=4)

  9557.                 {

  9558.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4);

  9559. 

  9560.                     const float* ktm0 = kernel_tm0.row(r);

  9561. 

  9562.                     asm volatile(

  9563.                         "eor    v16.16b, v16.16b, v16.16b  \n"

  9564.                         "eor    v17.16b, v17.16b, v17.16b  \n"

  9565.                         "eor    v18.16b, v18.16b, v18.16b  \n"

  9566.                         "eor    v19.16b, v19.16b, v19.16b  \n"

  9567.                         "eor    v20.16b, v20.16b, v20.16b  \n"

  9568.                         "eor    v21.16b, v21.16b, v21.16b  \n"

  9569.                         "eor    v22.16b, v22.16b, v22.16b  \n"

  9570.                         "eor    v23.16b, v23.16b, v23.16b  \n"

  9571. 

  9572.                         // inch loop

  9573.                         "lsr    w4, %w20, #2            \n"// w4 = nn = inch >> 2

  9574.                         "cmp    w4, #0                  \n"

  9575.                         "beq    1f                      \n"

  9576. 

  9577.                         "0:                             \n"

  9578. 

  9579.                         "prfm   pldl1keep, [%8, #512]   \n"

  9580.                         "ld1    {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64 \n"

  9581. 

  9582.                         "prfm   pldl1keep, [%9, #512]   \n"

  9583.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64   \n"

  9584. 

  9585.                         "fmla   v16.4s, v8.4s, v0.s[0]  \n"

  9586.                         "fmla   v17.4s, v8.4s, v0.s[1]  \n"

  9587.                         "fmla   v18.4s, v8.4s, v0.s[2]  \n"

  9588.                         "fmla   v19.4s, v8.4s, v0.s[3]  \n"

  9589.                         "fmla   v20.4s, v8.4s, v1.s[0]  \n"

  9590.                         "fmla   v21.4s, v8.4s, v1.s[1]  \n"

  9591.                         "fmla   v22.4s, v8.4s, v1.s[2]  \n"

  9592.                         "fmla   v23.4s, v8.4s, v1.s[3]  \n"

  9593. 

  9594.                         "prfm   pldl1keep, [%9, #512]   \n"

  9595.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64   \n"

  9596. 

  9597.                         "fmla   v16.4s, v9.4s, v2.s[0]  \n"

  9598.                         "fmla   v17.4s, v9.4s, v2.s[1]  \n"

  9599.                         "fmla   v18.4s, v9.4s, v2.s[2]  \n"

  9600.                         "fmla   v19.4s, v9.4s, v2.s[3]  \n"

  9601.                         "fmla   v20.4s, v9.4s, v3.s[0]  \n"

  9602.                         "fmla   v21.4s, v9.4s, v3.s[1]  \n"

  9603.                         "fmla   v22.4s, v9.4s, v3.s[2]  \n"

  9604.                         "fmla   v23.4s, v9.4s, v3.s[3]  \n"

  9605. 

  9606.                         "fmla   v16.4s, v10.4s, v4.s[0] \n"

  9607.                         "fmla   v17.4s, v10.4s, v4.s[1] \n"

  9608.                         "fmla   v18.4s, v10.4s, v4.s[2] \n"

  9609.                         "fmla   v19.4s, v10.4s, v4.s[3] \n"

  9610.                         "fmla   v20.4s, v10.4s, v5.s[0] \n"

  9611.                         "fmla   v21.4s, v10.4s, v5.s[1] \n"

  9612.                         "fmla   v22.4s, v10.4s, v5.s[2] \n"

  9613.                         "fmla   v23.4s, v10.4s, v5.s[3] \n"

  9614. 

  9615.                         "subs   w4, w4, #1              \n"

  9616. 

  9617.                         "fmla   v16.4s, v11.4s, v6.s[0] \n"

  9618.                         "fmla   v17.4s, v11.4s, v6.s[1] \n"

  9619.                         "fmla   v18.4s, v11.4s, v6.s[2] \n"

  9620.                         "fmla   v19.4s, v11.4s, v6.s[3] \n"

  9621.                         "fmla   v20.4s, v11.4s, v7.s[0] \n"

  9622.                         "fmla   v21.4s, v11.4s, v7.s[1] \n"

  9623.                         "fmla   v22.4s, v11.4s, v7.s[2] \n"

  9624.                         "fmla   v23.4s, v11.4s, v7.s[3] \n"

  9625. 

  9626.                         "bne    0b                      \n"

  9627. 

  9628.                         "1:                             \n"

  9629. 

  9630.                         // remain loop

  9631.                         "and    w4, %w20, #3            \n"// w4 = remain = tiles & 3;

  9632.                         "cmp    w4, #0                  \n"

  9633.                         "beq    3f                      \n"

  9634. 

  9635.                         "2:                             \n"

  9636. 

  9637.                         "prfm   pldl1keep, [%8, #128]   \n"

  9638.                         "ld1    {v8.4s}, [%8], #16      \n"

  9639. 

  9640.                         "prfm   pldl1keep, [%9, #256]   \n"

  9641.                         "ld1    {v0.4s, v1.4s}, [%9], #32   \n"

  9642. 

  9643.                         "fmla   v16.4s, v8.4s, v0.s[0]  \n"

  9644.                         "fmla   v17.4s, v8.4s, v0.s[1]  \n"

  9645.                         "fmla   v18.4s, v8.4s, v0.s[2]  \n"

  9646.                         "fmla   v19.4s, v8.4s, v0.s[3]  \n"

  9647. 

  9648.                         "subs   w4, w4, #1              \n"

  9649. 

  9650.                         "fmla   v20.4s, v8.4s, v1.s[0]  \n"

  9651.                         "fmla   v21.4s, v8.4s, v1.s[1]  \n"

  9652.                         "fmla   v22.4s, v8.4s, v1.s[2]  \n"

  9653.                         "fmla   v23.4s, v8.4s, v1.s[3]  \n"

  9654. 

  9655.                         "bne    2b                      \n"

  9656. 

  9657.                         "3:                             \n"

  9658. 

  9659.                         "st1    {v16.4s}, [%0], #16     \n"

  9660.                         "st1    {v17.4s}, [%1], #16     \n"

  9661.                         "st1    {v18.4s}, [%2], #16     \n"

  9662.                         "st1    {v19.4s}, [%3], #16     \n"

  9663.                         "st1    {v20.4s}, [%4], #16     \n"

  9664.                         "st1    {v21.4s}, [%5], #16     \n"

  9665.                         "st1    {v22.4s}, [%6], #16     \n"

  9666.                         "st1    {v23.4s}, [%7], #16     \n"

  9667. 

  9668.                         : "=r"(output0_tm), // %0

  9669.                           "=r"(output1_tm), // %1

  9670.                           "=r"(output2_tm), // %2

  9671.                           "=r"(output3_tm), // %3

  9672.                           "=r"(output4_tm), // %4

  9673.                           "=r"(output5_tm), // %5

  9674.                           "=r"(output6_tm), // %6

  9675.                           "=r"(output7_tm), // %7

  9676.                           "=r"(bb2p0),      // %8

  9677.                           "=r"(ktm0)        // %9

  9678.                         : "0"(output0_tm),

  9679.                           "1"(output1_tm),

  9680.                           "2"(output2_tm),

  9681.                           "3"(output3_tm),

  9682.                           "4"(output4_tm),

  9683.                           "5"(output5_tm),

  9684.                           "6"(output6_tm),

  9685.                           "7"(output7_tm),

  9686.                           "8"(bb2p0),

  9687.                           "9"(ktm0),

  9688.                           "r"(inch)         // %20

  9689.                         : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"

  9690.                     );

  9691.                 }

  9692.                 for (; i<tiles; i++)

  9693.                 {

  9694.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4+i%4);

  9695. 

  9696.                     const float* ktm0 = kernel_tm0.row(r);

  9697. 

  9698.                     float32x4_t _sum0123 = vdupq_n_f32(0.f);

  9699.                     float32x4_t _sum4567 = vdupq_n_f32(0.f);

  9700. 

  9701.                     int q=0;

  9702.                     for (; q+3<inch; q+=4)

  9703.                     {

  9704. //                         asm volatile("prfm pldl1keep, [%0, #128] \n" : :"r"(bb2p0) :);

  9705.                         float32x4_t _bb2p0 = vld1q_f32(bb2p0);

  9706.                         bb2p0 += 4;

  9707. 

  9708. //                         asm volatile("prfm pldl1keep, [%0, #512] \n" : :"r"(ktm0) :);

  9709.                         float32x4_t _ktm0 = vld1q_f32(ktm0 + 0);

  9710.                         float32x4_t _ktm1 = vld1q_f32(ktm0 + 4);

  9711.                         float32x4_t _ktm2 = vld1q_f32(ktm0 + 8);

  9712.                         float32x4_t _ktm3 = vld1q_f32(ktm0 + 12);

  9713.                         ktm0 += 16;

  9714. 

  9715.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm0, _bb2p0, 0);

  9716.                         _sum4567 = vmlaq_laneq_f32(_sum4567, _ktm1, _bb2p0, 0);

  9717.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm2, _bb2p0, 1);

  9718.                         _sum4567 = vmlaq_laneq_f32(_sum4567, _ktm3, _bb2p0, 1);

  9719. 

  9720. //                         asm volatile("prfm pldl1keep, [%0, #512] \n" : :"r"(ktm0) :);

  9721.                         float32x4_t _ktm4 = vld1q_f32(ktm0 + 0);

  9722.                         float32x4_t _ktm5 = vld1q_f32(ktm0 + 4);

  9723.                         float32x4_t _ktm6 = vld1q_f32(ktm0 + 8);

  9724.                         float32x4_t _ktm7 = vld1q_f32(ktm0 + 12);

  9725.                         ktm0 += 16;

  9726. 

  9727.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm4, _bb2p0, 2);

  9728.                         _sum4567 = vmlaq_laneq_f32(_sum4567, _ktm5, _bb2p0, 2);

  9729.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm6, _bb2p0, 3);

  9730.                         _sum4567 = vmlaq_laneq_f32(_sum4567, _ktm7, _bb2p0, 3);

  9731.                     }

  9732. 

  9733.                     for (; q<inch; q++)

  9734.                     {

  9735.                         float32x4_t _bb2p0 = vld1q_dup_f32(bb2p0);

  9736.                         float32x4_t _ktm0123 = vld1q_f32(ktm0 + 0);

  9737.                         float32x4_t _ktm4567 = vld1q_f32(ktm0 + 4);

  9738. 

  9739.                         _sum0123 = vmlaq_f32(_sum0123, _bb2p0, _ktm0123);

  9740.                         _sum4567 = vmlaq_f32(_sum4567, _bb2p0, _ktm4567);

  9741. 

  9742.                         bb2p0 += 1;

  9743.                         ktm0 += 8;

  9744.                     }

  9745. 

  9746.                     float sum0 = vgetq_lane_f32(_sum0123, 0);

  9747.                     float sum1 = vgetq_lane_f32(_sum0123, 1);

  9748.                     float sum2 = vgetq_lane_f32(_sum0123, 2);

  9749.                     float sum3 = vgetq_lane_f32(_sum0123, 3);

  9750.                     float sum4 = vgetq_lane_f32(_sum4567, 0);

  9751.                     float sum5 = vgetq_lane_f32(_sum4567, 1);

  9752.                     float sum6 = vgetq_lane_f32(_sum4567, 2);

  9753.                     float sum7 = vgetq_lane_f32(_sum4567, 3);

  9754. 

  9755.                     output0_tm[0] = sum0;

  9756.                     output1_tm[0] = sum1;

  9757.                     output2_tm[0] = sum2;

  9758.                     output3_tm[0] = sum3;

  9759.                     output4_tm[0] = sum4;

  9760.                     output5_tm[0] = sum5;

  9761.                     output6_tm[0] = sum6;

  9762.                     output7_tm[0] = sum7;

  9763. 

  9764.                     output0_tm += 1;

  9765.                     output1_tm += 1;

  9766.                     output2_tm += 1;

  9767.                     output3_tm += 1;

  9768.                     output4_tm += 1;

  9769.                     output5_tm += 1;

  9770.                     output6_tm += 1;

  9771.                     output7_tm += 1;

  9772.                 }

  9773.             }

  9774.         }

  9775. #endif // __aarch64__

  9776. 

  9777.         nn_outch = (outch - remain_outch_start) >> 2;

  9778. 

  9779.         #pragma omp parallel for num_threads(opt.num_threads)

  9780.         for (int pp=0; pp<nn_outch; pp++)

  9781.         {

  9782.             int p = remain_outch_start + pp * 4;

  9783. 

  9784. #if __ARM_NEON && __aarch64__

  9785.             const Mat kernel_tm0 = kernel_tm.channel(p/8+(p%8)/4);

  9786. #else

  9787.             const Mat kernel_tm0 = kernel_tm.channel(p/4);

  9788. #endif

  9789. 

  9790.             Mat out0_tm = top_blob_tm.channel(p);

  9791.             Mat out1_tm = top_blob_tm.channel(p+1);

  9792.             Mat out2_tm = top_blob_tm.channel(p+2);

  9793.             Mat out3_tm = top_blob_tm.channel(p+3);

  9794. 

  9795.             float* output0_tm = out0_tm;

  9796.             float* output1_tm = out1_tm;

  9797.             float* output2_tm = out2_tm;

  9798.             float* output3_tm = out3_tm;

  9799. 

  9800.             for (int r=0; r<64; r++)

  9801.             {

  9802.                 const Mat bb2 = bottom_blob_tm2.channel(r);

  9803. 

  9804.                 // tile

  9805.                 int i=0;

  9806.                 for (; i+7<tiles; i+=8)

  9807.                 {

  9808.                     const float* bb2p0 = bb2.row(i/8);

  9809. 

  9810.                     const float* ktm0 = kernel_tm0.row(r);

  9811. #if __ARM_NEON

  9812. #if __aarch64__

  9813.                     asm volatile(

  9814.                         "eor    v8.16b, v8.16b, v8.16b     \n"

  9815.                         "eor    v9.16b, v9.16b, v9.16b     \n"

  9816.                         "eor    v10.16b, v10.16b, v10.16b  \n"

  9817.                         "eor    v11.16b, v11.16b, v11.16b  \n"

  9818.                         "eor    v12.16b, v12.16b, v12.16b  \n"

  9819.                         "eor    v13.16b, v13.16b, v13.16b  \n"

  9820.                         "eor    v14.16b, v14.16b, v14.16b  \n"

  9821.                         "eor    v15.16b, v15.16b, v15.16b  \n"

  9822. 

  9823.                         // inch loop

  9824.                         "lsr    w4, %w12, #2            \n"// w4 = nn = inch >> 2

  9825.                         "cmp    w4, #0                  \n"

  9826.                         "beq    1f                      \n"

  9827. 

  9828.                         "0:                             \n"

  9829. 

  9830.                         "prfm   pldl1keep, [%4, #512]   \n"

  9831.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64     \n"

  9832. 

  9833.                         "prfm   pldl1keep, [%5, #512]   \n"

  9834.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64     \n"

  9835. 

  9836.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  9837.                         "fmla   v9.4s, v5.4s, v0.s[0]   \n"

  9838.                         "fmla   v10.4s, v4.4s, v0.s[1]  \n"

  9839.                         "fmla   v11.4s, v5.4s, v0.s[1]  \n"

  9840.                         "fmla   v12.4s, v4.4s, v0.s[2]  \n"

  9841.                         "fmla   v13.4s, v5.4s, v0.s[2]  \n"

  9842.                         "fmla   v14.4s, v4.4s, v0.s[3]  \n"

  9843.                         "fmla   v15.4s, v5.4s, v0.s[3]  \n"

  9844. 

  9845.                         "prfm   pldl1keep, [%4, #512]   \n"

  9846.                         "ld1    {v16.4s, v17.4s, v18.4s, v19.4s}, [%4], #64 \n"

  9847. 

  9848.                         "fmla   v8.4s, v6.4s, v1.s[0]   \n"

  9849.                         "fmla   v9.4s, v7.4s, v1.s[0]   \n"

  9850.                         "fmla   v10.4s, v6.4s, v1.s[1]  \n"

  9851.                         "fmla   v11.4s, v7.4s, v1.s[1]  \n"

  9852.                         "fmla   v12.4s, v6.4s, v1.s[2]  \n"

  9853.                         "fmla   v13.4s, v7.4s, v1.s[2]  \n"

  9854.                         "fmla   v14.4s, v6.4s, v1.s[3]  \n"

  9855.                         "fmla   v15.4s, v7.4s, v1.s[3]  \n"

  9856. 

  9857.                         "fmla   v8.4s, v16.4s, v2.s[0]  \n"

  9858.                         "fmla   v9.4s, v17.4s, v2.s[0]  \n"

  9859.                         "fmla   v10.4s, v16.4s, v2.s[1] \n"

  9860.                         "fmla   v11.4s, v17.4s, v2.s[1] \n"

  9861.                         "fmla   v12.4s, v16.4s, v2.s[2] \n"

  9862.                         "fmla   v13.4s, v17.4s, v2.s[2] \n"

  9863.                         "fmla   v14.4s, v16.4s, v2.s[3] \n"

  9864.                         "fmla   v15.4s, v17.4s, v2.s[3] \n"

  9865. 

  9866.                         "fmla   v8.4s, v18.4s, v3.s[0]  \n"

  9867.                         "fmla   v9.4s, v19.4s, v3.s[0]  \n"

  9868.                         "fmla   v10.4s, v18.4s, v3.s[1] \n"

  9869.                         "fmla   v11.4s, v19.4s, v3.s[1] \n"

  9870.                         "fmla   v12.4s, v18.4s, v3.s[2] \n"

  9871.                         "fmla   v13.4s, v19.4s, v3.s[2] \n"

  9872.                         "fmla   v14.4s, v18.4s, v3.s[3] \n"

  9873.                         "fmla   v15.4s, v19.4s, v3.s[3] \n"

  9874. 

  9875.                         "subs   w4, w4, #1              \n"

  9876.                         "bne    0b                      \n"

  9877. 

  9878.                         "1:                             \n"

  9879. 

  9880.                         // remain loop

  9881.                         "and    w4, %w12, #3            \n"// w4 = remain = tiles & 3;

  9882.                         "cmp    w4, #0                  \n"

  9883.                         "beq    3f                      \n"

  9884. 

  9885.                         "2:                             \n"

  9886. 

  9887.                         "prfm   pldl1keep, [%4, #256]   \n"

  9888.                         "ld1    {v4.4s, v5.4s}, [%4], #32      \n"

  9889. 

  9890.                         "prfm   pldl1keep, [%5, #128]   \n"

  9891.                         "ld1    {v0.4s}, [%5], #16      \n"

  9892. 

  9893.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  9894.                         "fmla   v9.4s, v5.4s, v0.s[0]   \n"

  9895.                         "fmla   v10.4s, v4.4s, v0.s[1]  \n"

  9896.                         "fmla   v11.4s, v5.4s, v0.s[1]  \n"

  9897.                         "fmla   v12.4s, v4.4s, v0.s[2]  \n"

  9898.                         "fmla   v13.4s, v5.4s, v0.s[2]  \n"

  9899.                         "fmla   v14.4s, v4.4s, v0.s[3]  \n"

  9900.                         "fmla   v15.4s, v5.4s, v0.s[3]  \n"

  9901. 

  9902.                         "subs   w4, w4, #1              \n"

  9903.                         "bne    2b                      \n"

  9904. 

  9905.                         "3:                             \n"

  9906. 

  9907.                         "st1    {v8.4s, v9.4s}, [%0], #32       \n"

  9908.                         "st1    {v10.4s, v11.4s}, [%1], #32     \n"

  9909.                         "st1    {v12.4s, v13.4s}, [%2], #32     \n"

  9910.                         "st1    {v14.4s, v15.4s}, [%3], #32     \n"

  9911. 

  9912.                         : "=r"(output0_tm), // %0

  9913.                           "=r"(output1_tm), // %1

  9914.                           "=r"(output2_tm), // %2

  9915.                           "=r"(output3_tm), // %3

  9916.                           "=r"(bb2p0),      // %4

  9917.                           "=r"(ktm0)        // %5

  9918.                         : "0"(output0_tm),

  9919.                           "1"(output1_tm),

  9920.                           "2"(output2_tm),

  9921.                           "3"(output3_tm),

  9922.                           "4"(bb2p0),

  9923.                           "5"(ktm0),

  9924.                           "r"(inch)         // %12

  9925.                         : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"

  9926.                     );

  9927. #else // __aarch64__

  9928.                     asm volatile(

  9929.                         "veor       q8, q8, q8      \n"

  9930.                         "veor       q9, q9, q9      \n"

  9931.                         "veor       q10, q10, q10   \n"

  9932.                         "veor       q11, q11, q11   \n"

  9933.                         "veor       q12, q12, q12   \n"

  9934.                         "veor       q13, q13, q13   \n"

  9935.                         "veor       q14, q14, q14   \n"

  9936.                         "veor       q15, q15, q15   \n"

  9937. 

  9938.                         // inch loop

  9939.                         "lsr        r4, %12, #2     \n"// r4 = nn = inch >> 2

  9940.                         "cmp        r4, #0          \n"

  9941.                         "beq        1f              \n"

  9942. 

  9943.                         "0:                         \n"

  9944. 

  9945.                         "pld        [%4, #512]      \n"

  9946.                         "vldm       %4!, {d8-d15}   \n"

  9947. //                         "vld1.f32   {d8-d11}, [%4 :128]! \n"

  9948. //                         "vld1.f32   {d12-d15}, [%4 :128]! \n"

  9949. 

  9950.                         "pld        [%5, #512]      \n"

  9951.                         "vldm       %5!, {d0-d7}    \n"

  9952. //                         "vld1.f32   {d0-d3}, [%5 :128]!  \n"

  9953. //                         "vld1.f32   {d4-d7}, [%5 :128]!  \n"

  9954. 

  9955.                         "vmla.f32   q8, q4, d0[0]   \n"

  9956.                         "vmla.f32   q9, q5, d0[0]   \n"

  9957.                         "vmla.f32   q10, q4, d0[1]  \n"

  9958.                         "vmla.f32   q11, q5, d0[1]  \n"

  9959.                         "vmla.f32   q12, q4, d1[0]  \n"

  9960.                         "vmla.f32   q13, q5, d1[0]  \n"

  9961.                         "vmla.f32   q14, q4, d1[1]  \n"

  9962.                         "vmla.f32   q15, q5, d1[1]  \n"

  9963. 

  9964.                         "vmla.f32   q8, q6, d2[0]   \n"

  9965.                         "vmla.f32   q9, q7, d2[0]   \n"

  9966.                         "vmla.f32   q10, q6, d2[1]  \n"

  9967.                         "vmla.f32   q11, q7, d2[1]  \n"

  9968.                         "vmla.f32   q12, q6, d3[0]  \n"

  9969.                         "vmla.f32   q13, q7, d3[0]  \n"

  9970.                         "vmla.f32   q14, q6, d3[1]  \n"

  9971.                         "vmla.f32   q15, q7, d3[1]  \n"

  9972. 

  9973.                         "pld        [%4, #512]      \n"

  9974.                         "vldm       %4!, {d8-d15}   \n"

  9975. //                         "vld1.f32   {d8-d11}, [%4 :128]! \n"

  9976. //                         "vld1.f32   {d12-d15}, [%4 :128]! \n"

  9977. 

  9978.                         "vmla.f32   q8, q4, d4[0]   \n"

  9979.                         "vmla.f32   q9, q5, d4[0]   \n"

  9980.                         "vmla.f32   q10, q4, d4[1]  \n"

  9981.                         "vmla.f32   q11, q5, d4[1]  \n"

  9982.                         "vmla.f32   q12, q4, d5[0]  \n"

  9983.                         "vmla.f32   q13, q5, d5[0]  \n"

  9984.                         "vmla.f32   q14, q4, d5[1]  \n"

  9985.                         "vmla.f32   q15, q5, d5[1]  \n"

  9986. 

  9987.                         "subs       r4, r4, #1      \n"

  9988. 

  9989.                         "vmla.f32   q8, q6, d6[0]   \n"

  9990.                         "vmla.f32   q9, q7, d6[0]   \n"

  9991.                         "vmla.f32   q10, q6, d6[1]  \n"

  9992.                         "vmla.f32   q11, q7, d6[1]  \n"

  9993.                         "vmla.f32   q12, q6, d7[0]  \n"

  9994.                         "vmla.f32   q13, q7, d7[0]  \n"

  9995.                         "vmla.f32   q14, q6, d7[1]  \n"

  9996.                         "vmla.f32   q15, q7, d7[1]  \n"

  9997. 

  9998.                         "bne        0b              \n"

  9999. 

  10000.                         "1:                         \n"

  10001. 

  10002.                         // remain loop

  10003.                         "and        r4, %12, #3     \n"// r4 = remain = tiles & 3;

  10004.                         "cmp        r4, #0          \n"

  10005.                         "beq        3f              \n"

  10006. 

  10007.                         "2:                         \n"

  10008. 

  10009.                         "pld        [%4, #256]      \n"

  10010.                         "vld1.f32   {d8-d11}, [%4 :128]! \n"

  10011. 

  10012.                         "pld        [%5, #128]      \n"

  10013.                         "vld1.f32   {d0-d1}, [%5 :128]!  \n"

  10014. 

  10015.                         "vmla.f32   q8, q4, d0[0]   \n"

  10016.                         "vmla.f32   q9, q5, d0[0]   \n"

  10017.                         "vmla.f32   q10, q4, d0[1]  \n"

  10018.                         "vmla.f32   q11, q5, d0[1]  \n"

  10019. 

  10020.                         "subs       r4, r4, #1      \n"

  10021. 

  10022.                         "vmla.f32   q12, q4, d1[0]  \n"

  10023.                         "vmla.f32   q13, q5, d1[0]  \n"

  10024.                         "vmla.f32   q14, q4, d1[1]  \n"

  10025.                         "vmla.f32   q15, q5, d1[1]  \n"

  10026. 

  10027.                         "bne        2b              \n"

  10028. 

  10029.                         "3:                         \n"

  10030. 

  10031.                         "vst1.f32   {d16-d19}, [%0]! \n"

  10032.                         "vst1.f32   {d20-d23}, [%1]! \n"

  10033.                         "vst1.f32   {d24-d27}, [%2]! \n"

  10034.                         "vst1.f32   {d28-d31}, [%3]! \n"

  10035. 

  10036.                         : "=r"(output0_tm), // %0

  10037.                           "=r"(output1_tm), // %1

  10038.                           "=r"(output2_tm), // %2

  10039.                           "=r"(output3_tm), // %3

  10040.                           "=r"(bb2p0),      // %4

  10041.                           "=r"(ktm0)        // %5

  10042.                         : "0"(output0_tm),

  10043.                           "1"(output1_tm),

  10044.                           "2"(output2_tm),

  10045.                           "3"(output3_tm),

  10046.                           "4"(bb2p0),

  10047.                           "5"(ktm0),

  10048.                           "r"(inch)         // %12

  10049.                         : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  10050.                     );

  10051. #endif // __aarch64__

  10052. #else

  10053.                     float sum0_0 = 0.f;

  10054.                     float sum0_1 = 0.f;

  10055.                     float sum0_2 = 0.f;

  10056.                     float sum0_3 = 0.f;

  10057.                     float sum0_4 = 0.f;

  10058.                     float sum0_5 = 0.f;

  10059.                     float sum0_6 = 0.f;

  10060.                     float sum0_7 = 0.f;

  10061. 

  10062.                     float sum1_0 = 0.f;

  10063.                     float sum1_1 = 0.f;

  10064.                     float sum1_2 = 0.f;

  10065.                     float sum1_3 = 0.f;

  10066.                     float sum1_4 = 0.f;

  10067.                     float sum1_5 = 0.f;

  10068.                     float sum1_6 = 0.f;

  10069.                     float sum1_7 = 0.f;

  10070. 

  10071.                     float sum2_0 = 0.f;

  10072.                     float sum2_1 = 0.f;

  10073.                     float sum2_2 = 0.f;

  10074.                     float sum2_3 = 0.f;

  10075.                     float sum2_4 = 0.f;

  10076.                     float sum2_5 = 0.f;

  10077.                     float sum2_6 = 0.f;

  10078.                     float sum2_7 = 0.f;

  10079. 

  10080.                     float sum3_0 = 0.f;

  10081.                     float sum3_1 = 0.f;

  10082.                     float sum3_2 = 0.f;

  10083.                     float sum3_3 = 0.f;

  10084.                     float sum3_4 = 0.f;

  10085.                     float sum3_5 = 0.f;

  10086.                     float sum3_6 = 0.f;

  10087.                     float sum3_7 = 0.f;

  10088. 

  10089.                     for (int q=0; q<inch; q++)

  10090.                     {

  10091.                         sum0_0 += bb2p0[0] * ktm0[0];

  10092.                         sum0_1 += bb2p0[1] * ktm0[0];

  10093.                         sum0_2 += bb2p0[2] * ktm0[0];

  10094.                         sum0_3 += bb2p0[3] * ktm0[0];

  10095.                         sum0_4 += bb2p0[4] * ktm0[0];

  10096.                         sum0_5 += bb2p0[5] * ktm0[0];

  10097.                         sum0_6 += bb2p0[6] * ktm0[0];

  10098.                         sum0_7 += bb2p0[7] * ktm0[0];

  10099. 

  10100.                         sum1_0 += bb2p0[0] * ktm0[1];

  10101.                         sum1_1 += bb2p0[1] * ktm0[1];

  10102.                         sum1_2 += bb2p0[2] * ktm0[1];

  10103.                         sum1_3 += bb2p0[3] * ktm0[1];

  10104.                         sum1_4 += bb2p0[4] * ktm0[1];

  10105.                         sum1_5 += bb2p0[5] * ktm0[1];

  10106.                         sum1_6 += bb2p0[6] * ktm0[1];

  10107.                         sum1_7 += bb2p0[7] * ktm0[1];

  10108. 

  10109.                         sum2_0 += bb2p0[0] * ktm0[2];

  10110.                         sum2_1 += bb2p0[1] * ktm0[2];

  10111.                         sum2_2 += bb2p0[2] * ktm0[2];

  10112.                         sum2_3 += bb2p0[3] * ktm0[2];

  10113.                         sum2_4 += bb2p0[4] * ktm0[2];

  10114.                         sum2_5 += bb2p0[5] * ktm0[2];

  10115.                         sum2_6 += bb2p0[6] * ktm0[2];

  10116.                         sum2_7 += bb2p0[7] * ktm0[2];

  10117. 

  10118.                         sum3_0 += bb2p0[0] * ktm0[3];

  10119.                         sum3_1 += bb2p0[1] * ktm0[3];

  10120.                         sum3_2 += bb2p0[2] * ktm0[3];

  10121.                         sum3_3 += bb2p0[3] * ktm0[3];

  10122.                         sum3_4 += bb2p0[4] * ktm0[3];

  10123.                         sum3_5 += bb2p0[5] * ktm0[3];

  10124.                         sum3_6 += bb2p0[6] * ktm0[3];

  10125.                         sum3_7 += bb2p0[7] * ktm0[3];

  10126. 

  10127.                         bb2p0 += 8;

  10128.                         ktm0 += 4;

  10129.                     }

  10130. 

  10131.                     output0_tm[0] = sum0_0;

  10132.                     output0_tm[1] = sum0_1;

  10133.                     output0_tm[2] = sum0_2;

  10134.                     output0_tm[3] = sum0_3;

  10135.                     output0_tm[4] = sum0_4;

  10136.                     output0_tm[5] = sum0_5;

  10137.                     output0_tm[6] = sum0_6;

  10138.                     output0_tm[7] = sum0_7;

  10139. 

  10140.                     output1_tm[0] = sum1_0;

  10141.                     output1_tm[1] = sum1_1;

  10142.                     output1_tm[2] = sum1_2;

  10143.                     output1_tm[3] = sum1_3;

  10144.                     output1_tm[4] = sum1_4;

  10145.                     output1_tm[5] = sum1_5;

  10146.                     output1_tm[6] = sum1_6;

  10147.                     output1_tm[7] = sum1_7;

  10148. 

  10149.                     output2_tm[0] = sum2_0;

  10150.                     output2_tm[1] = sum2_1;

  10151.                     output2_tm[2] = sum2_2;

  10152.                     output2_tm[3] = sum2_3;

  10153.                     output2_tm[4] = sum2_4;

  10154.                     output2_tm[5] = sum2_5;

  10155.                     output2_tm[6] = sum2_6;

  10156.                     output2_tm[7] = sum2_7;

  10157. 

  10158.                     output3_tm[0] = sum3_0;

  10159.                     output3_tm[1] = sum3_1;

  10160.                     output3_tm[2] = sum3_2;

  10161.                     output3_tm[3] = sum3_3;

  10162.                     output3_tm[4] = sum3_4;

  10163.                     output3_tm[5] = sum3_5;

  10164.                     output3_tm[6] = sum3_6;

  10165.                     output3_tm[7] = sum3_7;

  10166. 

  10167.                     output0_tm += 8;

  10168.                     output1_tm += 8;

  10169.                     output2_tm += 8;

  10170.                     output3_tm += 8;

  10171. #endif // __ARM_NEON

  10172.                 }

  10173.                 for (; i+3<tiles; i+=4)

  10174.                 {

  10175.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4);

  10176. 

  10177.                     const float* ktm0 = kernel_tm0.row(r);

  10178. #if __ARM_NEON

  10179. #if __aarch64__

  10180.                     asm volatile(

  10181.                         "eor    v8.16b, v8.16b, v8.16b     \n"

  10182.                         "eor    v9.16b, v9.16b, v9.16b     \n"

  10183.                         "eor    v10.16b, v10.16b, v10.16b  \n"

  10184.                         "eor    v11.16b, v11.16b, v11.16b  \n"

  10185. 

  10186.                         // inch loop

  10187.                         "lsr    w4, %w12, #2            \n"// w4 = nn = inch >> 2

  10188.                         "cmp    w4, #0                  \n"

  10189.                         "beq    1f                      \n"

  10190. 

  10191.                         "0:                             \n"

  10192. 

  10193.                         "prfm   pldl1keep, [%4, #512]   \n"

  10194.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64     \n"

  10195. 

  10196.                         "prfm   pldl1keep, [%5, #512]   \n"

  10197.                         "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64     \n"

  10198. 

  10199.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  10200.                         "fmla   v9.4s, v4.4s, v0.s[1]   \n"

  10201.                         "fmla   v10.4s, v4.4s, v0.s[2]  \n"

  10202.                         "fmla   v11.4s, v4.4s, v0.s[3]  \n"

  10203. 

  10204.                         "fmla   v8.4s, v5.4s, v1.s[0]   \n"

  10205.                         "fmla   v9.4s, v5.4s, v1.s[1]   \n"

  10206.                         "fmla   v10.4s, v5.4s, v1.s[2]  \n"

  10207.                         "fmla   v11.4s, v5.4s, v1.s[3]  \n"

  10208. 

  10209.                         "fmla   v8.4s, v6.4s, v2.s[0]   \n"

  10210.                         "fmla   v9.4s, v6.4s, v2.s[1]   \n"

  10211.                         "fmla   v10.4s, v6.4s, v2.s[2]  \n"

  10212.                         "fmla   v11.4s, v6.4s, v2.s[3]  \n"

  10213. 

  10214.                         "fmla   v8.4s, v7.4s, v3.s[0]   \n"

  10215.                         "fmla   v9.4s, v7.4s, v3.s[1]   \n"

  10216.                         "fmla   v10.4s, v7.4s, v3.s[2]  \n"

  10217.                         "fmla   v11.4s, v7.4s, v3.s[3]  \n"

  10218. 

  10219.                         "subs   w4, w4, #1              \n"

  10220.                         "bne    0b                      \n"

  10221. 

  10222.                         "1:                             \n"

  10223. 

  10224.                         // remain loop

  10225.                         "and    w4, %w12, #3            \n"// w4 = remain = tiles & 3;

  10226.                         "cmp    w4, #0                  \n"

  10227.                         "beq    3f                      \n"

  10228. 

  10229.                         "2:                             \n"

  10230. 

  10231.                         "prfm   pldl1keep, [%4, #128]   \n"

  10232.                         "ld1    {v4.4s}, [%4], #16      \n"

  10233. 

  10234.                         "prfm   pldl1keep, [%5, #128]   \n"

  10235.                         "ld1    {v0.4s}, [%5], #16      \n"

  10236. 

  10237.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  10238.                         "fmla   v9.4s, v4.4s, v0.s[1]   \n"

  10239.                         "fmla   v10.4s, v4.4s, v0.s[2]  \n"

  10240.                         "fmla   v11.4s, v4.4s, v0.s[3]  \n"

  10241. 

  10242.                         "subs   w4, w4, #1              \n"

  10243.                         "bne    2b                      \n"

  10244. 

  10245.                         "3:                             \n"

  10246. 

  10247.                         "st1    {v8.4s}, [%0], #16      \n"

  10248.                         "st1    {v9.4s}, [%1], #16      \n"

  10249.                         "st1    {v10.4s}, [%2], #16     \n"

  10250.                         "st1    {v11.4s}, [%3], #16     \n"

  10251. 

  10252.                         : "=r"(output0_tm), // %0

  10253.                           "=r"(output1_tm), // %1

  10254.                           "=r"(output2_tm), // %2

  10255.                           "=r"(output3_tm), // %3

  10256.                           "=r"(bb2p0),      // %4

  10257.                           "=r"(ktm0)        // %5

  10258.                         : "0"(output0_tm),

  10259.                           "1"(output1_tm),

  10260.                           "2"(output2_tm),

  10261.                           "3"(output3_tm),

  10262.                           "4"(bb2p0),

  10263.                           "5"(ktm0),

  10264.                           "r"(inch)         // %12

  10265.                         : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"

  10266.                     );

  10267. #else // __aarch64__

  10268.                     asm volatile(

  10269.                         "veor       q8, q8, q8      \n"

  10270.                         "veor       q9, q9, q9      \n"

  10271.                         "veor       q10, q10, q10   \n"

  10272.                         "veor       q11, q11, q11   \n"

  10273. 

  10274.                         // inch loop

  10275.                         "lsr        r4, %12, #2     \n"// r4 = nn = inch >> 2

  10276.                         "cmp        r4, #0          \n"

  10277.                         "beq        1f              \n"

  10278. 

  10279.                         "0:                         \n"

  10280. 

  10281.                         "pld        [%4, #512]      \n"

  10282.                         "vldm       %4!, {d8-d15}   \n"

  10283. //                         "vld1.f32   {d8-d11}, [%4 :128]! \n"

  10284. //                         "vld1.f32   {d12-d15}, [%4 :128]! \n"

  10285. 

  10286.                         "pld        [%5, #512]      \n"

  10287.                         "vldm       %5!, {d0-d7}    \n"

  10288. //                         "vld1.f32   {d0-d3}, [%5 :128]!  \n"

  10289. //                         "vld1.f32   {d4-d7}, [%5 :128]!  \n"

  10290. 

  10291.                         "vmla.f32   q8, q4, d0[0]   \n"

  10292.                         "vmla.f32   q9, q4, d0[1]   \n"

  10293.                         "vmla.f32   q10, q4, d1[0]  \n"

  10294.                         "vmla.f32   q11, q4, d1[1]  \n"

  10295. 

  10296.                         "vmla.f32   q8, q5, d2[0]   \n"

  10297.                         "vmla.f32   q9, q5, d2[1]   \n"

  10298.                         "vmla.f32   q10, q5, d3[0]  \n"

  10299.                         "vmla.f32   q11, q5, d3[1]  \n"

  10300. 

  10301.                         "subs       r4, r4, #1      \n"

  10302. 

  10303.                         "vmla.f32   q8, q6, d4[0]   \n"

  10304.                         "vmla.f32   q9, q6, d4[1]   \n"

  10305.                         "vmla.f32   q10, q6, d5[0]  \n"

  10306.                         "vmla.f32   q11, q6, d5[1]  \n"

  10307. 

  10308.                         "vmla.f32   q8, q7, d6[0]   \n"

  10309.                         "vmla.f32   q9, q7, d6[1]   \n"

  10310.                         "vmla.f32   q10, q7, d7[0]  \n"

  10311.                         "vmla.f32   q11, q7, d7[1]  \n"

  10312. 

  10313.                         "bne        0b              \n"

  10314. 

  10315.                         "1:                         \n"

  10316. 

  10317.                         // remain loop

  10318.                         "and        r4, %12, #3     \n"// r4 = remain = tiles & 3;

  10319.                         "cmp        r4, #0          \n"

  10320.                         "beq        3f              \n"

  10321. 

  10322.                         "2:                         \n"

  10323. 

  10324.                         "pld        [%4, #128]      \n"

  10325.                         "vld1.f32   {d8-d9}, [%4 :128]!  \n"

  10326. 

  10327.                         "pld        [%5, #128]      \n"

  10328.                         "vld1.f32   {d0-d1}, [%5 :128]!  \n"

  10329. 

  10330.                         "subs       r4, r4, #1      \n"

  10331. 

  10332.                         "vmla.f32   q8, q4, d0[0]   \n"

  10333.                         "vmla.f32   q9, q4, d0[1]   \n"

  10334.                         "vmla.f32   q10, q4, d1[0]  \n"

  10335.                         "vmla.f32   q11, q4, d1[1]  \n"

  10336. 

  10337.                         "bne        2b              \n"

  10338. 

  10339.                         "3:                         \n"

  10340. 

  10341.                         "vst1.f32   {d16-d17}, [%0]! \n"

  10342.                         "vst1.f32   {d18-d19}, [%1]! \n"

  10343.                         "vst1.f32   {d20-d21}, [%2]! \n"

  10344.                         "vst1.f32   {d22-d23}, [%3]! \n"

  10345. 

  10346.                         : "=r"(output0_tm), // %0

  10347.                           "=r"(output1_tm), // %1

  10348.                           "=r"(output2_tm), // %2

  10349.                           "=r"(output3_tm), // %3

  10350.                           "=r"(bb2p0),      // %4

  10351.                           "=r"(ktm0)        // %5

  10352.                         : "0"(output0_tm),

  10353.                           "1"(output1_tm),

  10354.                           "2"(output2_tm),

  10355.                           "3"(output3_tm),

  10356.                           "4"(bb2p0),

  10357.                           "5"(ktm0),

  10358.                           "r"(inch)         // %12

  10359.                         : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"

  10360.                     );

  10361. #endif // __aarch64__

  10362. #else

  10363.                     float sum0_0 = 0.f;

  10364.                     float sum0_1 = 0.f;

  10365.                     float sum0_2 = 0.f;

  10366.                     float sum0_3 = 0.f;

  10367. 

  10368.                     float sum1_0 = 0.f;

  10369.                     float sum1_1 = 0.f;

  10370.                     float sum1_2 = 0.f;

  10371.                     float sum1_3 = 0.f;

  10372. 

  10373.                     float sum2_0 = 0.f;

  10374.                     float sum2_1 = 0.f;

  10375.                     float sum2_2 = 0.f;

  10376.                     float sum2_3 = 0.f;

  10377. 

  10378.                     float sum3_0 = 0.f;

  10379.                     float sum3_1 = 0.f;

  10380.                     float sum3_2 = 0.f;

  10381.                     float sum3_3 = 0.f;

  10382. 

  10383.                     for (int q=0; q<inch; q++)

  10384.                     {

  10385.                         sum0_0 += bb2p0[0] * ktm0[0];

  10386.                         sum0_1 += bb2p0[1] * ktm0[0];

  10387.                         sum0_2 += bb2p0[2] * ktm0[0];

  10388.                         sum0_3 += bb2p0[3] * ktm0[0];

  10389. 

  10390.                         sum1_0 += bb2p0[0] * ktm0[1];

  10391.                         sum1_1 += bb2p0[1] * ktm0[1];

  10392.                         sum1_2 += bb2p0[2] * ktm0[1];

  10393.                         sum1_3 += bb2p0[3] * ktm0[1];

  10394. 

  10395.                         sum2_0 += bb2p0[0] * ktm0[2];

  10396.                         sum2_1 += bb2p0[1] * ktm0[2];

  10397.                         sum2_2 += bb2p0[2] * ktm0[2];

  10398.                         sum2_3 += bb2p0[3] * ktm0[2];

  10399. 

  10400.                         sum3_0 += bb2p0[0] * ktm0[3];

  10401.                         sum3_1 += bb2p0[1] * ktm0[3];

  10402.                         sum3_2 += bb2p0[2] * ktm0[3];

  10403.                         sum3_3 += bb2p0[3] * ktm0[3];

  10404. 

  10405.                         bb2p0 += 4;

  10406.                         ktm0 += 4;

  10407.                     }

  10408. 

  10409.                     output0_tm[0] = sum0_0;

  10410.                     output0_tm[1] = sum0_1;

  10411.                     output0_tm[2] = sum0_2;

  10412.                     output0_tm[3] = sum0_3;

  10413. 

  10414.                     output1_tm[0] = sum1_0;

  10415.                     output1_tm[1] = sum1_1;

  10416.                     output1_tm[2] = sum1_2;

  10417.                     output1_tm[3] = sum1_3;

  10418. 

  10419.                     output2_tm[0] = sum2_0;

  10420.                     output2_tm[1] = sum2_1;

  10421.                     output2_tm[2] = sum2_2;

  10422.                     output2_tm[3] = sum2_3;

  10423. 

  10424.                     output3_tm[0] = sum3_0;

  10425.                     output3_tm[1] = sum3_1;

  10426.                     output3_tm[2] = sum3_2;

  10427.                     output3_tm[3] = sum3_3;

  10428. 

  10429.                     output0_tm += 4;

  10430.                     output1_tm += 4;

  10431.                     output2_tm += 4;

  10432.                     output3_tm += 4;

  10433. #endif // __ARM_NEON

  10434.                 }

  10435.                 for (; i<tiles; i++)

  10436.                 {

  10437.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4+i%4);

  10438. 

  10439.                     const float* ktm0 = kernel_tm0.row(r);

  10440. 

  10441. #if __ARM_NEON

  10442.                     float32x4_t _sum0123 = vdupq_n_f32(0.f);

  10443. 

  10444.                     int q=0;

  10445.                     for (; q+3<inch; q+=4)

  10446.                     {

  10447. //                         asm volatile("prfm pldl1keep, [%0, #128] \n" : :"r"(bb2p0) :);

  10448.                         float32x4_t _bb2p0 = vld1q_f32(bb2p0);

  10449.                         bb2p0 += 4;

  10450. 

  10451. //                         asm volatile("prfm pldl1keep, [%0, #512] \n" : :"r"(ktm0) :);

  10452.                         float32x4_t _ktm0 = vld1q_f32(ktm0 + 0);

  10453.                         float32x4_t _ktm1 = vld1q_f32(ktm0 + 4);

  10454.                         float32x4_t _ktm2 = vld1q_f32(ktm0 + 8);

  10455.                         float32x4_t _ktm3 = vld1q_f32(ktm0 + 12);

  10456.                         ktm0 += 16;

  10457. 

  10458. #if __aarch64__

  10459.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm0, _bb2p0, 0);

  10460.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm1, _bb2p0, 1);

  10461.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm2, _bb2p0, 2);

  10462.                         _sum0123 = vmlaq_laneq_f32(_sum0123, _ktm3, _bb2p0, 3);

  10463. #else

  10464.                         _sum0123 = vmlaq_lane_f32(_sum0123, _ktm0, vget_low_f32(_bb2p0), 0);

  10465.                         _sum0123 = vmlaq_lane_f32(_sum0123, _ktm1, vget_low_f32(_bb2p0), 1);

  10466.                         _sum0123 = vmlaq_lane_f32(_sum0123, _ktm2, vget_high_f32(_bb2p0), 0);

  10467.                         _sum0123 = vmlaq_lane_f32(_sum0123, _ktm3, vget_high_f32(_bb2p0), 1);

  10468. #endif // __aarch64__

  10469.                     }

  10470. 

  10471.                     for (; q<inch; q++)

  10472.                     {

  10473.                         float32x4_t _bb2p0 = vld1q_dup_f32(bb2p0);

  10474.                         float32x4_t _ktm0 = vld1q_f32(ktm0);

  10475. 

  10476.                         _sum0123 = vmlaq_f32(_sum0123, _bb2p0, _ktm0);

  10477. 

  10478.                         bb2p0 += 1;

  10479.                         ktm0 += 4;

  10480.                     }

  10481. 

  10482.                     float sum0 = vgetq_lane_f32(_sum0123, 0);

  10483.                     float sum1 = vgetq_lane_f32(_sum0123, 1);

  10484.                     float sum2 = vgetq_lane_f32(_sum0123, 2);

  10485.                     float sum3 = vgetq_lane_f32(_sum0123, 3);

  10486. #else

  10487.                     float sum0 = 0.f;

  10488.                     float sum1 = 0.f;

  10489.                     float sum2 = 0.f;

  10490.                     float sum3 = 0.f;

  10491. 

  10492.                     for (int q=0; q<inch; q++)

  10493.                     {

  10494.                         sum0 += bb2p0[0] * ktm0[0];

  10495.                         sum1 += bb2p0[0] * ktm0[1];

  10496.                         sum2 += bb2p0[0] * ktm0[2];

  10497.                         sum3 += bb2p0[0] * ktm0[3];

  10498. 

  10499.                         bb2p0 += 1;

  10500.                         ktm0 += 4;

  10501.                     }

  10502. #endif // __ARM_NEON

  10503. 

  10504.                     output0_tm[0] = sum0;

  10505.                     output1_tm[0] = sum1;

  10506.                     output2_tm[0] = sum2;

  10507.                     output3_tm[0] = sum3;

  10508. 

  10509.                     output0_tm += 1;

  10510.                     output1_tm += 1;

  10511.                     output2_tm += 1;

  10512.                     output3_tm += 1;

  10513.                 }

  10514.             }

  10515.         }

  10516. 

  10517.         remain_outch_start += nn_outch << 2;

  10518. 

  10519.         #pragma omp parallel for num_threads(opt.num_threads)

  10520.         for (int p=remain_outch_start; p<outch; p++)

  10521.         {

  10522. #if __ARM_NEON && __aarch64__

  10523.             const Mat kernel_tm0 = kernel_tm.channel(p/8+(p%8)/4+p%4);

  10524. #else

  10525.             const Mat kernel_tm0 = kernel_tm.channel(p/4+p%4);

  10526. #endif

  10527. 

  10528.             Mat out0_tm = top_blob_tm.channel(p);

  10529. 

  10530.             float* output0_tm = out0_tm;

  10531. 

  10532.             for (int r=0; r<64; r++)

  10533.             {

  10534.                 const Mat bb2 = bottom_blob_tm2.channel(r);

  10535. 

  10536.                 // tile

  10537.                 int i=0;

  10538.                 for (; i+7<tiles; i+=8)

  10539.                 {

  10540.                     const float* bb2p0 = bb2.row(i/8);

  10541. 

  10542.                     const float* ktm0 = kernel_tm0.row(r);

  10543. #if __ARM_NEON

  10544. #if __aarch64__

  10545.                     asm volatile(

  10546.                         "eor    v8.16b, v8.16b, v8.16b     \n"

  10547.                         "eor    v9.16b, v9.16b, v9.16b     \n"

  10548. 

  10549.                         // inch loop

  10550.                         "lsr    w4, %w6, #2             \n"// w4 = nn = inch >> 2

  10551.                         "cmp    w4, #0                  \n"

  10552.                         "beq    1f                      \n"

  10553. 

  10554.                         "0:                             \n"

  10555. 

  10556.                         "prfm   pldl1keep, [%1, #512]   \n"

  10557.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64     \n"

  10558. 

  10559.                         "prfm   pldl1keep, [%2, #128]   \n"

  10560.                         "ld1    {v0.4s}, [%2], #16      \n"

  10561. 

  10562.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  10563.                         "fmla   v9.4s, v5.4s, v0.s[0]   \n"

  10564.                         "fmla   v8.4s, v6.4s, v0.s[1]   \n"

  10565.                         "fmla   v9.4s, v7.4s, v0.s[1]   \n"

  10566. 

  10567.                         "prfm   pldl1keep, [%1, #512]   \n"

  10568.                         "ld1    {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n"

  10569. 

  10570.                         "fmla   v8.4s, v12.4s, v0.s[2]  \n"

  10571.                         "fmla   v9.4s, v13.4s, v0.s[2]  \n"

  10572.                         "fmla   v8.4s, v14.4s, v0.s[3]  \n"

  10573.                         "fmla   v9.4s, v15.4s, v0.s[3]  \n"

  10574. 

  10575.                         "subs   w4, w4, #1              \n"

  10576.                         "bne    0b                      \n"

  10577. 

  10578.                         "1:                             \n"

  10579. 

  10580.                         // remain loop

  10581.                         "and    w4, %w6, #3             \n"// w4 = remain = tiles & 3;

  10582.                         "cmp    w4, #0                  \n"

  10583.                         "beq    3f                      \n"

  10584. 

  10585.                         "2:                             \n"

  10586. 

  10587.                         "prfm   pldl1keep, [%1, #256]   \n"

  10588.                         "ld1    {v4.4s, v5.4s}, [%1], #32      \n"

  10589. 

  10590.                         "prfm   pldl1keep, [%2, #32]    \n"

  10591.                         "ld1r   {v0.4s}, [%2], #4       \n"

  10592. 

  10593.                         "fmla   v8.4s, v4.4s, v0.4s     \n"

  10594.                         "fmla   v9.4s, v5.4s, v0.4s     \n"

  10595. 

  10596.                         "subs   w4, w4, #1              \n"

  10597.                         "bne    2b                      \n"

  10598. 

  10599.                         "3:                             \n"

  10600. 

  10601.                         "st1    {v8.4s, v9.4s}, [%0], #32       \n"

  10602. 

  10603.                         : "=r"(output0_tm), // %0

  10604.                           "=r"(bb2p0),      // %1

  10605.                           "=r"(ktm0)        // %2

  10606.                         : "0"(output0_tm),

  10607.                           "1"(bb2p0),

  10608.                           "2"(ktm0),

  10609.                           "r"(inch)         // %6

  10610.                         : "cc", "memory", "x4", "v0", "v4", "v5", "v6", "v7", "v8", "v9", "v12", "v13", "v14", "v15"

  10611.                     );

  10612. #else // __aarch64__

  10613.                     asm volatile(

  10614.                         "veor       q8, q8, q8          \n"

  10615.                         "veor       q9, q9, q9          \n"

  10616. 

  10617.                         // inch loop

  10618.                         "lsr        r4, %6, #2          \n"// r4 = nn = inch >> 2

  10619.                         "cmp        r4, #0              \n"

  10620.                         "beq        1f                  \n"

  10621. 

  10622.                         "0:                             \n"

  10623. 

  10624.                         "pld        [%1, #512]          \n"

  10625.                         "vldm       %1!, {d8-d15}       \n"

  10626. //                         "vld1.f32   {d8-d11}, [%1 :128]! \n"

  10627. //                         "vld1.f32   {d12-d15}, [%1 :128]! \n"

  10628. 

  10629.                         "pld        [%2, #128]          \n"

  10630.                         "vld1.f32   {d0-d1}, [%2 :128]! \n"

  10631. 

  10632.                         "vmla.f32   q8, q4, d0[0]       \n"

  10633.                         "vmla.f32   q9, q5, d0[0]       \n"

  10634.                         "vmla.f32   q8, q6, d0[1]       \n"

  10635.                         "vmla.f32   q9, q7, d0[1]       \n"

  10636. 

  10637.                         "pld        [%1, #512]          \n"

  10638.                         "vldm       %1!, {d24-d31}      \n"

  10639. //                         "vld1.f32   {d24-d27}, [%1 :128]! \n"

  10640. //                         "vld1.f32   {d28-d31}, [%1 :128]! \n"

  10641. 

  10642.                         "subs       r4, r4, #1          \n"

  10643. 

  10644.                         "vmla.f32   q8, q12, d1[0]      \n"

  10645.                         "vmla.f32   q9, q13, d1[0]      \n"

  10646.                         "vmla.f32   q8, q14, d1[1]      \n"

  10647.                         "vmla.f32   q9, q15, d1[1]      \n"

  10648. 

  10649.                         "bne        0b                  \n"

  10650. 

  10651.                         "1:                             \n"

  10652. 

  10653.                         // remain loop

  10654.                         "and        r4, %6, #3          \n"// r4 = remain = tiles & 3;

  10655.                         "cmp        r4, #0              \n"

  10656.                         "beq        3f                  \n"

  10657. 

  10658.                         "2:                             \n"

  10659. 

  10660.                         "pld        [%1, #256]          \n"

  10661.                         "vld1.f32   {d8-d11}, [%1 :128]! \n"

  10662. 

  10663.                         "pld        [%2, #32]           \n"

  10664.                         "vld1.f32   {d0[],d1[]}, [%2]!  \n"

  10665. 

  10666.                         "subs       r4, r4, #1          \n"

  10667. 

  10668.                         "vmla.f32   q8, q4, q0          \n"

  10669.                         "vmla.f32   q9, q5, q0          \n"

  10670. 

  10671.                         "bne        2b                  \n"

  10672. 

  10673.                         "3:                             \n"

  10674. 

  10675.                         "vst1.f32   {d16-d19}, [%0]!    \n"

  10676. 

  10677.                         : "=r"(output0_tm), // %0

  10678.                           "=r"(bb2p0),      // %1

  10679.                           "=r"(ktm0)        // %2

  10680.                         : "0"(output0_tm),

  10681.                           "1"(bb2p0),

  10682.                           "2"(ktm0),

  10683.                           "r"(inch)         // %6

  10684.                         : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8", "q9", "q12", "q13", "q14", "q15"

  10685.                     );

  10686. #endif // __aarch64__

  10687. #else

  10688.                     float sum0 = 0.f;

  10689.                     float sum1 = 0.f;

  10690.                     float sum2 = 0.f;

  10691.                     float sum3 = 0.f;

  10692.                     float sum4 = 0.f;

  10693.                     float sum5 = 0.f;

  10694.                     float sum6 = 0.f;

  10695.                     float sum7 = 0.f;

  10696. 

  10697.                     for (int q=0; q<inch; q++)

  10698.                     {

  10699.                         sum0 += bb2p0[0] * ktm0[0];

  10700.                         sum1 += bb2p0[1] * ktm0[0];

  10701.                         sum2 += bb2p0[2] * ktm0[0];

  10702.                         sum3 += bb2p0[3] * ktm0[0];

  10703.                         sum4 += bb2p0[4] * ktm0[0];

  10704.                         sum5 += bb2p0[5] * ktm0[0];

  10705.                         sum6 += bb2p0[6] * ktm0[0];

  10706.                         sum7 += bb2p0[7] * ktm0[0];

  10707. 

  10708.                         bb2p0 += 8;

  10709.                         ktm0 += 1;

  10710.                     }

  10711. 

  10712.                     output0_tm[0] = sum0;

  10713.                     output0_tm[1] = sum1;

  10714.                     output0_tm[2] = sum2;

  10715.                     output0_tm[3] = sum3;

  10716.                     output0_tm[4] = sum4;

  10717.                     output0_tm[5] = sum5;

  10718.                     output0_tm[6] = sum6;

  10719.                     output0_tm[7] = sum7;

  10720. 

  10721.                     output0_tm += 8;

  10722. #endif // __ARM_NEON

  10723.                 }

  10724.                 for (; i+3<tiles; i+=4)

  10725.                 {

  10726.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4);

  10727. 

  10728.                     const float* ktm0 = kernel_tm0.row(r);

  10729. #if __ARM_NEON

  10730. #if __aarch64__

  10731.                     asm volatile(

  10732.                         "eor    v8.16b, v8.16b, v8.16b     \n"

  10733. 

  10734.                         // inch loop

  10735.                         "lsr    w4, %w6, #2             \n"// w4 = nn = inch >> 2

  10736.                         "cmp    w4, #0                  \n"

  10737.                         "beq    1f                      \n"

  10738. 

  10739.                         "0:                             \n"

  10740. 

  10741.                         "prfm   pldl1keep, [%4, #512]   \n"

  10742.                         "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64     \n"

  10743. 

  10744.                         "prfm   pldl1keep, [%5, #128]   \n"

  10745.                         "ld1    {v0.4s}, [%5], #16      \n"

  10746. 

  10747.                         "fmla   v8.4s, v4.4s, v0.s[0]   \n"

  10748.                         "fmla   v8.4s, v5.4s, v0.s[1]   \n"

  10749.                         "fmla   v8.4s, v6.4s, v0.s[2]   \n"

  10750.                         "fmla   v8.4s, v7.4s, v0.s[3]   \n"

  10751. 

  10752.                         "subs   w4, w4, #1              \n"

  10753.                         "bne    0b                      \n"

  10754. 

  10755.                         "1:                             \n"

  10756. 

  10757.                         // remain loop

  10758.                         "and    w4, %w6, #3             \n"// w4 = remain = tiles & 3;

  10759.                         "cmp    w4, #0                  \n"

  10760.                         "beq    3f                      \n"

  10761. 

  10762.                         "2:                             \n"

  10763. 

  10764.                         "prfm   pldl1keep, [%4, #128]   \n"

  10765.                         "ld1    {v4.4s}, [%4], #16      \n"

  10766. 

  10767.                         "prfm   pldl1keep, [%5, #32]    \n"

  10768.                         "ld1r   {v0.4s}, [%5], #4       \n"

  10769. 

  10770.                         "fmla   v8.4s, v4.4s, v0.4s     \n"

  10771. 

  10772.                         "subs   w4, w4, #1              \n"

  10773.                         "bne    2b                      \n"

  10774. 

  10775.                         "3:                             \n"

  10776. 

  10777.                         "st1    {v8.4s}, [%0], #16      \n"

  10778. 

  10779.                         : "=r"(output0_tm), // %0

  10780.                           "=r"(bb2p0),      // %1

  10781.                           "=r"(ktm0)        // %2

  10782.                         : "0"(output0_tm),

  10783.                           "1"(bb2p0),

  10784.                           "2"(ktm0),

  10785.                           "r"(inch)         // %6

  10786.                         : "cc", "memory", "x4", "v0", "v4", "v5", "v6", "v7", "v8"

  10787.                     );

  10788. #else // __aarch64__

  10789.                     asm volatile(

  10790.                         "veor       q8, q8, q8          \n"

  10791. 

  10792.                         // inch loop

  10793.                         "lsr        r4, %6, #2          \n"// r4 = nn = inch >> 2

  10794.                         "cmp        r4, #0              \n"

  10795.                         "beq        1f                  \n"

  10796. 

  10797.                         "0:                             \n"

  10798. 

  10799.                         "pld        [%4, #512]          \n"

  10800.                         "vldm       %4!, {d8-d15}       \n"

  10801. //                         "vld1.f32   {d8-d11}, [%4 :128]! \n"

  10802. //                         "vld1.f32   {d12-d15}, [%4 :128]! \n"

  10803. 

  10804.                         "pld        [%5, #128]          \n"

  10805.                         "vld1.f32   {d0-d1}, [%5 :128]! \n"

  10806. 

  10807.                         "subs       r4, r4, #1          \n"

  10808. 

  10809.                         "vmla.f32   q8, q4, d0[0]       \n"

  10810.                         "vmla.f32   q8, q5, d0[1]       \n"

  10811.                         "vmla.f32   q8, q6, d1[0]       \n"

  10812.                         "vmla.f32   q8, q7, d1[1]       \n"

  10813. 

  10814.                         "bne        0b                  \n"

  10815. 

  10816.                         "1:                             \n"

  10817. 

  10818.                         // remain loop

  10819.                         "and        r4, %6, #3          \n"// r4 = remain = tiles & 3;

  10820.                         "cmp        r4, #0              \n"

  10821.                         "beq        3f                  \n"

  10822. 

  10823.                         "2:                             \n"

  10824. 

  10825.                         "pld        [%4, #128]          \n"

  10826.                         "vld1.f32   {d8-d9}, [%4]!      \n"

  10827. 

  10828.                         "pld        [%5, #32]           \n"

  10829.                         "vld1.f32   {d0[],d1[]}, [%5]!  \n"

  10830. 

  10831.                         "subs       r4, r4, #1          \n"

  10832. 

  10833.                         "vmla.f32   q8, q4, q0          \n"

  10834. 

  10835.                         "bne        2b                  \n"

  10836. 

  10837.                         "3:                             \n"

  10838. 

  10839.                         "vst1.f32   {d16-d17}, [%0]!    \n"

  10840. 

  10841.                         : "=r"(output0_tm), // %0

  10842.                           "=r"(bb2p0),      // %1

  10843.                           "=r"(ktm0)        // %2

  10844.                         : "0"(output0_tm),

  10845.                           "1"(bb2p0),

  10846.                           "2"(ktm0),

  10847.                           "r"(inch)         // %6

  10848.                         : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8"

  10849.                     );

  10850. #endif // __aarch64__

  10851. #else

  10852.                     float sum0 = 0.f;

  10853.                     float sum1 = 0.f;

  10854.                     float sum2 = 0.f;

  10855.                     float sum3 = 0.f;

  10856. 

  10857.                     for (int q=0; q<inch; q++)

  10858.                     {

  10859.                         sum0 += bb2p0[0] * ktm0[0];

  10860.                         sum1 += bb2p0[1] * ktm0[0];

  10861.                         sum2 += bb2p0[2] * ktm0[0];

  10862.                         sum3 += bb2p0[3] * ktm0[0];

  10863. 

  10864.                         bb2p0 += 4;

  10865.                         ktm0 += 1;

  10866.                     }

  10867. 

  10868.                     output0_tm[0] = sum0;

  10869.                     output0_tm[1] = sum1;

  10870.                     output0_tm[2] = sum2;

  10871.                     output0_tm[3] = sum3;

  10872. 

  10873.                     output0_tm += 4;

  10874. #endif // __ARM_NEON

  10875.                 }

  10876.                 for (; i<tiles; i++)

  10877.                 {

  10878.                     const float* bb2p0 = bb2.row(i/8+(i%8)/4+i%4);

  10879. 

  10880.                     const float* ktm0 = kernel_tm0.row(r);

  10881. 

  10882.                     int q=0;

  10883. #if __ARM_NEON

  10884.                     float32x4_t _sum0 = vdupq_n_f32(0.f);

  10885.                     for (; q+3<inch; q+=4)

  10886.                     {

  10887. //                         asm volatile("prfm pldl1keep, [%0, #128] \n" : :"r"(bb2p0) :);

  10888.                         float32x4_t _bb2p0 = vld1q_f32(bb2p0);

  10889.                         bb2p0 += 4;

  10890. 

  10891.                         float32x4_t _ktm0 = vld1q_f32(ktm0);

  10892.                         ktm0 += 4;

  10893. 

  10894.                         _sum0 = vmlaq_f32(_sum0, _bb2p0, _ktm0);

  10895.                     }

  10896. 

  10897. #if __aarch64__

  10898.                     float sum0 = vaddvq_f32(_sum0);

  10899. #else

  10900.                     float32x2_t _ss0 = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));

  10901.                     float sum0 = vget_lane_f32(vpadd_f32(_ss0, _ss0), 0);

  10902. #endif // __aarch64__

  10903. #else

  10904.                     float sum0 = 0.f;

  10905. #endif

  10906.                     for (; q<inch; q++)

  10907.                     {

  10908.                         sum0 += bb2p0[0] * ktm0[0];

  10909. 

  10910.                         bb2p0 += 1;

  10911.                         ktm0 += 1;

  10912.                     }

  10913. 

  10914.                     output0_tm[0] = sum0;

  10915. 

  10916.                     output0_tm += 1;

  10917.                 }

  10918.             }

  10919.         }

  10920.     }

  10921.     bottom_blob_tm = Mat();

  10922.     // END dot

  10923. 

  10924.     // BEGIN transform output

  10925.     Mat top_blob_bordered;

  10926.     if (outw == top_blob.w && outh == top_blob.h)

  10927.     {

  10928.         top_blob_bordered = top_blob;

  10929.     }

  10930.     else

  10931.     {

  10932.         top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator);

  10933.     }

  10934.     {

  10935. //         const float otm[6][8] = {

  10936. //             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},

  10937. //             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},

  10938. //             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},

  10939. //             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},

  10940. //             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},

  10941. //             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}

  10942. //         };

  10943. 

  10944.         // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32

  10945.         // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16

  10946.         // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8

  10947.         // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4

  10948.         // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2

  10949.         // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)

  10950. 

  10951. #if __ARM_NEON

  10952.         const float coeff[4] = { 4.f, 8.f, 16.f, 32.f };

  10953.         float32x4_t _coeff = vld1q_f32(coeff);

  10954. #endif // __ARM_NEON

  10955. 

  10956.         int w_tm = outw / 6 * 8;

  10957.         int h_tm = outh / 6 * 8;

  10958.         const int tiles = w_tm/8 * h_tm/8;

  10959. 

  10960.         #pragma omp parallel for num_threads(opt.num_threads)

  10961.         for (int p = 0; p<outch; p++)

  10962.         {

  10963.             const Mat out0_tm = top_blob_tm.channel(p);

  10964.             Mat out0 = top_blob_bordered.channel(p);

  10965. 

  10966.             const float bias0 = bias ? bias[p] : 0.f;

  10967. #if __ARM_NEON

  10968.             float32x2_t _bias0 = vdup_n_f32(bias0);

  10969. #endif // __ARM_NEON

  10970. 

  10971.             float tmp[6][8];

  10972. 

  10973.             // tile

  10974.             for (int i=0; i<outh/6; i++)

  10975.             {

  10976.                 for (int j=0; j<outw/6; j++)

  10977.                 {

  10978. #if __ARM_NEON

  10979. #if __aarch64__

  10980.                     const float* output0_tm0 = out0_tm.row(i * w_tm/8 + j);

  10981.                     const float* output0_tm1 = out0_tm.row(i * w_tm/8 + j + tiles*8);

  10982.                     const float* output0_tm2 = out0_tm.row(i * w_tm/8 + j + tiles*16);

  10983.                     const float* output0_tm3 = out0_tm.row(i * w_tm/8 + j + tiles*24);

  10984. 

  10985.                     for (int m=0; m+3<8; m+=4)

  10986.                     {

  10987.                         float32x4_t _output0_tm_00;

  10988.                         float32x4_t _output0_tm_11;

  10989.                         float32x4_t _output0_tm_22;

  10990.                         float32x4_t _output0_tm_33;

  10991.                         float32x4_t _output0_tm_44;

  10992.                         float32x4_t _output0_tm_55;

  10993.                         float32x4_t _output0_tm_66;

  10994.                         float32x4_t _output0_tm_77;

  10995. 

  10996.                         _output0_tm_00 = vsetq_lane_f32(output0_tm0[0], _output0_tm_00, 0);

  10997.                         output0_tm0 += out0_tm.w * tiles;

  10998.                         _output0_tm_00 = vsetq_lane_f32(output0_tm1[0], _output0_tm_00, 1);

  10999.                         output0_tm1 += out0_tm.w * tiles;

  11000.                         _output0_tm_00 = vsetq_lane_f32(output0_tm2[0], _output0_tm_00, 2);

  11001.                         output0_tm2 += out0_tm.w * tiles;

  11002.                         _output0_tm_00 = vsetq_lane_f32(output0_tm3[0], _output0_tm_00, 3);

  11003.                         output0_tm3 += out0_tm.w * tiles;

  11004. 

  11005.                         _output0_tm_11 = vsetq_lane_f32(output0_tm0[0], _output0_tm_11, 0);

  11006.                         output0_tm0 += out0_tm.w * tiles;

  11007.                         _output0_tm_11 = vsetq_lane_f32(output0_tm1[0], _output0_tm_11, 1);

  11008.                         output0_tm1 += out0_tm.w * tiles;

  11009.                         _output0_tm_11 = vsetq_lane_f32(output0_tm2[0], _output0_tm_11, 2);

  11010.                         output0_tm2 += out0_tm.w * tiles;

  11011.                         _output0_tm_11 = vsetq_lane_f32(output0_tm3[0], _output0_tm_11, 3);

  11012.                         output0_tm3 += out0_tm.w * tiles;

  11013. 

  11014.                         _output0_tm_22 = vsetq_lane_f32(output0_tm0[0], _output0_tm_22, 0);

  11015.                         output0_tm0 += out0_tm.w * tiles;

  11016.                         _output0_tm_22 = vsetq_lane_f32(output0_tm1[0], _output0_tm_22, 1);

  11017.                         output0_tm1 += out0_tm.w * tiles;

  11018.                         _output0_tm_22 = vsetq_lane_f32(output0_tm2[0], _output0_tm_22, 2);

  11019.                         output0_tm2 += out0_tm.w * tiles;

  11020.                         _output0_tm_22 = vsetq_lane_f32(output0_tm3[0], _output0_tm_22, 3);

  11021.                         output0_tm3 += out0_tm.w * tiles;

  11022. 

  11023.                         _output0_tm_33 = vsetq_lane_f32(output0_tm0[0], _output0_tm_33, 0);

  11024.                         output0_tm0 += out0_tm.w * tiles;

  11025.                         _output0_tm_33 = vsetq_lane_f32(output0_tm1[0], _output0_tm_33, 1);

  11026.                         output0_tm1 += out0_tm.w * tiles;

  11027.                         _output0_tm_33 = vsetq_lane_f32(output0_tm2[0], _output0_tm_33, 2);

  11028.                         output0_tm2 += out0_tm.w * tiles;

  11029.                         _output0_tm_33 = vsetq_lane_f32(output0_tm3[0], _output0_tm_33, 3);

  11030.                         output0_tm3 += out0_tm.w * tiles;

  11031. 

  11032.                         _output0_tm_44 = vsetq_lane_f32(output0_tm0[0], _output0_tm_44, 0);

  11033.                         output0_tm0 += out0_tm.w * tiles;

  11034.                         _output0_tm_44 = vsetq_lane_f32(output0_tm1[0], _output0_tm_44, 1);

  11035.                         output0_tm1 += out0_tm.w * tiles;

  11036.                         _output0_tm_44 = vsetq_lane_f32(output0_tm2[0], _output0_tm_44, 2);

  11037.                         output0_tm2 += out0_tm.w * tiles;

  11038.                         _output0_tm_44 = vsetq_lane_f32(output0_tm3[0], _output0_tm_44, 3);

  11039.                         output0_tm3 += out0_tm.w * tiles;

  11040. 

  11041.                         _output0_tm_55 = vsetq_lane_f32(output0_tm0[0], _output0_tm_55, 0);

  11042.                         output0_tm0 += out0_tm.w * tiles;

  11043.                         _output0_tm_55 = vsetq_lane_f32(output0_tm1[0], _output0_tm_55, 1);

  11044.                         output0_tm1 += out0_tm.w * tiles;

  11045.                         _output0_tm_55 = vsetq_lane_f32(output0_tm2[0], _output0_tm_55, 2);

  11046.                         output0_tm2 += out0_tm.w * tiles;

  11047.                         _output0_tm_55 = vsetq_lane_f32(output0_tm3[0], _output0_tm_55, 3);

  11048.                         output0_tm3 += out0_tm.w * tiles;

  11049. 

  11050.                         _output0_tm_66 = vsetq_lane_f32(output0_tm0[0], _output0_tm_66, 0);

  11051.                         output0_tm0 += out0_tm.w * tiles;

  11052.                         _output0_tm_66 = vsetq_lane_f32(output0_tm1[0], _output0_tm_66, 1);

  11053.                         output0_tm1 += out0_tm.w * tiles;

  11054.                         _output0_tm_66 = vsetq_lane_f32(output0_tm2[0], _output0_tm_66, 2);

  11055.                         output0_tm2 += out0_tm.w * tiles;

  11056.                         _output0_tm_66 = vsetq_lane_f32(output0_tm3[0], _output0_tm_66, 3);

  11057.                         output0_tm3 += out0_tm.w * tiles;

  11058. 

  11059.                         _output0_tm_77 = vsetq_lane_f32(output0_tm0[0], _output0_tm_77, 0);

  11060.                         _output0_tm_77 = vsetq_lane_f32(output0_tm1[0], _output0_tm_77, 1);

  11061.                         _output0_tm_77 = vsetq_lane_f32(output0_tm2[0], _output0_tm_77, 2);

  11062.                         _output0_tm_77 = vsetq_lane_f32(output0_tm3[0], _output0_tm_77, 3);

  11063. 

  11064.                         float32x4_t _tmp024a = vaddq_f32(_output0_tm_11, _output0_tm_22);

  11065.                         float32x4_t _tmp135a = vsubq_f32(_output0_tm_11, _output0_tm_22);

  11066. 

  11067.                         float32x4_t _tmp024b = vaddq_f32(_output0_tm_33, _output0_tm_44);

  11068.                         float32x4_t _tmp135b = vsubq_f32(_output0_tm_33, _output0_tm_44);

  11069. 

  11070.                         float32x4_t _tmp024c = vaddq_f32(_output0_tm_55, _output0_tm_66);

  11071.                         float32x4_t _tmp135c = vsubq_f32(_output0_tm_55, _output0_tm_66);

  11072. 

  11073.                         float32x4_t _tmp0 = vaddq_f32(_output0_tm_00, _tmp024a);

  11074.                         _tmp0 = vmlaq_lane_f32(_tmp0, _tmp024c, vget_high_f32(_coeff), 1);

  11075.                         _tmp0 = vaddq_f32(_tmp0, _tmp024b);

  11076. 

  11077.                         float32x4_t _tmp2 = vmlaq_lane_f32(_tmp024a, _tmp024b, vget_low_f32(_coeff), 0);

  11078.                         _tmp2 = vmlaq_lane_f32(_tmp2, _tmp024c, vget_low_f32(_coeff), 1);

  11079. 

  11080.                         float32x4_t _tmp4 = vmlaq_lane_f32(_tmp024a, _tmp024b, vget_high_f32(_coeff), 0);

  11081.                         _tmp4 = vaddq_f32(_tmp4, _tmp024c);

  11082.                         _tmp4 = vaddq_f32(_tmp4, _tmp024c);

  11083. 

  11084.                         vst1q_f32(&tmp[0][m], _tmp0);

  11085.                         vst1q_f32(&tmp[2][m], _tmp2);

  11086.                         vst1q_f32(&tmp[4][m], _tmp4);

  11087. 

  11088.                         float32x4_t _tmp1 = vmlaq_lane_f32(_tmp135a, _tmp135c, vget_high_f32(_coeff), 0);

  11089.                         _tmp1 = vaddq_f32(_tmp1, _tmp135b);

  11090.                         _tmp1 = vaddq_f32(_tmp1, _tmp135b);

  11091. 

  11092.                         float32x4_t _tmp3 = vmlaq_lane_f32(_tmp135a, _tmp135b, vget_low_f32(_coeff), 1);

  11093.                         _tmp3 = vmlaq_lane_f32(_tmp3, _tmp135c, vget_low_f32(_coeff), 0);

  11094. 

  11095.                         float32x4_t _tmp5 = vaddq_f32(_output0_tm_77, _tmp135a);

  11096.                         _tmp5 = vmlaq_lane_f32(_tmp5, _tmp135b, vget_high_f32(_coeff), 1);

  11097.                         _tmp5 = vaddq_f32(_tmp5, _tmp135c);

  11098. 

  11099.                         vst1q_f32(&tmp[1][m], _tmp1);

  11100.                         vst1q_f32(&tmp[3][m], _tmp3);

  11101.                         vst1q_f32(&tmp[5][m], _tmp5);

  11102. 

  11103.                         output0_tm0 += out0_tm.w*tiles*25;

  11104.                         output0_tm1 += out0_tm.w*tiles*25;

  11105.                         output0_tm2 += out0_tm.w*tiles*25;

  11106.                         output0_tm3 += out0_tm.w*tiles*25;

  11107.                     }

  11108. 

  11109.                     const float* t0 = tmp[0];

  11110.                     const float* t1 = tmp[1];

  11111. 

  11112.                     float* output0 = out0.row(i * 6) + j * 6;

  11113.                     float* output1 = output0 + outw;

  11114. 

  11115.                     for (int m=0; m+1<6; m+=2)

  11116.                     {

  11117.                         float32x4_t _t0_0123 = vld1q_f32(t0);

  11118.                         float32x4_t _t0_4567 = vld1q_f32(t0+4);

  11119.                         float32x4_t _t1_0123 = vld1q_f32(t1);

  11120.                         float32x4_t _t1_4567 = vld1q_f32(t1+4);

  11121. 

  11122.                         float32x4x2_t _t01_00221133 = vtrnq_f32(_t0_0123, _t1_0123);

  11123.                         float32x4x2_t _t01_44665577 = vtrnq_f32(_t0_4567, _t1_4567);

  11124. 

  11125.                         float32x2_t _t_00 = vget_low_f32(_t01_00221133.val[0]);

  11126.                         float32x2_t _t_11 = vget_low_f32(_t01_00221133.val[1]);

  11127.                         float32x2_t _t_22 = vget_high_f32(_t01_00221133.val[0]);

  11128.                         float32x2_t _t_33 = vget_high_f32(_t01_00221133.val[1]);

  11129.                         float32x2_t _t_44 = vget_low_f32(_t01_44665577.val[0]);

  11130.                         float32x2_t _t_55 = vget_low_f32(_t01_44665577.val[1]);

  11131.                         float32x2_t _t_66 = vget_high_f32(_t01_44665577.val[0]);

  11132.                         float32x2_t _t_77 = vget_high_f32(_t01_44665577.val[1]);

  11133. 

  11134.                         float32x2_t _tmp024a = vadd_f32(_t_11, _t_22);

  11135.                         float32x2_t _tmp135a = vsub_f32(_t_11, _t_22);

  11136. 

  11137.                         float32x2_t _tmp024b = vadd_f32(_t_33, _t_44);

  11138.                         float32x2_t _tmp135b = vsub_f32(_t_33, _t_44);

  11139. 

  11140.                         float32x2_t _tmp024c = vadd_f32(_t_55, _t_66);

  11141.                         float32x2_t _tmp135c = vsub_f32(_t_55, _t_66);

  11142. 

  11143.                         float32x2_t _output_0 = vadd_f32(_t_00, _tmp024a);

  11144.                         _output_0 = vmla_lane_f32(_output_0, _tmp024c, vget_high_f32(_coeff), 1);

  11145.                         _output_0 = vadd_f32(_output_0, _tmp024b);

  11146.                         _output_0 = vadd_f32(_output_0, _bias0);

  11147. 

  11148.                         float32x2_t _output_2 = vmla_lane_f32(_tmp024a, _tmp024b, vget_low_f32(_coeff), 0);

  11149.                         _output_2 = vmla_lane_f32(_output_2, _tmp024c, vget_low_f32(_coeff), 1);

  11150.                         _output_2 = vadd_f32(_output_2, _bias0);

  11151. 

  11152.                         float32x2_t _output_4 = vmla_lane_f32(_tmp024a, _tmp024b, vget_high_f32(_coeff), 0);

  11153.                         _output_4 = vadd_f32(_output_4, _tmp024c);

  11154.                         _output_4 = vadd_f32(_output_4, _tmp024c);

  11155.                         _output_4 = vadd_f32(_output_4, _bias0);

  11156. 

  11157.                         output0[0] = vget_lane_f32(_output_0, 0);

  11158.                         output1[0] = vget_lane_f32(_output_0, 1);

  11159.                         output0[2] = vget_lane_f32(_output_2, 0);

  11160.                         output1[2] = vget_lane_f32(_output_2, 1);

  11161.                         output0[4] = vget_lane_f32(_output_4, 0);

  11162.                         output1[4] = vget_lane_f32(_output_4, 1);

  11163. 

  11164.                         float32x2_t _output_1 = vmla_lane_f32(_tmp135a, _tmp135c, vget_high_f32(_coeff), 0);

  11165.                         _output_1 = vadd_f32(_output_1, _tmp135b);

  11166.                         _output_1 = vadd_f32(_output_1, _tmp135b);

  11167.                         _output_1 = vadd_f32(_output_1, _bias0);

  11168. 

  11169.                         float32x2_t _output_3 = vmla_lane_f32(_tmp135a, _tmp135b, vget_low_f32(_coeff), 1);

  11170.                         _output_3 = vmla_lane_f32(_output_3, _tmp135c, vget_low_f32(_coeff), 0);

  11171.                         _output_3 = vadd_f32(_output_3, _bias0);

  11172. 

  11173.                         float32x2_t _output_5 = vadd_f32(_t_77, _tmp135a);

  11174.                         _output_5 = vmla_lane_f32(_output_5, _tmp135b, vget_high_f32(_coeff), 1);

  11175.                         _output_5 = vadd_f32(_output_5, _tmp135c);

  11176.                         _output_5 = vadd_f32(_output_5, _bias0);

  11177. 

  11178.                         output0[1] = vget_lane_f32(_output_1, 0);

  11179.                         output1[1] = vget_lane_f32(_output_1, 1);

  11180.                         output0[3] = vget_lane_f32(_output_3, 0);

  11181.                         output1[3] = vget_lane_f32(_output_3, 1);

  11182.                         output0[5] = vget_lane_f32(_output_5, 0);

  11183.                         output1[5] = vget_lane_f32(_output_5, 1);

  11184. 

  11185.                         t0 += 8*2;

  11186.                         t1 += 8*2;

  11187.                         output0 += outw*2;

  11188.                         output1 += outw*2;

  11189.                     }

  11190. #else // __aarch64__

  11191.                     const float* output0_tm0_0 = out0_tm.row(i * w_tm/8 + j);

  11192.                     const float* output0_tm1_0 = out0_tm.row(i * w_tm/8 + j + tiles*8);

  11193.                     const float* output0_tm2_0 = out0_tm.row(i * w_tm/8 + j + tiles*16);

  11194.                     const float* output0_tm3_0 = out0_tm.row(i * w_tm/8 + j + tiles*24);

  11195.                     const float* output0_tm0_4 = out0_tm.row(i * w_tm/8 + j + tiles*32);

  11196.                     const float* output0_tm1_4 = out0_tm.row(i * w_tm/8 + j + tiles*40);

  11197.                     const float* output0_tm2_4 = out0_tm.row(i * w_tm/8 + j + tiles*48);

  11198.                     const float* output0_tm3_4 = out0_tm.row(i * w_tm/8 + j + tiles*56);

  11199. 

  11200.                     float* t0 = tmp[0];

  11201.                     float* t1 = tmp[1];

  11202. 

  11203. //                     int step = out0_tm.w * tiles * 2*4 *4;

  11204.                     int step = out0_tm.w * tiles *4;

  11205. 

  11206.                     asm volatile(

  11207. 

  11208.                         // loop0

  11209. //                         "vld1.f32   {d16-d17}, [%2], %21 \n"

  11210. //                         "vld1.f32   {d18-d19}, [%3], %21 \n"

  11211. //                         "vld1.f32   {d20-d21}, [%4], %21 \n"

  11212. //                         "vld1.f32   {d22-d23}, [%5], %21 \n"

  11213. //                         "vld1.f32   {d24-d25}, [%6], %21 \n"

  11214. //                         "vld1.f32   {d26-d27}, [%7], %21 \n"

  11215. //                         "vld1.f32   {d28-d29}, [%8], %21 \n"

  11216. //                         "vld1.f32   {d30-d31}, [%9], %21 \n"

  11217. 

  11218. //                         "vtrn.32    q8, q10             \n"

  11219. //                         "vtrn.32    q9, q11             \n"

  11220. //                         "vtrn.32    q12, q14            \n"

  11221. //                         "vtrn.32    q13, q15            \n"

  11222. 

  11223. //                         "vswp       d17, d24            \n"

  11224. //                         "vswp       d19, d26            \n"

  11225. //                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  11226. //                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  11227.                         "vld1.f32   {d16[0]}, [%2], %21 \n"

  11228.                         "vld1.f32   {d16[1]}, [%3], %21 \n"

  11229.                         "vld1.f32   {d17[0]}, [%4], %21 \n"

  11230.                         "vld1.f32   {d17[1]}, [%5], %21 \n"

  11231. 

  11232.                         "vld1.f32   {d20[0]}, [%2], %21 \n"

  11233.                         "vld1.f32   {d20[1]}, [%3], %21 \n"

  11234.                         "vld1.f32   {d21[0]}, [%4], %21 \n"

  11235.                         "vld1.f32   {d21[1]}, [%5], %21 \n"

  11236. 

  11237.                         "vld1.f32   {d24[0]}, [%2], %21 \n"

  11238.                         "vld1.f32   {d24[1]}, [%3], %21 \n"

  11239.                         "vld1.f32   {d25[0]}, [%4], %21 \n"

  11240.                         "vld1.f32   {d25[1]}, [%5], %21 \n"

  11241. 

  11242.                         "vadd.f32   q2, q10, q12        \n"

  11243.                         "vsub.f32   q3, q10, q12        \n"

  11244. 

  11245.                         "vld1.f32   {d28[0]}, [%2], %21 \n"

  11246.                         "vld1.f32   {d28[1]}, [%3], %21 \n"

  11247.                         "vld1.f32   {d29[0]}, [%4], %21 \n"

  11248.                         "vld1.f32   {d29[1]}, [%5], %21 \n"

  11249. 

  11250.                         "vld1.f32   {d18[0]}, [%2], %21 \n"

  11251.                         "vld1.f32   {d18[1]}, [%3], %21 \n"

  11252.                         "vld1.f32   {d19[0]}, [%4], %21 \n"

  11253.                         "vld1.f32   {d19[1]}, [%5], %21 \n"

  11254. 

  11255.                         "vadd.f32   q4, q14, q9         \n"

  11256.                         "vsub.f32   q5, q14, q9         \n"

  11257. 

  11258.                         "vld1.f32   {d22[0]}, [%2], %21 \n"

  11259.                         "vld1.f32   {d22[1]}, [%3], %21 \n"

  11260.                         "vld1.f32   {d23[0]}, [%4], %21 \n"

  11261.                         "vld1.f32   {d23[1]}, [%5], %21 \n"

  11262. 

  11263.                         "vld1.f32   {d26[0]}, [%2], %21 \n"

  11264.                         "vld1.f32   {d26[1]}, [%3], %21 \n"

  11265.                         "vld1.f32   {d27[0]}, [%4], %21 \n"

  11266.                         "vld1.f32   {d27[1]}, [%5], %21 \n"

  11267. 

  11268.                         "vadd.f32   q6, q11, q13        \n"

  11269.                         "vsub.f32   q7, q11, q13        \n"// spare q9 q10 q11 q12 q13 q14

  11270. 

  11271.                         "vld1.f32   {d30[0]}, [%2]      \n"

  11272.                         "vld1.f32   {d30[1]}, [%3]      \n"

  11273.                         "vld1.f32   {d31[0]}, [%4]      \n"

  11274.                         "vld1.f32   {d31[1]}, [%5]      \n"

  11275. 

  11276.                         "vmov       q9, q3              \n"

  11277.                         "vadd.f32   q8, q8, q2          \n"

  11278.                         "vmla.f32   q9, q7, %f20[0]     \n"

  11279.                         "vmov       q12, q2             \n"

  11280.                         "vmov       q10, q2             \n"

  11281.                         "vmov       q11, q3             \n"

  11282.                         "vmla.f32   q12, q4, %f20[0]    \n"

  11283.                         "vadd.f32   q15, q15, q3        \n"

  11284.                         "vmla.f32   q8, q6, %f20[1]     \n"

  11285.                         "vadd.f32   q9, q9, q5          \n"

  11286.                         "vmla.f32   q10, q4, %e20[0]    \n"

  11287.                         "vmla.f32   q11, q5, %e20[1]    \n"

  11288.                         "vadd.f32   q12, q12, q6        \n"

  11289.                         "vmla.f32   q15, q5, %f20[1]    \n"

  11290.                         "vadd.f32   q8, q8, q4          \n"

  11291.                         "vadd.f32   q9, q9, q5          \n"

  11292.                         "vmla.f32   q10, q6, %e20[1]    \n"

  11293.                         "vmla.f32   q11, q7, %e20[0]    \n"

  11294.                         "vadd.f32   q12, q12, q6        \n"

  11295.                         "vadd.f32   q15, q15, q7        \n"

  11296. 

  11297.                         "vst1.f32   {d16-d17}, [%0]     \n"

  11298.                         "add        %0, %0, #64         \n"

  11299. 

  11300.                         "vst1.f32   {d18-d19}, [%1]     \n"

  11301.                         "add        %1, %1, #64         \n"

  11302. 

  11303.                         "vst1.f32   {d20-d21}, [%0]     \n"

  11304.                         "add        %0, %0, #64         \n"

  11305. 

  11306.                         "vst1.f32   {d22-d23}, [%1]     \n"

  11307.                         "add        %1, %1, #64         \n"

  11308. 

  11309.                         "vst1.f32   {d24-d25}, [%0]     \n"

  11310.                         "sub        %0, %0, #112        \n"

  11311. 

  11312.                         "vst1.f32   {d30-d31}, [%1]     \n"

  11313.                         "sub        %1, %1, #112        \n"

  11314. 

  11315.                         // loop1

  11316. //                         "vld1.f32   {d16-d17}, [%2]     \n"

  11317. //                         "vld1.f32   {d18-d19}, [%3]     \n"

  11318. //                         "vld1.f32   {d20-d21}, [%4]     \n"

  11319. //                         "vld1.f32   {d22-d23}, [%5]     \n"

  11320. //                         "vld1.f32   {d24-d25}, [%6]     \n"

  11321. //                         "vld1.f32   {d26-d27}, [%7]     \n"

  11322. //                         "vld1.f32   {d28-d29}, [%8]     \n"

  11323. //                         "vld1.f32   {d30-d31}, [%9]     \n"

  11324. 

  11325. //                         "vtrn.32    q8, q10             \n"

  11326. //                         "vtrn.32    q9, q11             \n"

  11327. //                         "vtrn.32    q12, q14            \n"

  11328. //                         "vtrn.32    q13, q15            \n"

  11329. 

  11330. //                         "vswp       d17, d24            \n"

  11331. //                         "vswp       d19, d26            \n"

  11332. //                         "vswp       d21, d28            \n"//  q8 = 00   q9 = 44  q10 = 11  q11 = 55

  11333. //                         "vswp       d23, d30            \n"// q12 = 22  q13 = 66  q14 = 33  q15 = 77

  11334.                         "vld1.f32   {d16[0]}, [%6], %21 \n"

  11335.                         "vld1.f32   {d16[1]}, [%7], %21 \n"

  11336.                         "vld1.f32   {d17[0]}, [%8], %21 \n"

  11337.                         "vld1.f32   {d17[1]}, [%9], %21 \n"

  11338. 

  11339.                         "vld1.f32   {d20[0]}, [%6], %21 \n"

  11340.                         "vld1.f32   {d20[1]}, [%7], %21 \n"

  11341.                         "vld1.f32   {d21[0]}, [%8], %21 \n"

  11342.                         "vld1.f32   {d21[1]}, [%9], %21 \n"

  11343. 

  11344.                         "vld1.f32   {d24[0]}, [%6], %21 \n"

  11345.                         "vld1.f32   {d24[1]}, [%7], %21 \n"

  11346.                         "vld1.f32   {d25[0]}, [%8], %21 \n"

  11347.                         "vld1.f32   {d25[1]}, [%9], %21 \n"

  11348. 

  11349.                         "vadd.f32   q2, q10, q12        \n"

  11350.                         "vsub.f32   q3, q10, q12        \n"

  11351. 

  11352.                         "vld1.f32   {d28[0]}, [%6], %21 \n"

  11353.                         "vld1.f32   {d28[1]}, [%7], %21 \n"

  11354.                         "vld1.f32   {d29[0]}, [%8], %21 \n"

  11355.                         "vld1.f32   {d29[1]}, [%9], %21 \n"

  11356. 

  11357.                         "vld1.f32   {d18[0]}, [%6], %21 \n"

  11358.                         "vld1.f32   {d18[1]}, [%7], %21 \n"

  11359.                         "vld1.f32   {d19[0]}, [%8], %21 \n"

  11360.                         "vld1.f32   {d19[1]}, [%9], %21 \n"

  11361. 

  11362.                         "vadd.f32   q4, q14, q9         \n"

  11363.                         "vsub.f32   q5, q14, q9         \n"

  11364. 

  11365.                         "vld1.f32   {d22[0]}, [%6], %21 \n"

  11366.                         "vld1.f32   {d22[1]}, [%7], %21 \n"

  11367.                         "vld1.f32   {d23[0]}, [%8], %21 \n"

  11368.                         "vld1.f32   {d23[1]}, [%9], %21 \n"

  11369. 

  11370.                         "vld1.f32   {d26[0]}, [%6], %21 \n"

  11371.                         "vld1.f32   {d26[1]}, [%7], %21 \n"

  11372.                         "vld1.f32   {d27[0]}, [%8], %21 \n"

  11373.                         "vld1.f32   {d27[1]}, [%9], %21 \n"

  11374. 

  11375.                         "vadd.f32   q6, q11, q13        \n"

  11376.                         "vsub.f32   q7, q11, q13        \n"// spare q9 q10 q11 q12 q13 q14

  11377. 

  11378.                         "vld1.f32   {d30[0]}, [%6]      \n"

  11379.                         "vld1.f32   {d30[1]}, [%7]      \n"

  11380.                         "vld1.f32   {d31[0]}, [%8]      \n"

  11381.                         "vld1.f32   {d31[1]}, [%9]      \n"

  11382. 

  11383.                         "vmov       q9, q3              \n"

  11384.                         "vadd.f32   q8, q8, q2          \n"

  11385.                         "vmla.f32   q9, q7, %f20[0]     \n"

  11386.                         "vmov       q12, q2             \n"

  11387.                         "vmov       q10, q2             \n"

  11388.                         "vmov       q11, q3             \n"

  11389.                         "vmla.f32   q12, q4, %f20[0]    \n"

  11390.                         "vadd.f32   q15, q15, q3        \n"

  11391.                         "vmla.f32   q8, q6, %f20[1]     \n"

  11392.                         "vadd.f32   q9, q9, q5          \n"

  11393.                         "vmla.f32   q10, q4, %e20[0]    \n"

  11394.                         "vmla.f32   q11, q5, %e20[1]    \n"

  11395.                         "vadd.f32   q12, q12, q6        \n"

  11396.                         "vmla.f32   q15, q5, %f20[1]    \n"

  11397.                         "vadd.f32   q8, q8, q4          \n"

  11398.                         "vadd.f32   q9, q9, q5          \n"

  11399.                         "vmla.f32   q10, q6, %e20[1]    \n"

  11400.                         "vmla.f32   q11, q7, %e20[0]    \n"

  11401.                         "vadd.f32   q12, q12, q6        \n"

  11402.                         "vadd.f32   q15, q15, q7        \n"

  11403. 

  11404.                         "vst1.f32   {d16-d17}, [%0]     \n"

  11405.                         "add        %0, %0, #64         \n"

  11406. 

  11407.                         "vst1.f32   {d18-d19}, [%1]     \n"

  11408.                         "add        %1, %1, #64         \n"

  11409. 

  11410.                         "vst1.f32   {d20-d21}, [%0]     \n"

  11411.                         "add        %0, %0, #64         \n"

  11412. 

  11413.                         "vst1.f32   {d22-d23}, [%1]     \n"

  11414.                         "add        %1, %1, #64         \n"

  11415. 

  11416.                         "vst1.f32   {d24-d25}, [%0]     \n"

  11417. 

  11418.                         "vst1.f32   {d30-d31}, [%1]     \n"

  11419. 

  11420.                         : "=r"(t0),             // %0

  11421.                           "=r"(t1),             // %1

  11422.                           "=r"(output0_tm0_0),  // %2

  11423.                           "=r"(output0_tm1_0),  // %3

  11424.                           "=r"(output0_tm2_0),  // %4

  11425.                           "=r"(output0_tm3_0),  // %5

  11426.                           "=r"(output0_tm0_4),  // %6

  11427.                           "=r"(output0_tm1_4),  // %7

  11428.                           "=r"(output0_tm2_4),  // %8

  11429.                           "=r"(output0_tm3_4)   // %9

  11430.                         : "0"(t0),

  11431.                           "1"(t1),

  11432.                           "2"(output0_tm0_0),

  11433.                           "3"(output0_tm1_0),

  11434.                           "4"(output0_tm2_0),

  11435.                           "5"(output0_tm3_0),

  11436.                           "6"(output0_tm0_4),

  11437.                           "7"(output0_tm1_4),

  11438.                           "8"(output0_tm2_4),

  11439.                           "9"(output0_tm3_4),

  11440.                           "w"(_coeff),          // %20

  11441.                           "r"(step)             // %21

  11442.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  11443.                     );

  11444. 

  11445.                     t0 = tmp[0];

  11446.                     t1 = tmp[1];

  11447. 

  11448.                     float* output0 = out0.row(i * 6) + j * 6;

  11449.                     float* output1 = output0 + outw;

  11450. 

  11451.                     int stepw = outw*2 * 4;

  11452. 

  11453.                     asm volatile(

  11454. 

  11455.                         // loop0

  11456.                         "vld1.f32   {d16-d19}, [%2]     \n"

  11457.                         "vld1.f32   {d20-d23}, [%3]     \n"

  11458. 

  11459.                         "add        %2, %2, #64         \n"

  11460.                         "add        %3, %3, #64         \n"

  11461. 

  11462.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  11463.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  11464. 

  11465.                         "vadd.f32   d4, d20, d17        \n"

  11466.                         "vsub.f32   d5, d20, d17        \n"

  11467. 

  11468.                         "vadd.f32   d6, d21, d18        \n"

  11469.                         "vsub.f32   d7, d21, d18        \n"

  11470. 

  11471.                         "vadd.f32   d8, d22, d19        \n"

  11472.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  11473. 

  11474.                         "vmov       d20, d5             \n"

  11475.                         "vmov       d18, d4             \n"

  11476. 

  11477.                         "vadd.f32   d16, d16, d4        \n"

  11478.                         "vmla.f32   d20, d9, %f8[0]     \n"

  11479.                         "vmov       d17, d4             \n"

  11480.                         "vmov       d21, d5             \n"

  11481.                         "vmla.f32   d18, d6, %f8[0]     \n"

  11482.                         "vadd.f32   d22, d23, d5        \n"

  11483. 

  11484.                         "vmla.f32   d16, d8, %f8[1]     \n"

  11485.                         "vadd.f32   d20, d20, d7        \n"

  11486.                         "vmla.f32   d17, d6, %e8[0]     \n"

  11487.                         "vmla.f32   d21, d7, %e8[1]     \n"

  11488.                         "vadd.f32   d18, d18, d8        \n"

  11489.                         "vmla.f32   d22, d7, %f8[1]     \n"

  11490. 

  11491.                         "vadd.f32   d16, d16, d6        \n"

  11492.                         "vadd.f32   d20, d20, d7        \n"

  11493.                         "vmla.f32   d17, d8, %e8[1]     \n"

  11494.                         "vmla.f32   d21, d9, %e8[0]     \n"

  11495.                         "vadd.f32   d18, d18, d8        \n"

  11496.                         "vadd.f32   d22, d22, d9        \n"

  11497. 

  11498.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  11499.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  11500.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  11501.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  11502.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  11503.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  11504. 

  11505.                         "vtrn.f32   q8, q10             \n"

  11506.                         "vtrn.f32   d18, d22            \n"

  11507. 

  11508.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  11509.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  11510. 

  11511.                         // loop1

  11512.                         "vld1.f32   {d16-d19}, [%2]     \n"

  11513.                         "vld1.f32   {d20-d23}, [%3]     \n"

  11514. 

  11515.                         "add        %2, %2, #64         \n"

  11516.                         "add        %3, %3, #64         \n"

  11517. 

  11518.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  11519.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  11520. 

  11521.                         "vadd.f32   d4, d20, d17        \n"

  11522.                         "vsub.f32   d5, d20, d17        \n"

  11523. 

  11524.                         "vadd.f32   d6, d21, d18        \n"

  11525.                         "vsub.f32   d7, d21, d18        \n"

  11526. 

  11527.                         "vadd.f32   d8, d22, d19        \n"

  11528.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  11529. 

  11530.                         "vmov       d20, d5             \n"

  11531.                         "vmov       d18, d4             \n"

  11532. 

  11533.                         "vadd.f32   d16, d16, d4        \n"

  11534.                         "vmla.f32   d20, d9, %f8[0]     \n"

  11535.                         "vmov       d17, d4             \n"

  11536.                         "vmov       d21, d5             \n"

  11537.                         "vmla.f32   d18, d6, %f8[0]     \n"

  11538.                         "vadd.f32   d22, d23, d5        \n"

  11539. 

  11540.                         "vmla.f32   d16, d8, %f8[1]     \n"

  11541.                         "vadd.f32   d20, d20, d7        \n"

  11542.                         "vmla.f32   d17, d6, %e8[0]     \n"

  11543.                         "vmla.f32   d21, d7, %e8[1]     \n"

  11544.                         "vadd.f32   d18, d18, d8        \n"

  11545.                         "vmla.f32   d22, d7, %f8[1]     \n"

  11546. 

  11547.                         "vadd.f32   d16, d16, d6        \n"

  11548.                         "vadd.f32   d20, d20, d7        \n"

  11549.                         "vmla.f32   d17, d8, %e8[1]     \n"

  11550.                         "vmla.f32   d21, d9, %e8[0]     \n"

  11551.                         "vadd.f32   d18, d18, d8        \n"

  11552.                         "vadd.f32   d22, d22, d9        \n"

  11553. 

  11554.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  11555.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  11556.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  11557.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  11558.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  11559.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  11560. 

  11561.                         "vtrn.f32   q8, q10             \n"

  11562.                         "vtrn.f32   d18, d22            \n"

  11563. 

  11564.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  11565.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  11566. 

  11567.                         // loop2

  11568.                         "vld1.f32   {d16-d19}, [%2]     \n"

  11569.                         "vld1.f32   {d20-d23}, [%3]     \n"

  11570. 

  11571.                         "add        %2, %2, #64         \n"

  11572.                         "add        %3, %3, #64         \n"

  11573. 

  11574.                         "vtrn.32    q8, q10             \n"// q8 = 0 2  q10 = 1 3

  11575.                         "vtrn.32    q9, q11             \n"// q9 = 4 6  q11 = 5 7

  11576. 

  11577.                         "vadd.f32   d4, d20, d17        \n"

  11578.                         "vsub.f32   d5, d20, d17        \n"

  11579. 

  11580.                         "vadd.f32   d6, d21, d18        \n"

  11581.                         "vsub.f32   d7, d21, d18        \n"

  11582. 

  11583.                         "vadd.f32   d8, d22, d19        \n"

  11584.                         "vsub.f32   d9, d22, d19        \n"// spare d17 ~ d22

  11585. 

  11586.                         "vmov       d20, d5             \n"

  11587.                         "vmov       d18, d4             \n"

  11588. 

  11589.                         "vadd.f32   d16, d16, d4        \n"

  11590.                         "vmla.f32   d20, d9, %f8[0]     \n"

  11591.                         "vmov       d17, d4             \n"

  11592.                         "vmov       d21, d5             \n"

  11593.                         "vmla.f32   d18, d6, %f8[0]     \n"

  11594.                         "vadd.f32   d22, d23, d5        \n"

  11595. 

  11596.                         "vmla.f32   d16, d8, %f8[1]     \n"

  11597.                         "vadd.f32   d20, d20, d7        \n"

  11598.                         "vmla.f32   d17, d6, %e8[0]     \n"

  11599.                         "vmla.f32   d21, d7, %e8[1]     \n"

  11600.                         "vadd.f32   d18, d18, d8        \n"

  11601.                         "vmla.f32   d22, d7, %f8[1]     \n"

  11602. 

  11603.                         "vadd.f32   d16, d16, d6        \n"

  11604.                         "vadd.f32   d20, d20, d7        \n"

  11605.                         "vmla.f32   d17, d8, %e8[1]     \n"

  11606.                         "vmla.f32   d21, d9, %e8[0]     \n"

  11607.                         "vadd.f32   d18, d18, d8        \n"

  11608.                         "vadd.f32   d22, d22, d9        \n"

  11609. 

  11610.                         "vadd.f32   d16, d16, %P9       \n"// _bias0

  11611.                         "vadd.f32   d20, d20, %P9       \n"// _bias0

  11612.                         "vadd.f32   d17, d17, %P9       \n"// _bias0

  11613.                         "vadd.f32   d21, d21, %P9       \n"// _bias0

  11614.                         "vadd.f32   d18, d18, %P9       \n"// _bias0

  11615.                         "vadd.f32   d22, d22, %P9       \n"// _bias0

  11616. 

  11617.                         "vtrn.f32   q8, q10             \n"

  11618.                         "vtrn.f32   d18, d22            \n"

  11619. 

  11620.                         "vst1.f32   {d16-d18}, [%0], %10 \n"

  11621.                         "vst1.f32   {d20-d22}, [%1], %10 \n"

  11622. 

  11623.                         : "=r"(output0),    // %0

  11624.                           "=r"(output1),    // %1

  11625.                           "=r"(t0),         // %2

  11626.                           "=r"(t1)          // %3

  11627.                         : "0"(output0),

  11628.                           "1"(output1),

  11629.                           "2"(t0),

  11630.                           "3"(t1),

  11631.                           "w"(_coeff),      // %8

  11632.                           "w"(_bias0),      // %9

  11633.                           "r"(stepw)        // %10

  11634.                         : "memory", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  11635.                     );

  11636. #endif // __aarch64__

  11637. #else

  11638.                     const float* output0_tm_0 = out0_tm.row(i * w_tm/8 + j);

  11639.                     const float* output0_tm_1 = out0_tm.row(i * w_tm/8 + j + tiles);

  11640.                     const float* output0_tm_2 = out0_tm.row(i * w_tm/8 + j + tiles*2);

  11641.                     const float* output0_tm_3 = out0_tm.row(i * w_tm/8 + j + tiles*3);

  11642.                     const float* output0_tm_4 = out0_tm.row(i * w_tm/8 + j + tiles*4);

  11643.                     const float* output0_tm_5 = out0_tm.row(i * w_tm/8 + j + tiles*5);

  11644.                     const float* output0_tm_6 = out0_tm.row(i * w_tm/8 + j + tiles*6);

  11645.                     const float* output0_tm_7 = out0_tm.row(i * w_tm/8 + j + tiles*7);

  11646. 

  11647.                     for (int m=0; m<8; m++)

  11648.                     {

  11649.                         float tmp024a = output0_tm_1[0] + output0_tm_2[0];

  11650.                         float tmp135a = output0_tm_1[0] - output0_tm_2[0];

  11651. 

  11652.                         float tmp024b = output0_tm_3[0] + output0_tm_4[0];

  11653.                         float tmp135b = output0_tm_3[0] - output0_tm_4[0];

  11654. 

  11655.                         float tmp024c = output0_tm_5[0] + output0_tm_6[0];

  11656.                         float tmp135c = output0_tm_5[0] - output0_tm_6[0];

  11657. 

  11658.                         tmp[0][m] = output0_tm_0[0] + tmp024a + tmp024b + tmp024c * 32;

  11659.                         tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;

  11660.                         tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  11661. 

  11662.                         tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;

  11663.                         tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;

  11664.                         tmp[5][m] = output0_tm_7[0] + tmp135a + tmp135b * 32 + tmp135c;

  11665. 

  11666.                         output0_tm_0 += out0_tm.w * tiles * 8;

  11667.                         output0_tm_1 += out0_tm.w * tiles * 8;

  11668.                         output0_tm_2 += out0_tm.w * tiles * 8;

  11669.                         output0_tm_3 += out0_tm.w * tiles * 8;

  11670.                         output0_tm_4 += out0_tm.w * tiles * 8;

  11671.                         output0_tm_5 += out0_tm.w * tiles * 8;

  11672.                         output0_tm_6 += out0_tm.w * tiles * 8;

  11673.                         output0_tm_7 += out0_tm.w * tiles * 8;

  11674.                     }

  11675. 

  11676.                     float* output0 = out0.row(i * 6) + j * 6;

  11677. 

  11678.                     for (int m=0; m<6; m++)

  11679.                     {

  11680.                         const float* tmp0 = tmp[m];

  11681. 

  11682.                         float tmp024a = tmp0[1] + tmp0[2];

  11683.                         float tmp135a = tmp0[1] - tmp0[2];

  11684. 

  11685.                         float tmp024b = tmp0[3] + tmp0[4];

  11686.                         float tmp135b = tmp0[3] - tmp0[4];

  11687. 

  11688.                         float tmp024c = tmp0[5] + tmp0[6];

  11689.                         float tmp135c = tmp0[5] - tmp0[6];

  11690. 

  11691.                         output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;

  11692.                         output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;

  11693.                         output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;

  11694. 

  11695.                         output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;

  11696.                         output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;

  11697.                         output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;

  11698. 

  11699.                         output0 += outw;

  11700.                     }

  11701. #endif // __ARM_NEON

  11702.                 }

  11703.             }

  11704.         }

  11705.     }

  11706.     // END transform output

  11707. 

  11708.     // cut result pad

  11709.     if (top_blob_bordered.w != top_blob.w || top_blob_bordered.h != top_blob.h)

  11710.         copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt.blob_allocator, opt.num_threads);

  11711. }

  11712. 

  11713. static void conv3x3s2_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  11714. {

  11715.     int w = bottom_blob.w;

  11716.     int inch = bottom_blob.c;

  11717. 

  11718.     int outw = top_blob.w;

  11719.     int outh = top_blob.h;

  11720.     int outch = top_blob.c;

  11721. 

  11722.     const int tailstep = w - 2*outw + w;

  11723. 

  11724.     const float* kernel = _kernel;

  11725.     const float* bias = _bias;

  11726. 

  11727.     int nn_outch = outch >> 1;

  11728.     int remain_outch_start = nn_outch << 1;

  11729. 

  11730.     #pragma omp parallel for num_threads(opt.num_threads)

  11731.     for (int pp=0; pp<nn_outch; pp++)

  11732.     {

  11733.         int p = pp * 2;

  11734. 

  11735.         Mat out0 = top_blob.channel(p);

  11736.         Mat out1 = top_blob.channel(p+1);

  11737. 

  11738.         const float bias0 = bias ? bias[p] : 0.f;

  11739.         const float bias1 = bias ? bias[p+1] : 0.f;

  11740. 

  11741.         out0.fill(bias0);

  11742.         out1.fill(bias1);

  11743. 

  11744.         const float* k0 = kernel + p*inch*9;

  11745.         const float* k1 = kernel + (p+1)*inch*9;

  11746. 

  11747.         for (int q=0; q<inch; q++)

  11748.         {

  11749.             float* outptr0 = out0;

  11750.             float* outptr1 = out1;

  11751. 

  11752.             const float* img0 = bottom_blob.channel(q);

  11753. 

  11754.             const float* r0 = img0;

  11755.             const float* r1 = img0 + w;

  11756.             const float* r2 = img0 + w*2;

  11757. 

  11758. #if __ARM_NEON

  11759.             float32x4_t _k00 = vld1q_f32(k0);

  11760.             float32x4_t _k03 = vld1q_f32(k0+3);

  11761.             float32x4_t _k06 = vld1q_f32(k0+6);

  11762. 

  11763.             float32x4_t _k10 = vld1q_f32(k1);

  11764.             float32x4_t _k13 = vld1q_f32(k1+3);

  11765.             float32x4_t _k16 = vld1q_f32(k1+6);

  11766. #endif // __ARM_NEON

  11767. 

  11768.             int i = 0;

  11769. 

  11770.             for (; i < outh; i++)

  11771.             {

  11772. #if __ARM_NEON

  11773.                 int nn = outw >> 2;

  11774.                 int remain = outw & 3;

  11775. #else

  11776.                 int remain = outw;

  11777. #endif // __ARM_NEON

  11778. 

  11779. #if __ARM_NEON

  11780. #if __aarch64__

  11781.                 if (nn > 0)

  11782.                 {

  11783.                 asm volatile(

  11784.                     "prfm   pldl1keep, [%3, #256]       \n"

  11785.                     "ld2    {v8.4s, v9.4s}, [%3], #32   \n"// v8 v9 = r0

  11786. 

  11787.                     "0:                                 \n"

  11788. 

  11789.                     "prfm   pldl1keep, [%1, #128]       \n"

  11790.                     "ld1    {v6.4s}, [%1]               \n"// v6 = _sum0

  11791. 

  11792.                     "fmul   v12.4s, v8.4s, %12.s[0]     \n"

  11793. 

  11794.                     "prfm   pldl1keep, [%2, #128]       \n"

  11795.                     "ld1    {v7.4s}, [%2]               \n"// v7 = _sum1

  11796. 

  11797.                     "fmul   v13.4s, v8.4s, %15.s[0]     \n"

  11798. 

  11799.                     "prfm   pldl1keep, [%3, #128]       \n"

  11800.                     "ld2    {v10.4s, v11.4s}, [%3]      \n"// v10

  11801. 

  11802.                     "fmla   v6.4s, v9.4s, %12.s[1]      \n"

  11803. 

  11804.                     "ext    v14.16b, v8.16b, v10.16b, #4\n"

  11805. 

  11806.                     "fmla   v7.4s, v9.4s, %15.s[1]      \n"

  11807. 

  11808.                     "prfm   pldl1keep, [%4, #256]       \n"

  11809.                     "ld2    {v8.4s, v9.4s}, [%4], #32   \n"// r1

  11810. 

  11811.                     "fmla   v12.4s, v14.4s, %12.s[2]    \n"

  11812.                     "fmla   v13.4s, v14.4s, %15.s[2]    \n"

  11813. 

  11814.                     "prfm   pldl1keep, [%4, #128]       \n"

  11815.                     "ld2    {v10.4s, v11.4s}, [%4]      \n"

  11816. 

  11817.                     "fmla   v6.4s, v8.4s, %13.s[0]      \n"

  11818.                     "fmla   v7.4s, v8.4s, %16.s[0]      \n"

  11819. 

  11820.                     "ext    v14.16b, v8.16b, v10.16b, #4\n"

  11821. 

  11822.                     "fmla   v12.4s, v9.4s, %13.s[1]     \n"

  11823.                     "fmla   v13.4s, v9.4s, %16.s[1]     \n"

  11824. 

  11825.                     "prfm   pldl1keep, [%5, #256]       \n"

  11826.                     "ld2    {v8.4s, v9.4s}, [%5], #32   \n"// r2

  11827. 

  11828.                     "fmla   v6.4s, v14.4s, %13.s[2]     \n"

  11829.                     "fmla   v7.4s, v14.4s, %16.s[2]     \n"

  11830. 

  11831.                     "prfm   pldl1keep, [%5, #128]       \n"

  11832.                     "ld2    {v10.4s, v11.4s}, [%5]      \n"

  11833. 

  11834.                     "fmla   v12.4s, v8.4s, %14.s[0]     \n"

  11835.                     "fmla   v13.4s, v8.4s, %17.s[0]     \n"

  11836. 

  11837.                     "ext    v14.16b, v8.16b, v10.16b, #4\n"

  11838. 

  11839.                     "fmla   v6.4s, v9.4s, %14.s[1]      \n"

  11840.                     "fmla   v7.4s, v9.4s, %17.s[1]      \n"

  11841. 

  11842.                     "fmla   v12.4s, v14.4s, %14.s[2]    \n"

  11843.                     "fmla   v13.4s, v14.4s, %17.s[2]    \n"

  11844. 

  11845.                     "prfm   pldl1keep, [%3, #256]       \n"

  11846.                     "ld2    {v8.4s, v9.4s}, [%3], #32   \n"// v8 v9 = r0

  11847. 

  11848.                     "fadd   v6.4s, v6.4s, v12.4s        \n"

  11849.                     "fadd   v7.4s, v7.4s, v13.4s        \n"

  11850. 

  11851.                     "subs   %w0, %w0, #1                \n"

  11852. 

  11853.                     "st1    {v6.4s}, [%1], #16          \n"

  11854.                     "st1    {v7.4s}, [%2], #16          \n"

  11855. 

  11856.                     "bne    0b                          \n"

  11857.                     "sub    %3, %3, #32                 \n"

  11858. 

  11859.                     : "=r"(nn),         // %0

  11860.                       "=r"(outptr0),    // %1

  11861.                       "=r"(outptr1),    // %2

  11862.                       "=r"(r0),         // %3

  11863.                       "=r"(r1),         // %4

  11864.                       "=r"(r2)          // %5

  11865.                     : "0"(nn),

  11866.                       "1"(outptr0),

  11867.                       "2"(outptr1),

  11868.                       "3"(r0),

  11869.                       "4"(r1),

  11870.                       "5"(r2),

  11871.                       "w"(_k00),      // %12

  11872.                       "w"(_k03),      // %13

  11873.                       "w"(_k06),      // %14

  11874.                       "w"(_k10),      // %15

  11875.                       "w"(_k13),      // %16

  11876.                       "w"(_k16)       // %17

  11877.                     : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  11878.                 );

  11879.                 }

  11880. #else

  11881.                 if (nn > 0)

  11882.                 {

  11883.                 asm volatile(

  11884.                     "pld        [%3, #256]          \n"

  11885.                     "vld2.f32   {d16-d19}, [%3]!    \n"// q8 q9 = r0

  11886. 

  11887.                     "0:                             \n"

  11888. 

  11889.                     "pld        [%1, #128]          \n"

  11890.                     "vld1.f32   {d12-d13}, [%1]     \n"// q6 = _sum0

  11891. 

  11892.                     "vmul.f32   q12, q8, %e12[0]    \n"

  11893. 

  11894.                     "pld        [%2, #128]          \n"

  11895.                     "vld1.f32   {d14-d15}, [%2]     \n"// q7 = _sum1

  11896. 

  11897.                     "vmul.f32   q13, q8, %e15[0]    \n"

  11898. 

  11899.                     "pld        [%3, #128]          \n"

  11900.                     "vld2.f32   {d20-d21}, [%3]     \n"// q10

  11901. 

  11902.                     "vmla.f32   q6, q9, %e12[1]     \n"

  11903. 

  11904.                     "vext.32    q11, q8, q10, #1    \n"

  11905. 

  11906.                     "vmla.f32   q7, q9, %e15[1]     \n"

  11907. 

  11908.                     "pld        [%4, #256]          \n"

  11909.                     "vld2.f32   {d16-d19}, [%4]!    \n"// r1

  11910. 

  11911.                     "vmla.f32   q12, q11, %f12[0]   \n"

  11912.                     "vmla.f32   q13, q11, %f15[0]   \n"

  11913. 

  11914.                     "pld        [%4, #128]          \n"

  11915.                     "vld2.f32   {d20-d21}, [%4]     \n"

  11916. 

  11917.                     "vmla.f32   q6, q8, %e13[0]     \n"

  11918.                     "vmla.f32   q7, q8, %e16[0]     \n"

  11919. 

  11920.                     "vext.32    q11, q8, q10, #1    \n"

  11921. 

  11922.                     "vmla.f32   q12, q9, %e13[1]    \n"

  11923.                     "vmla.f32   q13, q9, %e16[1]    \n"

  11924. 

  11925.                     "pld        [%5, #256]          \n"

  11926.                     "vld2.f32   {d16-d19}, [%5]!    \n"// r2

  11927. 

  11928.                     "vmla.f32   q6, q11, %f13[0]    \n"

  11929.                     "vmla.f32   q7, q11, %f16[0]    \n"

  11930. 

  11931.                     "pld        [%5, #128]          \n"

  11932.                     "vld2.f32   {d20-d21}, [%5]     \n"

  11933. 

  11934.                     "vmla.f32   q12, q8, %e14[0]    \n"

  11935.                     "vmla.f32   q13, q8, %e17[0]    \n"

  11936. 

  11937.                     "vext.32    q11, q8, q10, #1    \n"

  11938. 

  11939.                     "vmla.f32   q6, q9, %e14[1]     \n"

  11940.                     "vmla.f32   q7, q9, %e17[1]     \n"

  11941. 

  11942.                     "vmla.f32   q12, q11, %f14[0]   \n"

  11943.                     "vmla.f32   q13, q11, %f17[0]   \n"

  11944. 

  11945.                     "pld        [%3, #256]          \n"

  11946.                     "vld2.f32   {d16-d19}, [%3]!    \n"// q8 q9 = r0

  11947. 

  11948.                     "vadd.f32   q6, q6, q12         \n"

  11949.                     "vadd.f32   q7, q7, q13         \n"

  11950. 

  11951.                     "subs       %0, #1              \n"

  11952. 

  11953.                     "vst1.f32   {d12-d13}, [%1]!    \n"

  11954.                     "vst1.f32   {d14-d15}, [%2]!    \n"

  11955. 

  11956.                     "bne        0b                  \n"

  11957.                     "sub        %3, #32             \n"

  11958. 

  11959.                     : "=r"(nn),         // %0

  11960.                       "=r"(outptr0),    // %1

  11961.                       "=r"(outptr1),    // %2

  11962.                       "=r"(r0),         // %3

  11963.                       "=r"(r1),         // %4

  11964.                       "=r"(r2)          // %5

  11965.                     : "0"(nn),

  11966.                       "1"(outptr0),

  11967.                       "2"(outptr1),

  11968.                       "3"(r0),

  11969.                       "4"(r1),

  11970.                       "5"(r2),

  11971.                       "w"(_k00),      // %12

  11972.                       "w"(_k03),      // %13

  11973.                       "w"(_k06),      // %14

  11974.                       "w"(_k10),      // %15

  11975.                       "w"(_k13),      // %16

  11976.                       "w"(_k16)       // %17

  11977.                     : "cc", "memory", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  11978.                 );

  11979.                 }

  11980. #endif // __aarch64__

  11981. #endif // __ARM_NEON

  11982.                 for (; remain>0; remain--)

  11983.                 {

  11984. #if __ARM_NEON

  11985.                     float32x4_t _r00 = vld1q_f32(r0);

  11986.                     float32x4_t _r10 = vld1q_f32(r1);

  11987.                     float32x4_t _r20 = vld1q_f32(r2);

  11988. 

  11989.                     float32x4_t _sum0 = vmulq_f32(_r00, _k00);

  11990.                     float32x4_t _sum1 = vmulq_f32(_r00, _k10);

  11991.                     _sum0 = vmlaq_f32(_sum0, _r10, _k03);

  11992.                     _sum1 = vmlaq_f32(_sum1, _r10, _k13);

  11993.                     _sum0 = vmlaq_f32(_sum0, _r20, _k06);

  11994.                     _sum1 = vmlaq_f32(_sum1, _r20, _k16);

  11995. 

  11996.                     _sum0 = vsetq_lane_f32(*outptr0, _sum0, 3);

  11997.                     _sum1 = vsetq_lane_f32(*outptr1, _sum1, 3);

  11998. #if __aarch64__

  11999.                     *outptr0 = vaddvq_f32(_sum0);

  12000.                     *outptr1 = vaddvq_f32(_sum1);

  12001. #else

  12002.                     float32x2_t _ss0 = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));

  12003.                     float32x2_t _ss1 = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1));

  12004.                     float32x2_t _ss01 = vpadd_f32(_ss0, _ss1);

  12005. 

  12006.                     *outptr0 = vget_lane_f32(_ss01, 0);

  12007.                     *outptr1 = vget_lane_f32(_ss01, 1);

  12008. #endif // __aarch64__

  12009. #else

  12010.                     float sum0 = 0.f;

  12011.                     float sum1 = 0.f;

  12012. 

  12013.                     sum0 += r0[0] * k0[0];

  12014.                     sum0 += r0[1] * k0[1];

  12015.                     sum0 += r0[2] * k0[2];

  12016.                     sum0 += r1[0] * k0[3];

  12017.                     sum0 += r1[1] * k0[4];

  12018.                     sum0 += r1[2] * k0[5];

  12019.                     sum0 += r2[0] * k0[6];

  12020.                     sum0 += r2[1] * k0[7];

  12021.                     sum0 += r2[2] * k0[8];

  12022. 

  12023.                     sum1 += r0[0] * k1[0];

  12024.                     sum1 += r0[1] * k1[1];

  12025.                     sum1 += r0[2] * k1[2];

  12026.                     sum1 += r1[0] * k1[3];

  12027.                     sum1 += r1[1] * k1[4];

  12028.                     sum1 += r1[2] * k1[5];

  12029.                     sum1 += r2[0] * k1[6];

  12030.                     sum1 += r2[1] * k1[7];

  12031.                     sum1 += r2[2] * k1[8];

  12032. 

  12033.                     *outptr0 += sum0;

  12034.                     *outptr1 += sum1;

  12035. #endif // __ARM_NEON

  12036. 

  12037.                     r0 += 2;

  12038.                     r1 += 2;

  12039.                     r2 += 2;

  12040.                     outptr0++;

  12041.                     outptr1++;

  12042.                 }

  12043. 

  12044.                 r0 += tailstep;

  12045.                 r1 += tailstep;

  12046.                 r2 += tailstep;

  12047.             }

  12048. 

  12049.             k0 += 9;

  12050.             k1 += 9;

  12051.         }

  12052.     }

  12053. 

  12054.     #pragma omp parallel for num_threads(opt.num_threads)

  12055.     for (int p=remain_outch_start; p<outch; p++)

  12056.     {

  12057.         Mat out = top_blob.channel(p);

  12058. 

  12059.         const float bias0 = bias ? bias[p] : 0.f;

  12060. 

  12061.         out.fill(bias0);

  12062. 

  12063.         const float* kernel0 = kernel + p*inch*9;

  12064. 

  12065.         for (int q=0; q<inch; q++)

  12066.         {

  12067.             float* outptr = out;

  12068. 

  12069.             const float* img0 = bottom_blob.channel(q);

  12070. 

  12071.             const float* r0 = img0;

  12072.             const float* r1 = img0 + w;

  12073.             const float* r2 = img0 + w*2;

  12074. 

  12075.             const float* k0 = kernel0;

  12076.             const float* k1 = kernel0 + 3;

  12077.             const float* k2 = kernel0 + 6;

  12078. 

  12079. #if __ARM_NEON

  12080.             float32x4_t _k0123 = vld1q_f32(k0);

  12081.             float32x4_t _k3456 = vld1q_f32(k1);

  12082.             float32x4_t _k6789 = vld1q_f32(k2);

  12083. #endif // __ARM_NEON

  12084. 

  12085.             int i = 0;

  12086. 

  12087.             for (; i < outh; i++)

  12088.             {

  12089. #if __ARM_NEON

  12090.                 int nn = outw >> 2;

  12091.                 int remain = outw & 3;

  12092. #else

  12093.                 int remain = outw;

  12094. #endif // __ARM_NEON

  12095. 

  12096. #if __ARM_NEON

  12097. #if __aarch64__

  12098.                 if (nn > 0)

  12099.                 {

  12100.                 asm volatile(

  12101.                     "prfm       pldl1keep, [%2, #256]          \n"

  12102.                     "ld2        {v2.4s, v3.4s}, [%2], #32      \n"

  12103.                     "0:                                        \n"

  12104. 

  12105.                     "prfm       pldl1keep, [%1, #128]          \n"

  12106.                     "ld1        {v0.4s}, [%1]                  \n"

  12107. 

  12108.                     "fmla       v0.4s,  v2.4s, %10.s[0]        \n"

  12109.                     "fmul       v10.4s, v3.4s, %10.s[1]        \n"

  12110. 

  12111.                     "prfm       pldl1keep, [%2, #256]          \n"

  12112.                     "ld2        {v8.4s, v9.4s}, [%2]           \n"

  12113.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  12114. 

  12115.                     "fmul       v11.4s, v1.4s, %10.s[2]        \n"

  12116. 

  12117.                     "prfm       pldl1keep, [%3, #256]          \n"

  12118.                     "ld2        {v2.4s, v3.4s}, [%3], #32      \n"

  12119. 

  12120.                     "fmla       v0.4s,  v2.4s, %11.s[0]        \n"

  12121.                     "fmla       v10.4s, v3.4s, %11.s[1]        \n"

  12122. 

  12123.                     "prfm       pldl1keep, [%3, #256]          \n"

  12124.                     "ld2        {v8.4s, v9.4s}, [%3]           \n"

  12125.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  12126. 

  12127.                     "fmla       v11.4s, v1.4s, %11.s[2]        \n"

  12128. 

  12129.                     "prfm       pldl1keep, [%4, #256]          \n"

  12130.                     "ld2        {v2.4s, v3.4s}, [%4], #32      \n"

  12131. 

  12132.                     "fmla       v0.4s,  v2.4s, %12.s[0]        \n"

  12133.                     "fmla       v10.4s, v3.4s, %12.s[1]        \n"

  12134. 

  12135.                     "prfm       pldl1keep, [%4, #256]          \n"

  12136.                     "ld2        {v8.4s, v9.4s}, [%4]           \n"

  12137.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  12138. 

  12139.                     "fmla       v11.4s, v1.4s, %12.s[2]        \n"

  12140. 

  12141.                     "prfm       pldl1keep, [%2, #256]          \n"

  12142.                     "ld2        {v2.4s, v3.4s}, [%2], #32      \n"

  12143. 

  12144.                     "fadd       v0.4s, v0.4s, v10.4s           \n"

  12145.                     "fadd       v0.4s, v0.4s, v11.4s           \n"

  12146. 

  12147.                     "subs       %w0, %w0, #1                   \n"

  12148.                     "st1        {v0.4s}, [%1], #16             \n"

  12149.                     "bne        0b                             \n"

  12150.                     "sub        %2, %2, #32                    \n"

  12151.                     : "=r"(nn),     // %0

  12152.                       "=r"(outptr), // %1

  12153.                       "=r"(r0),     // %2

  12154.                       "=r"(r1),     // %3

  12155.                       "=r"(r2)      // %4

  12156.                     : "0"(nn),

  12157.                       "1"(outptr),

  12158.                       "2"(r0),

  12159.                       "3"(r1),

  12160.                       "4"(r2),

  12161.                       "w"(_k0123),  // %10

  12162.                       "w"(_k3456),  // %11

  12163.                       "w"(_k6789)   // %12

  12164.                     : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  12165.                 );

  12166.                 }

  12167. #else

  12168.                 if (nn > 0)

  12169.                 {

  12170.                 asm volatile(

  12171.                     "pld        [%2, #256]          \n"

  12172.                     "vld2.f32   {d4-d7}, [%2]!      \n"

  12173. 

  12174.                     "0:                             \n"

  12175.                     "pld        [%1, #128]          \n"

  12176.                     "vld1.f32   {d0-d1}, [%1]       \n"

  12177. 

  12178.                     "vmla.f32   q0, q2, %e10[0]     \n"

  12179.                     "vmul.f32   q10, q3, %e10[1]    \n"

  12180. 

  12181.                     "pld        [%2, #128]          \n"

  12182.                     "vld2.f32   {d16-d17}, [%2]     \n"

  12183.                     "vext.32    q1, q2, q8, #1      \n"

  12184. 

  12185.                     "vmul.f32   q11, q1, %f10[0]    \n"

  12186. 

  12187.                     "pld        [%3, #256]          \n"

  12188.                     "vld2.f32   {d4-d7}, [%3]!      \n"

  12189. 

  12190.                     "vmla.f32   q0, q2, %e11[0]     \n"

  12191.                     "vmla.f32   q10, q3, %e11[1]    \n"

  12192. 

  12193.                     "pld        [%3, #128]          \n"

  12194.                     "vld2.f32   {d16-d17}, [%3]     \n"

  12195.                     "vext.32    q1, q2, q8, #1      \n"

  12196. 

  12197.                     "vmla.f32   q11, q1, %f11[0]    \n"

  12198. 

  12199.                     "pld        [%4, #256]          \n"

  12200.                     "vld2.f32   {d4-d7}, [%4]!      \n"

  12201. 

  12202.                     "vmla.f32   q0, q2, %e12[0]     \n"

  12203.                     "vmla.f32   q10, q3, %e12[1]    \n"

  12204. 

  12205.                     "pld        [%4, #128]          \n"

  12206.                     "vld2.f32   {d16-d17}, [%4]     \n"

  12207.                     "vext.32    q1, q2, q8, #1      \n"

  12208. 

  12209.                     "vmla.f32   q11, q1, %f12[0]    \n"

  12210. 

  12211.                     "pld        [%2, #256]          \n"

  12212.                     "vld2.f32   {d4-d7}, [%2]!      \n"

  12213. 

  12214.                     "vadd.f32   q0, q0, q10         \n"

  12215.                     "vadd.f32   q0, q0, q11         \n"

  12216. 

  12217.                     "subs       %0, #1              \n"

  12218.                     "vst1.f32   {d0-d1}, [%1]!      \n"

  12219.                     "bne        0b                  \n"

  12220.                     "sub        %2, #32             \n"

  12221.                     : "=r"(nn),     // %0

  12222.                       "=r"(outptr), // %1

  12223.                       "=r"(r0),     // %2

  12224.                       "=r"(r1),     // %3

  12225.                       "=r"(r2)      // %4

  12226.                     : "0"(nn),

  12227.                       "1"(outptr),

  12228.                       "2"(r0),

  12229.                       "3"(r1),

  12230.                       "4"(r2),

  12231.                       "w"(_k0123),  // %10

  12232.                       "w"(_k3456),  // %11

  12233.                       "w"(_k6789)   // %12

  12234.                     : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  12235.                 );

  12236.                 }

  12237. #endif // __aarch64__

  12238. #endif // __ARM_NEON

  12239.                 for (; remain>0; remain--)

  12240.                 {

  12241. #if __ARM_NEON

  12242.                     float32x4_t _r00 = vld1q_f32(r0);

  12243.                     float32x4_t _r10 = vld1q_f32(r1);

  12244.                     float32x4_t _r20 = vld1q_f32(r2);

  12245. 

  12246.                     float32x4_t _sum = vmulq_f32(_r00, _k0123);

  12247.                     _sum = vmlaq_f32(_sum, _r10, _k3456);

  12248.                     _sum = vmlaq_f32(_sum, _r20, _k6789);

  12249. 

  12250.                     _sum = vsetq_lane_f32(*outptr, _sum, 3);

  12251. 

  12252. #if __aarch64__

  12253.                     *outptr = vaddvq_f32(_sum);

  12254. #else

  12255.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  12256.                     _ss = vpadd_f32(_ss, _ss);

  12257. 

  12258.                     *outptr = vget_lane_f32(_ss, 0);

  12259. #endif // __aarch64__

  12260. #else

  12261.                     float sum = 0;

  12262. 

  12263.                     sum += r0[0] * k0[0];

  12264.                     sum += r0[1] * k0[1];

  12265.                     sum += r0[2] * k0[2];

  12266.                     sum += r1[0] * k1[0];

  12267.                     sum += r1[1] * k1[1];

  12268.                     sum += r1[2] * k1[2];

  12269.                     sum += r2[0] * k2[0];

  12270.                     sum += r2[1] * k2[1];

  12271.                     sum += r2[2] * k2[2];

  12272. 

  12273.                     *outptr += sum;

  12274. #endif // __ARM_NEON

  12275. 

  12276.                     r0 += 2;

  12277.                     r1 += 2;

  12278.                     r2 += 2;

  12279.                     outptr++;

  12280.                 }

  12281. 

  12282.                 r0 += tailstep;

  12283.                 r1 += tailstep;

  12284.                 r2 += tailstep;

  12285.             }

  12286. 

  12287.             kernel0 += 9;

  12288.         }

  12289.     }

  12290. }

  12291. 

  12292. static void conv3x3s2_transform_kernel_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch)

  12293. {

  12294.     kernel_tm.create(8*9, inch, outch/8 + outch%8);

  12295. 

  12296.     const float* kernel = _kernel;

  12297. 

  12298.     int p=0;

  12299.     for (; p+7<outch; p+=8)

  12300.     {

  12301.         const float* k0 = kernel + (p+0)*inch*9;

  12302.         const float* k1 = kernel + (p+1)*inch*9;

  12303.         const float* k2 = kernel + (p+2)*inch*9;

  12304.         const float* k3 = kernel + (p+3)*inch*9;

  12305.         const float* k4 = kernel + (p+4)*inch*9;

  12306.         const float* k5 = kernel + (p+5)*inch*9;

  12307.         const float* k6 = kernel + (p+6)*inch*9;

  12308.         const float* k7 = kernel + (p+7)*inch*9;

  12309. 

  12310.         float* ktmp = kernel_tm.channel(p/8);

  12311. 

  12312.         for (int q=0; q<inch; q++)

  12313.         {

  12314.             for (int k=0; k<9; k++)

  12315.             {

  12316.                 ktmp[0] = k0[k];

  12317.                 ktmp[1] = k1[k];

  12318.                 ktmp[2] = k2[k];

  12319.                 ktmp[3] = k3[k];

  12320.                 ktmp[4] = k4[k];

  12321.                 ktmp[5] = k5[k];

  12322.                 ktmp[6] = k6[k];

  12323.                 ktmp[7] = k7[k];

  12324.                 ktmp += 8;

  12325.             }

  12326. 

  12327.             k0 += 9;

  12328.             k1 += 9;

  12329.             k2 += 9;

  12330.             k3 += 9;

  12331.             k4 += 9;

  12332.             k5 += 9;

  12333.             k6 += 9;

  12334.             k7 += 9;

  12335.         }

  12336.     }

  12337.     for (; p<outch; p++)

  12338.     {

  12339.         const float* k0 = kernel + (p+0)*inch*9;

  12340. 

  12341.         float* ktmp = kernel_tm.channel(p/8 + p%8);

  12342. 

  12343.         for (int q=0; q<inch; q++)

  12344.         {

  12345.             for (int k=0; k<9; k++)

  12346.             {

  12347.                 ktmp[k] = k0[k];

  12348.             }

  12349.             ktmp += 9;

  12350. 

  12351.             k0 += 9;

  12352.         }

  12353.     }

  12354. }

  12355. 

  12356. static void conv3x3s2_packed_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  12357. {

  12358.     int w = bottom_blob.w;

  12359.     int inch = bottom_blob.c;

  12360. 

  12361.     int outw = top_blob.w;

  12362.     int outh = top_blob.h;

  12363.     int outch = top_blob.c;

  12364. 

  12365.     const int tailstep = w - 2*outw + w;

  12366. 

  12367. //     const float* kernel = _kernel;

  12368.     const float* bias = _bias;

  12369. 

  12370.     int nn_outch = outch >> 3;

  12371.     int remain_outch_start = nn_outch << 3;

  12372. 

  12373.     #pragma omp parallel for num_threads(opt.num_threads)

  12374.     for (int pp=0; pp<nn_outch; pp++)

  12375.     {

  12376.         int p = pp * 8;

  12377. 

  12378.         Mat out0 = top_blob.channel(p+0);

  12379.         Mat out1 = top_blob.channel(p+1);

  12380.         Mat out2 = top_blob.channel(p+2);

  12381.         Mat out3 = top_blob.channel(p+3);

  12382.         Mat out4 = top_blob.channel(p+4);

  12383.         Mat out5 = top_blob.channel(p+5);

  12384.         Mat out6 = top_blob.channel(p+6);

  12385.         Mat out7 = top_blob.channel(p+7);

  12386. 

  12387.         const float bias0 = bias ? bias[p+0] : 0.f;

  12388.         const float bias1 = bias ? bias[p+1] : 0.f;

  12389.         const float bias2 = bias ? bias[p+2] : 0.f;

  12390.         const float bias3 = bias ? bias[p+3] : 0.f;

  12391.         const float bias4 = bias ? bias[p+4] : 0.f;

  12392.         const float bias5 = bias ? bias[p+5] : 0.f;

  12393.         const float bias6 = bias ? bias[p+6] : 0.f;

  12394.         const float bias7 = bias ? bias[p+7] : 0.f;

  12395. 

  12396.         out0.fill(bias0);

  12397.         out1.fill(bias1);

  12398.         out2.fill(bias2);

  12399.         out3.fill(bias3);

  12400.         out4.fill(bias4);

  12401.         out5.fill(bias5);

  12402.         out6.fill(bias6);

  12403.         out7.fill(bias7);

  12404. 

  12405.         const float* ktmp = _kernel.channel(p/8);

  12406. 

  12407.         for (int q=0; q<inch; q++)

  12408.         {

  12409.             float* outptr0 = out0;

  12410.             float* outptr1 = out1;

  12411.             float* outptr2 = out2;

  12412.             float* outptr3 = out3;

  12413.             float* outptr4 = out4;

  12414.             float* outptr5 = out5;

  12415.             float* outptr6 = out6;

  12416.             float* outptr7 = out7;

  12417. 

  12418.             const float* img0 = bottom_blob.channel(q);

  12419. 

  12420.             const float* r0 = img0;

  12421.             const float* r1 = img0 + w;

  12422.             const float* r2 = img0 + w*2;

  12423. 

  12424.             int i = 0;

  12425. 

  12426.             for (; i < outh; i++)

  12427.             {

  12428. #if __ARM_NEON

  12429.                 int nn = outw >> 2;

  12430.                 int remain = outw & 3;

  12431. #else

  12432.                 int remain = outw;

  12433. #endif // __ARM_NEON

  12434. 

  12435. #if __ARM_NEON

  12436. #if __aarch64__

  12437.                 if (nn > 0)

  12438.                 {

  12439.                 asm volatile(

  12440.                     "0:                                 \n"

  12441. 

  12442.                     "prfm   pldl1keep, [%1, #128]       \n"

  12443.                     "ld1    {v8.4s}, [%1]               \n"

  12444.                     "prfm   pldl1keep, [%2, #128]       \n"

  12445.                     "ld1    {v9.4s}, [%2]               \n"

  12446. 

  12447.                     "prfm   pldl1keep, [%3, #128]       \n"

  12448.                     "ld1    {v10.4s}, [%3]              \n"

  12449.                     "prfm   pldl1keep, [%4, #128]       \n"

  12450.                     "ld1    {v11.4s}, [%4]              \n"

  12451. 

  12452.                     ///

  12453.                     "prfm   pldl1keep, [%9, #256]       \n"

  12454.                     "ld2    {v4.4s, v5.4s}, [%9], #32   \n"// v4=00 v5=01

  12455. 

  12456.                     "ld1    {v0.4s, v1.4s}, [%12], #32  \n"

  12457. 

  12458.                     "fmla   v8.4s, v4.4s, v0.s[0]       \n"

  12459.                     "fmla   v9.4s, v4.4s, v0.s[1]       \n"

  12460. 

  12461.                     "prfm   pldl1keep, [%5, #128]       \n"

  12462.                     "ld1    {v12.4s}, [%5]              \n"

  12463.                     "prfm   pldl1keep, [%6, #128]       \n"

  12464.                     "ld1    {v13.4s}, [%6]              \n"

  12465. 

  12466.                     "fmla   v10.4s, v4.4s, v0.s[2]      \n"

  12467.                     "fmla   v11.4s, v4.4s, v0.s[3]      \n"

  12468. 

  12469.                     "prfm   pldl1keep, [%7, #128]       \n"

  12470.                     "ld1    {v14.4s}, [%7]              \n"

  12471.                     "prfm   pldl1keep, [%8, #128]       \n"

  12472.                     "ld1    {v15.4s}, [%8]              \n"

  12473. 

  12474.                     "ld1    {v2.4s, v3.4s}, [%12], #32  \n"

  12475. 

  12476.                     "fmla   v12.4s, v4.4s, v1.s[0]      \n"

  12477.                     "fmla   v13.4s, v4.4s, v1.s[1]      \n"

  12478.                     "fmla   v14.4s, v4.4s, v1.s[2]      \n"

  12479.                     "fmla   v15.4s, v4.4s, v1.s[3]      \n"

  12480. 

  12481.                     "prfm   pldl1keep, [%9, #256]       \n"

  12482.                     "ld2    {v6.4s, v7.4s}, [%9]        \n"// v6

  12483. 

  12484.                     "fmla   v8.4s, v5.4s, v2.s[0]       \n"

  12485.                     "fmla   v9.4s, v5.4s, v2.s[1]       \n"

  12486.                     "fmla   v10.4s, v5.4s, v2.s[2]      \n"

  12487.                     "fmla   v11.4s, v5.4s, v2.s[3]      \n"

  12488. 

  12489.                     "ext    v6.16b, v4.16b, v6.16b, #4  \n"// v6=02

  12490. 

  12491.                     "ld1    {v0.4s, v1.4s}, [%12], #32  \n"

  12492. 

  12493.                     "fmla   v12.4s, v5.4s, v3.s[0]      \n"

  12494.                     "fmla   v13.4s, v5.4s, v3.s[1]      \n"

  12495.                     "fmla   v14.4s, v5.4s, v3.s[2]      \n"

  12496.                     "fmla   v15.4s, v5.4s, v3.s[3]      \n"

  12497. 

  12498.                     ///

  12499.                     "prfm   pldl1keep, [%10, #256]      \n"

  12500.                     "ld2    {v4.4s, v5.4s}, [%10], #32  \n"// v4=10 v5=11

  12501. 

  12502.                     "fmla   v8.4s, v6.4s, v0.s[0]       \n"

  12503.                     "fmla   v9.4s, v6.4s, v0.s[1]       \n"

  12504.                     "fmla   v10.4s, v6.4s, v0.s[2]      \n"

  12505.                     "fmla   v11.4s, v6.4s, v0.s[3]      \n"

  12506. 

  12507.                     "ld1    {v2.4s, v3.4s}, [%12], #32  \n"

  12508. 

  12509.                     "fmla   v12.4s, v6.4s, v1.s[0]      \n"

  12510.                     "fmla   v13.4s, v6.4s, v1.s[1]      \n"

  12511.                     "fmla   v14.4s, v6.4s, v1.s[2]      \n"

  12512.                     "fmla   v15.4s, v6.4s, v1.s[3]      \n"

  12513. 

  12514.                     "fmla   v8.4s, v4.4s, v2.s[0]       \n"

  12515.                     "fmla   v9.4s, v4.4s, v2.s[1]       \n"

  12516.                     "fmla   v10.4s, v4.4s, v2.s[2]      \n"

  12517.                     "fmla   v11.4s, v4.4s, v2.s[3]      \n"

  12518. 

  12519.                     "ld1    {v0.4s, v1.4s}, [%12], #32  \n"

  12520. 

  12521.                     "fmla   v12.4s, v4.4s, v3.s[0]      \n"

  12522.                     "fmla   v13.4s, v4.4s, v3.s[1]      \n"

  12523.                     "fmla   v14.4s, v4.4s, v3.s[2]      \n"

  12524.                     "fmla   v15.4s, v4.4s, v3.s[3]      \n"

  12525. 

  12526.                     "prfm   pldl1keep, [%10, #256]      \n"

  12527.                     "ld2    {v6.4s, v7.4s}, [%10]       \n"// v6

  12528. 

  12529.                     "fmla   v8.4s, v5.4s, v0.s[0]       \n"

  12530.                     "fmla   v9.4s, v5.4s, v0.s[1]       \n"

  12531.                     "fmla   v10.4s, v5.4s, v0.s[2]      \n"

  12532.                     "fmla   v11.4s, v5.4s, v0.s[3]      \n"

  12533. 

  12534.                     "ld1    {v2.4s, v3.4s}, [%12], #32  \n"

  12535. 

  12536.                     "ext    v6.16b, v4.16b, v6.16b, #4  \n"// v6=12

  12537. 

  12538.                     "fmla   v12.4s, v5.4s, v1.s[0]      \n"

  12539.                     "fmla   v13.4s, v5.4s, v1.s[1]      \n"

  12540.                     "fmla   v14.4s, v5.4s, v1.s[2]      \n"

  12541.                     "fmla   v15.4s, v5.4s, v1.s[3]      \n"

  12542. 

  12543.                     ///

  12544.                     "prfm   pldl1keep, [%11, #256]      \n"

  12545.                     "ld2    {v4.4s, v5.4s}, [%11], #32  \n"// v4=20 v5=21

  12546. 

  12547.                     "fmla   v8.4s, v6.4s, v2.s[0]       \n"

  12548.                     "fmla   v9.4s, v6.4s, v2.s[1]       \n"

  12549.                     "fmla   v10.4s, v6.4s, v2.s[2]      \n"

  12550.                     "fmla   v11.4s, v6.4s, v2.s[3]      \n"

  12551. 

  12552.                     "ld1    {v0.4s, v1.4s}, [%12], #32  \n"

  12553. 

  12554.                     "fmla   v12.4s, v6.4s, v3.s[0]      \n"

  12555.                     "fmla   v13.4s, v6.4s, v3.s[1]      \n"

  12556.                     "fmla   v14.4s, v6.4s, v3.s[2]      \n"

  12557.                     "fmla   v15.4s, v6.4s, v3.s[3]      \n"

  12558. 

  12559.                     "fmla   v8.4s, v4.4s, v0.s[0]       \n"

  12560.                     "fmla   v9.4s, v4.4s, v0.s[1]       \n"

  12561.                     "fmla   v10.4s, v4.4s, v0.s[2]      \n"

  12562.                     "fmla   v11.4s, v4.4s, v0.s[3]      \n"

  12563. 

  12564.                     "ld1    {v2.4s, v3.4s}, [%12], #32  \n"

  12565. 

  12566.                     "fmla   v12.4s, v4.4s, v1.s[0]      \n"

  12567.                     "fmla   v13.4s, v4.4s, v1.s[1]      \n"

  12568.                     "fmla   v14.4s, v4.4s, v1.s[2]      \n"

  12569.                     "fmla   v15.4s, v4.4s, v1.s[3]      \n"

  12570. 

  12571.                     "prfm   pldl1keep, [%11, #256]      \n"

  12572.                     "ld2    {v6.4s, v7.4s}, [%11]       \n"// v6

  12573. 

  12574.                     "fmla   v8.4s, v5.4s, v2.s[0]       \n"

  12575.                     "fmla   v9.4s, v5.4s, v2.s[1]       \n"

  12576.                     "fmla   v10.4s, v5.4s, v2.s[2]      \n"

  12577.                     "fmla   v11.4s, v5.4s, v2.s[3]      \n"

  12578. 

  12579.                     "ext    v6.16b, v4.16b, v6.16b, #4  \n"// v6=22

  12580. 

  12581.                     "ld1    {v0.4s, v1.4s}, [%12], #32  \n"

  12582. 

  12583.                     "fmla   v12.4s, v5.4s, v3.s[0]      \n"

  12584.                     "fmla   v13.4s, v5.4s, v3.s[1]      \n"

  12585.                     "fmla   v14.4s, v5.4s, v3.s[2]      \n"

  12586.                     "fmla   v15.4s, v5.4s, v3.s[3]      \n"

  12587. 

  12588.                     "fmla   v8.4s, v6.4s, v0.s[0]       \n"

  12589.                     "fmla   v9.4s, v6.4s, v0.s[1]       \n"

  12590.                     "fmla   v10.4s, v6.4s, v0.s[2]      \n"

  12591.                     "fmla   v11.4s, v6.4s, v0.s[3]      \n"

  12592. 

  12593.                     "fmla   v12.4s, v6.4s, v1.s[0]      \n"

  12594.                     "fmla   v13.4s, v6.4s, v1.s[1]      \n"

  12595. 

  12596.                     "st1    {v8.4s}, [%1], #16          \n"

  12597.                     "st1    {v9.4s}, [%2], #16          \n"

  12598. 

  12599.                     "fmla   v14.4s, v6.4s, v1.s[2]      \n"

  12600.                     "fmla   v15.4s, v6.4s, v1.s[3]      \n"

  12601. 

  12602.                     "st1    {v10.4s}, [%3], #16         \n"

  12603.                     "st1    {v11.4s}, [%4], #16         \n"

  12604. 

  12605.                     "sub    %12, %12, #288              \n"

  12606. 

  12607.                     "st1    {v12.4s}, [%5], #16         \n"

  12608.                     "st1    {v13.4s}, [%6], #16         \n"

  12609. 

  12610.                     "subs   %w0, %w0, #1                \n"

  12611. 

  12612.                     "st1    {v14.4s}, [%7], #16         \n"

  12613.                     "st1    {v15.4s}, [%8], #16         \n"

  12614. 

  12615.                     "bne    0b                          \n"

  12616.                     : "=r"(nn),         // %0

  12617.                       "=r"(outptr0),    // %1

  12618.                       "=r"(outptr1),    // %2

  12619.                       "=r"(outptr2),    // %3

  12620.                       "=r"(outptr3),    // %4

  12621.                       "=r"(outptr4),    // %5

  12622.                       "=r"(outptr5),    // %6

  12623.                       "=r"(outptr6),    // %7

  12624.                       "=r"(outptr7),    // %8

  12625.                       "=r"(r0),         // %9

  12626.                       "=r"(r1),         // %10

  12627.                       "=r"(r2),         // %11

  12628.                       "=r"(ktmp)        // %12

  12629.                     : "0"(nn),

  12630.                       "1"(outptr0),

  12631.                       "2"(outptr1),

  12632.                       "3"(outptr2),

  12633.                       "4"(outptr3),

  12634.                       "5"(outptr4),

  12635.                       "6"(outptr5),

  12636.                       "7"(outptr6),

  12637.                       "8"(outptr7),

  12638.                       "9"(r0),

  12639.                       "10"(r1),

  12640.                       "11"(r2),

  12641.                       "12"(ktmp)

  12642.                     : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  12643.                 );

  12644.                 }

  12645. #else // __aarch64__

  12646.                 if (nn > 0)

  12647.                 {

  12648.                 asm volatile(

  12649.                     "0:                             \n"

  12650. 

  12651.                     "pld        [%1, #128]          \n"

  12652.                     "vld1.f32   {d16-d17}, [%1]     \n"

  12653.                     "pld        [%2, #128]          \n"

  12654.                     "vld1.f32   {d18-d19}, [%2]     \n"

  12655. 

  12656.                     "pld        [%3, #128]          \n"

  12657.                     "vld1.f32   {d20-d21}, [%3]     \n"

  12658.                     "pld        [%4, #128]          \n"

  12659.                     "vld1.f32   {d22-d23}, [%4]     \n"

  12660. 

  12661.                     ///

  12662.                     "pld        [%9, #256]          \n"

  12663.                     "vld2.f32   {d8-d11}, [%9]!     \n"// q4=00 q5=01

  12664. 

  12665.                     "vld1.f32   {d0-d3}, [%12 :128]! \n"

  12666. 

  12667.                     "vmla.f32   q8, q4, d0[0]       \n"

  12668.                     "vmla.f32   q9, q4, d0[1]       \n"

  12669. 

  12670.                     "pld        [%5, #128]          \n"

  12671.                     "vld1.f32   {d24-d25}, [%5]     \n"

  12672.                     "pld        [%6, #128]          \n"

  12673.                     "vld1.f32   {d26-d27}, [%6]     \n"

  12674. 

  12675.                     "vmla.f32   q10, q4, d1[0]      \n"

  12676.                     "vmla.f32   q11, q4, d1[1]      \n"

  12677. 

  12678.                     "pld        [%7, #128]          \n"

  12679.                     "vld1.f32   {d28-d29}, [%7]     \n"

  12680.                     "pld        [%8, #128]          \n"

  12681.                     "vld1.f32   {d30-d31}, [%8]     \n"

  12682. 

  12683.                     "vld1.f32   {d4-d7}, [%12 :128]! \n"

  12684. 

  12685.                     "vmla.f32   q12, q4, d2[0]      \n"

  12686.                     "vmla.f32   q13, q4, d2[1]      \n"

  12687.                     "vmla.f32   q14, q4, d3[0]      \n"

  12688.                     "vmla.f32   q15, q4, d3[1]      \n"

  12689. 

  12690.                     "pld        [%9, #128]          \n"

  12691.                     "vld2.f32   {d12-d13}, [%9]     \n"// q6

  12692. 

  12693.                     "vmla.f32   q8, q5, d4[0]       \n"

  12694.                     "vmla.f32   q9, q5, d4[1]       \n"

  12695.                     "vmla.f32   q10, q5, d5[0]      \n"

  12696.                     "vmla.f32   q11, q5, d5[1]      \n"

  12697. 

  12698.                     "vext.f32   q6, q4, q6, #1      \n"// q6=02

  12699. 

  12700.                     "vld1.f32   {d0-d3}, [%12 :128]! \n"

  12701. 

  12702.                     "vmla.f32   q12, q5, d6[0]      \n"

  12703.                     "vmla.f32   q13, q5, d6[1]      \n"

  12704.                     "vmla.f32   q14, q5, d7[0]      \n"

  12705.                     "vmla.f32   q15, q5, d7[1]      \n"

  12706. 

  12707.                     ///

  12708.                     "pld        [%10, #256]         \n"

  12709.                     "vld2.f32   {d8-d11}, [%10]!    \n"// q4=10 q5=11

  12710. 

  12711.                     "vmla.f32   q8, q6, d0[0]       \n"

  12712.                     "vmla.f32   q9, q6, d0[1]       \n"

  12713.                     "vmla.f32   q10, q6, d1[0]      \n"

  12714.                     "vmla.f32   q11, q6, d1[1]      \n"

  12715. 

  12716.                     "vld1.f32   {d4-d7}, [%12 :128]! \n"

  12717. 

  12718.                     "vmla.f32   q12, q6, d2[0]      \n"

  12719.                     "vmla.f32   q13, q6, d2[1]      \n"

  12720.                     "vmla.f32   q14, q6, d3[0]      \n"

  12721.                     "vmla.f32   q15, q6, d3[1]      \n"

  12722. 

  12723.                     "vmla.f32   q8, q4, d4[0]       \n"

  12724.                     "vmla.f32   q9, q4, d4[1]       \n"

  12725.                     "vmla.f32   q10, q4, d5[0]      \n"

  12726.                     "vmla.f32   q11, q4, d5[1]      \n"

  12727. 

  12728.                     "vld1.f32   {d0-d3}, [%12 :128]! \n"

  12729. 

  12730.                     "vmla.f32   q12, q4, d6[0]      \n"

  12731.                     "vmla.f32   q13, q4, d6[1]      \n"

  12732.                     "vmla.f32   q14, q4, d7[0]      \n"

  12733.                     "vmla.f32   q15, q4, d7[1]      \n"

  12734. 

  12735.                     "pld        [%10, #128]         \n"

  12736.                     "vld2.f32   {d12-d13}, [%10]    \n"// q6

  12737. 

  12738.                     "vmla.f32   q8, q5, d0[0]       \n"

  12739.                     "vmla.f32   q9, q5, d0[1]       \n"

  12740.                     "vmla.f32   q10, q5, d1[0]      \n"

  12741.                     "vmla.f32   q11, q5, d1[1]      \n"

  12742. 

  12743.                     "vld1.f32   {d4-d7}, [%12 :128]! \n"

  12744. 

  12745.                     "vext.f32   q6, q4, q6, #1      \n"// q6=12

  12746. 

  12747.                     "vmla.f32   q12, q5, d2[0]      \n"

  12748.                     "vmla.f32   q13, q5, d2[1]      \n"

  12749.                     "vmla.f32   q14, q5, d3[0]      \n"

  12750.                     "vmla.f32   q15, q5, d3[1]      \n"

  12751. 

  12752.                     ///

  12753.                     "pld        [%11, #256]         \n"

  12754.                     "vld2.f32   {d8-d11}, [%11]!    \n"// q4=20 q5=21

  12755. 

  12756.                     "vmla.f32   q8, q6, d4[0]       \n"

  12757.                     "vmla.f32   q9, q6, d4[1]       \n"

  12758.                     "vmla.f32   q10, q6, d5[0]      \n"

  12759.                     "vmla.f32   q11, q6, d5[1]      \n"

  12760. 

  12761.                     "vld1.f32   {d0-d3}, [%12 :128]! \n"

  12762. 

  12763.                     "vmla.f32   q12, q6, d6[0]      \n"

  12764.                     "vmla.f32   q13, q6, d6[1]      \n"

  12765.                     "vmla.f32   q14, q6, d7[0]      \n"

  12766.                     "vmla.f32   q15, q6, d7[1]      \n"

  12767. 

  12768.                     "vmla.f32   q8, q4, d0[0]       \n"

  12769.                     "vmla.f32   q9, q4, d0[1]       \n"

  12770.                     "vmla.f32   q10, q4, d1[0]      \n"

  12771.                     "vmla.f32   q11, q4, d1[1]      \n"

  12772. 

  12773.                     "vld1.f32   {d4-d7}, [%12 :128]! \n"

  12774. 

  12775.                     "vmla.f32   q12, q4, d2[0]      \n"

  12776.                     "vmla.f32   q13, q4, d2[1]      \n"

  12777.                     "vmla.f32   q14, q4, d3[0]      \n"

  12778.                     "vmla.f32   q15, q4, d3[1]      \n"

  12779. 

  12780.                     "pld        [%11, #128]         \n"

  12781.                     "vld2.f32   {d12-d13}, [%11]    \n"// q6

  12782. 

  12783.                     "vmla.f32   q8, q5, d4[0]       \n"

  12784.                     "vmla.f32   q9, q5, d4[1]       \n"

  12785.                     "vmla.f32   q10, q5, d5[0]      \n"

  12786.                     "vmla.f32   q11, q5, d5[1]      \n"

  12787. 

  12788.                     "vext.f32   q6, q4, q6, #1      \n"// q6=22

  12789. 

  12790.                     "vld1.f32   {d0-d3}, [%12 :128]! \n"

  12791. 

  12792.                     "vmla.f32   q12, q5, d6[0]      \n"

  12793.                     "vmla.f32   q13, q5, d6[1]      \n"

  12794.                     "vmla.f32   q14, q5, d7[0]      \n"

  12795.                     "vmla.f32   q15, q5, d7[1]      \n"

  12796. 

  12797.                     "vmla.f32   q8, q6, d0[0]       \n"

  12798.                     "vmla.f32   q9, q6, d0[1]       \n"

  12799.                     "vmla.f32   q10, q6, d1[0]      \n"

  12800.                     "vmla.f32   q11, q6, d1[1]      \n"

  12801. 

  12802.                     "vmla.f32   q12, q6, d2[0]      \n"

  12803.                     "vmla.f32   q13, q6, d2[1]      \n"

  12804. 

  12805.                     "vst1.f32   {d16-d17}, [%1]!    \n"

  12806.                     "vst1.f32   {d18-d19}, [%2]!    \n"

  12807. 

  12808.                     "vmla.f32   q14, q6, d3[0]      \n"

  12809.                     "vmla.f32   q15, q6, d3[1]      \n"

  12810. 

  12811.                     "vst1.f32   {d20-d21}, [%3]!    \n"

  12812.                     "vst1.f32   {d22-d23}, [%4]!    \n"

  12813. 

  12814.                     "sub        %12, %12, #288      \n"

  12815. 

  12816.                     "vst1.f32   {d24-d25}, [%5]!    \n"

  12817.                     "vst1.f32   {d26-d27}, [%6]!    \n"

  12818. 

  12819.                     "subs       %0, #1              \n"

  12820. 

  12821.                     "vst1.f32   {d28-d29}, [%7]!    \n"

  12822.                     "vst1.f32   {d30-d31}, [%8]!    \n"

  12823. 

  12824.                     "bne        0b                  \n"

  12825.                     : "=r"(nn),         // %0

  12826.                       "=r"(outptr0),    // %1

  12827.                       "=r"(outptr1),    // %2

  12828.                       "=r"(outptr2),    // %3

  12829.                       "=r"(outptr3),    // %4

  12830.                       "=r"(outptr4),    // %5

  12831.                       "=r"(outptr5),    // %6

  12832.                       "=r"(outptr6),    // %7

  12833.                       "=r"(outptr7),    // %8

  12834.                       "=r"(r0),         // %9

  12835.                       "=r"(r1),         // %10

  12836.                       "=r"(r2),         // %11

  12837.                       "=r"(ktmp)        // %12

  12838.                     : "0"(nn),

  12839.                       "1"(outptr0),

  12840.                       "2"(outptr1),

  12841.                       "3"(outptr2),

  12842.                       "4"(outptr3),

  12843.                       "5"(outptr4),

  12844.                       "6"(outptr5),

  12845.                       "7"(outptr6),

  12846.                       "8"(outptr7),

  12847.                       "9"(r0),

  12848.                       "10"(r1),

  12849.                       "11"(r2),

  12850.                       "12"(ktmp)

  12851.                     : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  12852.                 );

  12853.                 }

  12854. #endif // __aarch64__

  12855. #endif // __ARM_NEON

  12856.                 for (; remain>0; remain--)

  12857.                 {

  12858. #if __ARM_NEON

  12859. #if __aarch64__

  12860.                     asm volatile(

  12861.                         "ld1    {v10.4s, v11.4s}, [%11], #32    \n"

  12862. 

  12863.                         "prfm   pldl1keep, [%8, #128]   \n"

  12864.                         "ld1    {v0.4s}, [%8]           \n"

  12865. 

  12866.                         "ld1    {v12.4s, v13.4s}, [%11], #32    \n"

  12867. 

  12868.                         "ld1    {v8.s}[0], [%0]         \n"

  12869.                         "ld1    {v8.s}[1], [%1]         \n"

  12870.                         "ld1    {v8.s}[2], [%2]         \n"

  12871.                         "ld1    {v8.s}[3], [%3]         \n"

  12872. 

  12873.                         "fmul   v14.4s, v10.4s, v0.s[0] \n"

  12874.                         "fmul   v15.4s, v11.4s, v0.s[0] \n"

  12875. 

  12876.                         "ld1    {v9.s}[0], [%4]         \n"

  12877.                         "ld1    {v9.s}[1], [%5]         \n"

  12878.                         "ld1    {v9.s}[2], [%6]         \n"

  12879.                         "ld1    {v9.s}[3], [%7]         \n"

  12880. 

  12881.                         "ld1    {v10.4s, v11.4s}, [%11], #32    \n"

  12882. 

  12883.                         "fmla   v8.4s, v12.4s, v0.s[1]  \n"

  12884.                         "fmla   v9.4s, v13.4s, v0.s[1]  \n"

  12885. 

  12886.                         "ld1    {v12.4s, v13.4s}, [%11], #32    \n"

  12887. 

  12888.                         "fmla   v14.4s, v10.4s, v0.s[2] \n"

  12889.                         "fmla   v15.4s, v11.4s, v0.s[2] \n"

  12890. 

  12891.                         "prfm   pldl1keep, [%9, #128]   \n"

  12892.                         "ld1    {v1.4s}, [%9]           \n"

  12893. 

  12894.                         "ld1    {v10.4s, v11.4s}, [%11], #32    \n"

  12895. 

  12896.                         "fmla   v8.4s, v12.4s, v1.s[0]  \n"

  12897.                         "fmla   v9.4s, v13.4s, v1.s[0]  \n"

  12898. 

  12899.                         "ld1    {v12.4s, v13.4s}, [%11], #32    \n"

  12900. 

  12901.                         "fmla   v14.4s, v10.4s, v1.s[1] \n"

  12902.                         "fmla   v15.4s, v11.4s, v1.s[1] \n"

  12903. 

  12904.                         "ld1    {v10.4s, v11.4s}, [%11], #32    \n"

  12905. 

  12906.                         "fmla   v8.4s, v12.4s, v1.s[2]  \n"

  12907.                         "fmla   v9.4s, v13.4s, v1.s[2]  \n"

  12908. 

  12909.                         "prfm   pldl1keep, [%10, #128]  \n"

  12910.                         "ld1    {v0.4s}, [%10]          \n"

  12911. 

  12912.                         "ld1    {v12.4s, v13.4s}, [%11], #32    \n"

  12913. 

  12914.                         "fmla   v14.4s, v10.4s, v0.s[0] \n"

  12915.                         "fmla   v15.4s, v11.4s, v0.s[0] \n"

  12916. 

  12917.                         "ld1    {v10.4s, v11.4s}, [%11], #32    \n"

  12918. 

  12919.                         "fmla   v8.4s, v12.4s, v0.s[1]  \n"

  12920.                         "fmla   v9.4s, v13.4s, v0.s[1]  \n"

  12921. 

  12922.                         "fmla   v14.4s, v10.4s, v0.s[2] \n"

  12923.                         "fmla   v15.4s, v11.4s, v0.s[2] \n"

  12924. 

  12925.                         "fadd   v8.4s, v8.4s, v14.4s    \n"

  12926.                         "fadd   v9.4s, v9.4s, v15.4s    \n"

  12927. 

  12928.                         "sub    %11, %11, #288          \n"

  12929. 

  12930.                         "st1    {v8.s}[0], [%0], #4     \n"

  12931.                         "st1    {v8.s}[1], [%1], #4     \n"

  12932.                         "st1    {v8.s}[2], [%2], #4     \n"

  12933.                         "st1    {v8.s}[3], [%3], #4     \n"

  12934. 

  12935.                         "st1    {v9.s}[0], [%4], #4     \n"

  12936.                         "st1    {v9.s}[1], [%5], #4     \n"

  12937.                         "st1    {v9.s}[2], [%6], #4     \n"

  12938.                         "st1    {v9.s}[3], [%7], #4     \n"

  12939. 

  12940.                         : "=r"(outptr0),    // %0

  12941.                           "=r"(outptr1),    // %1

  12942.                           "=r"(outptr2),    // %2

  12943.                           "=r"(outptr3),    // %3

  12944.                           "=r"(outptr4),    // %4

  12945.                           "=r"(outptr5),    // %5

  12946.                           "=r"(outptr6),    // %6

  12947.                           "=r"(outptr7),    // %7

  12948.                           "=r"(r0),         // %8

  12949.                           "=r"(r1),         // %9

  12950.                           "=r"(r2),         // %10

  12951.                           "=r"(ktmp)        // %11

  12952.                         : "0"(outptr0),

  12953.                           "1"(outptr1),

  12954.                           "2"(outptr2),

  12955.                           "3"(outptr3),

  12956.                           "4"(outptr4),

  12957.                           "5"(outptr5),

  12958.                           "6"(outptr6),

  12959.                           "7"(outptr7),

  12960.                           "8"(r0),

  12961.                           "9"(r1),

  12962.                           "10"(r2),

  12963.                           "11"(ktmp)

  12964.                         : "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  12965.                     );

  12966. #else // __aarch64__

  12967.                     asm volatile(

  12968.                         "vld1.f32   {d20-d23}, [%11 :128]! \n"

  12969. 

  12970.                         "pld        [%8, #128]      \n"

  12971.                         "vld1.f32   {d0-d1}, [%8]   \n"

  12972. 

  12973.                         "vld1.f32   {d24-d27}, [%11 :128]! \n"

  12974. 

  12975.                         "vld1.f32   {d16[0]}, [%0]  \n"

  12976.                         "vld1.f32   {d16[1]}, [%1]  \n"

  12977.                         "vld1.f32   {d17[0]}, [%2]  \n"

  12978.                         "vld1.f32   {d17[1]}, [%3]  \n"

  12979. 

  12980.                         "vmul.f32   q14, q10, d0[0] \n"

  12981.                         "vmul.f32   q15, q11, d0[0] \n"

  12982. 

  12983.                         "vld1.f32   {d18[0]}, [%4]  \n"

  12984.                         "vld1.f32   {d18[1]}, [%5]  \n"

  12985.                         "vld1.f32   {d19[0]}, [%6]  \n"

  12986.                         "vld1.f32   {d19[1]}, [%7]  \n"

  12987. 

  12988.                         "vld1.f32   {d20-d23}, [%11 :128]! \n"

  12989. 

  12990.                         "vmla.f32   q8, q12, d0[1]  \n"

  12991.                         "vmla.f32   q9, q13, d0[1]  \n"

  12992. 

  12993.                         "vld1.f32   {d24-d27}, [%11 :128]! \n"

  12994. 

  12995.                         "vmla.f32   q14, q10, d1[0] \n"

  12996.                         "vmla.f32   q15, q11, d1[0] \n"

  12997. 

  12998.                         "pld        [%9, #128]      \n"

  12999.                         "vld1.f32   {d2-d3}, [%9]   \n"

  13000. 

  13001.                         "vld1.f32   {d20-d23}, [%11 :128]! \n"

  13002. 

  13003.                         "vmla.f32   q8, q12, d2[0]  \n"

  13004.                         "vmla.f32   q9, q13, d2[0]  \n"

  13005. 

  13006.                         "vld1.f32   {d24-d27}, [%11 :128]! \n"

  13007. 

  13008.                         "vmla.f32   q14, q10, d2[1] \n"

  13009.                         "vmla.f32   q15, q11, d2[1] \n"

  13010. 

  13011.                         "vld1.f32   {d20-d23}, [%11 :128]! \n"

  13012. 

  13013.                         "vmla.f32   q8, q12, d3[0]  \n"

  13014.                         "vmla.f32   q9, q13, d3[0]  \n"

  13015. 

  13016.                         "pld        [%10, #128]     \n"

  13017.                         "vld1.f32   {d0-d1}, [%10]  \n"

  13018. 

  13019.                         "vld1.f32   {d24-d27}, [%11 :128]! \n"

  13020. 

  13021.                         "vmla.f32   q14, q10, d0[0] \n"

  13022.                         "vmla.f32   q15, q11, d0[0] \n"

  13023. 

  13024.                         "vld1.f32   {d20-d23}, [%11 :128]! \n"

  13025. 

  13026.                         "vmla.f32   q8, q12, d0[1]  \n"

  13027.                         "vmla.f32   q9, q13, d0[1]  \n"

  13028. 

  13029.                         "vmla.f32   q14, q10, d1[0] \n"

  13030.                         "vmla.f32   q15, q11, d1[0] \n"

  13031. 

  13032.                         "vadd.f32   q8, q8, q14     \n"

  13033.                         "vadd.f32   q9, q9, q15     \n"

  13034. 

  13035.                         "sub        %11, %11, #288  \n"

  13036. 

  13037.                         "vst1.f32   {d16[0]}, [%0]! \n"

  13038.                         "vst1.f32   {d16[1]}, [%1]! \n"

  13039.                         "vst1.f32   {d17[0]}, [%2]! \n"

  13040.                         "vst1.f32   {d17[1]}, [%3]! \n"

  13041. 

  13042.                         "vst1.f32   {d18[0]}, [%4]! \n"

  13043.                         "vst1.f32   {d18[1]}, [%5]! \n"

  13044.                         "vst1.f32   {d19[0]}, [%6]! \n"

  13045.                         "vst1.f32   {d19[1]}, [%7]! \n"

  13046. 

  13047.                         : "=r"(outptr0),    // %0

  13048.                           "=r"(outptr1),    // %1

  13049.                           "=r"(outptr2),    // %2

  13050.                           "=r"(outptr3),    // %3

  13051.                           "=r"(outptr4),    // %4

  13052.                           "=r"(outptr5),    // %5

  13053.                           "=r"(outptr6),    // %6

  13054.                           "=r"(outptr7),    // %7

  13055.                           "=r"(r0),         // %8

  13056.                           "=r"(r1),         // %9

  13057.                           "=r"(r2),         // %10

  13058.                           "=r"(ktmp)        // %11

  13059.                         : "0"(outptr0),

  13060.                           "1"(outptr1),

  13061.                           "2"(outptr2),

  13062.                           "3"(outptr3),

  13063.                           "4"(outptr4),

  13064.                           "5"(outptr5),

  13065.                           "6"(outptr6),

  13066.                           "7"(outptr7),

  13067.                           "8"(r0),

  13068.                           "9"(r1),

  13069.                           "10"(r2),

  13070.                           "11"(ktmp)

  13071.                         : "memory", "q0", "q1", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  13072.                     );

  13073. #endif // __aarch64__

  13074. #else // __ARM_NEON

  13075.                     float sum0 = 0.f;

  13076.                     float sum1 = 0.f;

  13077.                     float sum2 = 0.f;

  13078.                     float sum3 = 0.f;

  13079.                     float sum4 = 0.f;

  13080.                     float sum5 = 0.f;

  13081.                     float sum6 = 0.f;

  13082.                     float sum7 = 0.f;

  13083. 

  13084.                     sum0 += r0[0] * ktmp[0];

  13085.                     sum1 += r0[0] * ktmp[1];

  13086.                     sum2 += r0[0] * ktmp[2];

  13087.                     sum3 += r0[0] * ktmp[3];

  13088.                     sum4 += r0[0] * ktmp[4];

  13089.                     sum5 += r0[0] * ktmp[5];

  13090.                     sum6 += r0[0] * ktmp[6];

  13091.                     sum7 += r0[0] * ktmp[7];

  13092.                     ktmp += 8;

  13093. 

  13094.                     sum0 += r0[1] * ktmp[0];

  13095.                     sum1 += r0[1] * ktmp[1];

  13096.                     sum2 += r0[1] * ktmp[2];

  13097.                     sum3 += r0[1] * ktmp[3];

  13098.                     sum4 += r0[1] * ktmp[4];

  13099.                     sum5 += r0[1] * ktmp[5];

  13100.                     sum6 += r0[1] * ktmp[6];

  13101.                     sum7 += r0[1] * ktmp[7];

  13102.                     ktmp += 8;

  13103. 

  13104.                     sum0 += r0[2] * ktmp[0];

  13105.                     sum1 += r0[2] * ktmp[1];

  13106.                     sum2 += r0[2] * ktmp[2];

  13107.                     sum3 += r0[2] * ktmp[3];

  13108.                     sum4 += r0[2] * ktmp[4];

  13109.                     sum5 += r0[2] * ktmp[5];

  13110.                     sum6 += r0[2] * ktmp[6];

  13111.                     sum7 += r0[2] * ktmp[7];

  13112.                     ktmp += 8;

  13113. 

  13114.                     sum0 += r1[0] * ktmp[0];

  13115.                     sum1 += r1[0] * ktmp[1];

  13116.                     sum2 += r1[0] * ktmp[2];

  13117.                     sum3 += r1[0] * ktmp[3];

  13118.                     sum4 += r1[0] * ktmp[4];

  13119.                     sum5 += r1[0] * ktmp[5];

  13120.                     sum6 += r1[0] * ktmp[6];

  13121.                     sum7 += r1[0] * ktmp[7];

  13122.                     ktmp += 8;

  13123. 

  13124.                     sum0 += r1[1] * ktmp[0];

  13125.                     sum1 += r1[1] * ktmp[1];

  13126.                     sum2 += r1[1] * ktmp[2];

  13127.                     sum3 += r1[1] * ktmp[3];

  13128.                     sum4 += r1[1] * ktmp[4];

  13129.                     sum5 += r1[1] * ktmp[5];

  13130.                     sum6 += r1[1] * ktmp[6];

  13131.                     sum7 += r1[1] * ktmp[7];

  13132.                     ktmp += 8;

  13133. 

  13134.                     sum0 += r1[2] * ktmp[0];

  13135.                     sum1 += r1[2] * ktmp[1];

  13136.                     sum2 += r1[2] * ktmp[2];

  13137.                     sum3 += r1[2] * ktmp[3];

  13138.                     sum4 += r1[2] * ktmp[4];

  13139.                     sum5 += r1[2] * ktmp[5];

  13140.                     sum6 += r1[2] * ktmp[6];

  13141.                     sum7 += r1[2] * ktmp[7];

  13142.                     ktmp += 8;

  13143. 

  13144.                     sum0 += r2[0] * ktmp[0];

  13145.                     sum1 += r2[0] * ktmp[1];

  13146.                     sum2 += r2[0] * ktmp[2];

  13147.                     sum3 += r2[0] * ktmp[3];

  13148.                     sum4 += r2[0] * ktmp[4];

  13149.                     sum5 += r2[0] * ktmp[5];

  13150.                     sum6 += r2[0] * ktmp[6];

  13151.                     sum7 += r2[0] * ktmp[7];

  13152.                     ktmp += 8;

  13153. 

  13154.                     sum0 += r2[1] * ktmp[0];

  13155.                     sum1 += r2[1] * ktmp[1];

  13156.                     sum2 += r2[1] * ktmp[2];

  13157.                     sum3 += r2[1] * ktmp[3];

  13158.                     sum4 += r2[1] * ktmp[4];

  13159.                     sum5 += r2[1] * ktmp[5];

  13160.                     sum6 += r2[1] * ktmp[6];

  13161.                     sum7 += r2[1] * ktmp[7];

  13162.                     ktmp += 8;

  13163. 

  13164.                     sum0 += r2[2] * ktmp[0];

  13165.                     sum1 += r2[2] * ktmp[1];

  13166.                     sum2 += r2[2] * ktmp[2];

  13167.                     sum3 += r2[2] * ktmp[3];

  13168.                     sum4 += r2[2] * ktmp[4];

  13169.                     sum5 += r2[2] * ktmp[5];

  13170.                     sum6 += r2[2] * ktmp[6];

  13171.                     sum7 += r2[2] * ktmp[7];

  13172.                     ktmp += 8;

  13173. 

  13174.                     *outptr0 += sum0;

  13175.                     *outptr1 += sum1;

  13176.                     *outptr2 += sum2;

  13177.                     *outptr3 += sum3;

  13178.                     *outptr4 += sum4;

  13179.                     *outptr5 += sum5;

  13180.                     *outptr6 += sum6;

  13181.                     *outptr7 += sum7;

  13182. 

  13183.                     ktmp -= 8*9;

  13184. 

  13185.                     outptr0++;

  13186.                     outptr1++;

  13187.                     outptr2++;

  13188.                     outptr3++;

  13189.                     outptr4++;

  13190.                     outptr5++;

  13191.                     outptr6++;

  13192.                     outptr7++;

  13193. #endif // __ARM_NEON

  13194.                     r0 += 2;

  13195.                     r1 += 2;

  13196.                     r2 += 2;

  13197.                 }

  13198. 

  13199.                 r0 += tailstep;

  13200.                 r1 += tailstep;

  13201.                 r2 += tailstep;

  13202.             }

  13203. 

  13204.             ktmp += 8*9;

  13205.         }

  13206.     }

  13207. 

  13208.     #pragma omp parallel for num_threads(opt.num_threads)

  13209.     for (int p=remain_outch_start; p<outch; p++)

  13210.     {

  13211.         Mat out = top_blob.channel(p);

  13212. 

  13213.         const float bias0 = bias ? bias[p] : 0.f;

  13214. 

  13215.         out.fill(bias0);

  13216. 

  13217.         const float* ktmp = _kernel.channel(p/8 + p%8);

  13218. 

  13219.         for (int q=0; q<inch; q++)

  13220.         {

  13221.             float* outptr = out;

  13222. 

  13223.             const float* img0 = bottom_blob.channel(q);

  13224. 

  13225.             const float* r0 = img0;

  13226.             const float* r1 = img0 + w;

  13227.             const float* r2 = img0 + w*2;

  13228. 

  13229.             const float* k0 = ktmp;

  13230.             const float* k1 = ktmp + 3;

  13231.             const float* k2 = ktmp + 6;

  13232. 

  13233. #if __ARM_NEON

  13234.             float32x4_t _k0123 = vld1q_f32(k0);

  13235.             float32x4_t _k3456 = vld1q_f32(k1);

  13236.             float32x4_t _k6789 = vld1q_f32(k2);

  13237. #endif // __ARM_NEON

  13238. 

  13239.             int i = 0;

  13240. 

  13241.             for (; i < outh; i++)

  13242.             {

  13243. #if __ARM_NEON

  13244.                 int nn = outw >> 2;

  13245.                 int remain = outw & 3;

  13246. #else

  13247.                 int remain = outw;

  13248. #endif // __ARM_NEON

  13249. 

  13250. #if __ARM_NEON

  13251. #if __aarch64__

  13252.                 if (nn > 0)

  13253.                 {

  13254.                 asm volatile(

  13255.                     "prfm       pldl1keep, [%2, #256]          \n"

  13256.                     "ld2        {v2.4s, v3.4s}, [%2], #32      \n"

  13257.                     "0:                                        \n"

  13258. 

  13259.                     "prfm       pldl1keep, [%1, #128]          \n"

  13260.                     "ld1        {v0.4s}, [%1]                  \n"

  13261. 

  13262.                     "fmla       v0.4s,  v2.4s, %10.s[0]        \n"

  13263.                     "fmul       v10.4s, v3.4s, %10.s[1]        \n"

  13264. 

  13265.                     "prfm       pldl1keep, [%2, #256]          \n"

  13266.                     "ld2        {v8.4s, v9.4s}, [%2]           \n"

  13267.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  13268. 

  13269.                     "fmul       v11.4s, v1.4s, %10.s[2]        \n"

  13270. 

  13271.                     "prfm       pldl1keep, [%3, #256]          \n"

  13272.                     "ld2        {v2.4s, v3.4s}, [%3], #32      \n"

  13273. 

  13274.                     "fmla       v0.4s,  v2.4s, %11.s[0]        \n"

  13275.                     "fmla       v10.4s, v3.4s, %11.s[1]        \n"

  13276. 

  13277.                     "prfm       pldl1keep, [%3, #256]          \n"

  13278.                     "ld2        {v8.4s, v9.4s}, [%3]           \n"

  13279.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  13280. 

  13281.                     "fmla       v11.4s, v1.4s, %11.s[2]        \n"

  13282. 

  13283.                     "prfm       pldl1keep, [%4, #256]          \n"

  13284.                     "ld2        {v2.4s, v3.4s}, [%4], #32      \n"

  13285. 

  13286.                     "fmla       v0.4s,  v2.4s, %12.s[0]        \n"

  13287.                     "fmla       v10.4s, v3.4s, %12.s[1]        \n"

  13288. 

  13289.                     "prfm       pldl1keep, [%4, #256]          \n"

  13290.                     "ld2        {v8.4s, v9.4s}, [%4]           \n"

  13291.                     "ext        v1.16b, v2.16b, v8.16b, #4     \n"

  13292. 

  13293.                     "fmla       v11.4s, v1.4s, %12.s[2]        \n"

  13294. 

  13295.                     "prfm       pldl1keep, [%2, #256]          \n"

  13296.                     "ld2        {v2.4s, v3.4s}, [%2], #32      \n"

  13297. 

  13298.                     "fadd       v0.4s, v0.4s, v10.4s           \n"

  13299.                     "fadd       v0.4s, v0.4s, v11.4s           \n"

  13300. 

  13301.                     "subs       %w0, %w0, #1                   \n"

  13302.                     "st1        {v0.4s}, [%1], #16             \n"

  13303.                     "bne        0b                             \n"

  13304.                     "sub        %2, %2, #32                    \n"

  13305.                     : "=r"(nn),     // %0

  13306.                       "=r"(outptr), // %1

  13307.                       "=r"(r0),     // %2

  13308.                       "=r"(r1),     // %3

  13309.                       "=r"(r2)      // %4

  13310.                     : "0"(nn),

  13311.                       "1"(outptr),

  13312.                       "2"(r0),

  13313.                       "3"(r1),

  13314.                       "4"(r2),

  13315.                       "w"(_k0123),  // %10

  13316.                       "w"(_k3456),  // %11

  13317.                       "w"(_k6789)   // %12

  13318.                     : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  13319.                 );

  13320.                 }

  13321. #else

  13322.                 if (nn > 0)

  13323.                 {

  13324.                 asm volatile(

  13325.                     "pld        [%2, #256]          \n"

  13326.                     "vld2.f32   {d4-d7}, [%2]!      \n"

  13327. 

  13328.                     "0:                             \n"

  13329.                     "pld        [%1, #128]          \n"

  13330.                     "vld1.f32   {d0-d1}, [%1]       \n"

  13331. 

  13332.                     "vmla.f32   q0, q2, %e10[0]     \n"

  13333.                     "vmul.f32   q10, q3, %e10[1]    \n"

  13334. 

  13335.                     "pld        [%2, #128]          \n"

  13336.                     "vld2.f32   {d16-d17}, [%2]     \n"

  13337.                     "vext.32    q1, q2, q8, #1      \n"

  13338. 

  13339.                     "vmul.f32   q11, q1, %f10[0]    \n"

  13340. 

  13341.                     "pld        [%3, #256]          \n"

  13342.                     "vld2.f32   {d4-d7}, [%3]!      \n"

  13343. 

  13344.                     "vmla.f32   q0, q2, %e11[0]     \n"

  13345.                     "vmla.f32   q10, q3, %e11[1]    \n"

  13346. 

  13347.                     "pld        [%3, #128]          \n"

  13348.                     "vld2.f32   {d16-d17}, [%3]     \n"

  13349.                     "vext.32    q1, q2, q8, #1      \n"

  13350. 

  13351.                     "vmla.f32   q11, q1, %f11[0]    \n"

  13352. 

  13353.                     "pld        [%4, #256]          \n"

  13354.                     "vld2.f32   {d4-d7}, [%4]!      \n"

  13355. 

  13356.                     "vmla.f32   q0, q2, %e12[0]     \n"

  13357.                     "vmla.f32   q10, q3, %e12[1]    \n"

  13358. 

  13359.                     "pld        [%4, #128]          \n"

  13360.                     "vld2.f32   {d16-d17}, [%4]     \n"

  13361.                     "vext.32    q1, q2, q8, #1      \n"

  13362. 

  13363.                     "vmla.f32   q11, q1, %f12[0]    \n"

  13364. 

  13365.                     "pld        [%2, #256]          \n"

  13366.                     "vld2.f32   {d4-d7}, [%2]!      \n"

  13367. 

  13368.                     "vadd.f32   q0, q0, q10         \n"

  13369.                     "vadd.f32   q0, q0, q11         \n"

  13370. 

  13371.                     "subs       %0, #1              \n"

  13372.                     "vst1.f32   {d0-d1}, [%1]!      \n"

  13373.                     "bne        0b                  \n"

  13374.                     "sub        %2, #32             \n"

  13375.                     : "=r"(nn),     // %0

  13376.                       "=r"(outptr), // %1

  13377.                       "=r"(r0),     // %2

  13378.                       "=r"(r1),     // %3

  13379.                       "=r"(r2)      // %4

  13380.                     : "0"(nn),

  13381.                       "1"(outptr),

  13382.                       "2"(r0),

  13383.                       "3"(r1),

  13384.                       "4"(r2),

  13385.                       "w"(_k0123),  // %10

  13386.                       "w"(_k3456),  // %11

  13387.                       "w"(_k6789)   // %12

  13388.                     : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  13389.                 );

  13390.                 }

  13391. #endif // __aarch64__

  13392. #endif // __ARM_NEON

  13393.                 for (; remain>0; remain--)

  13394.                 {

  13395. #if __ARM_NEON

  13396.                     float32x4_t _r00 = vld1q_f32(r0);

  13397.                     float32x4_t _r10 = vld1q_f32(r1);

  13398.                     float32x4_t _r20 = vld1q_f32(r2);

  13399. 

  13400.                     float32x4_t _sum = vmulq_f32(_r00, _k0123);

  13401.                     _sum = vmlaq_f32(_sum, _r10, _k3456);

  13402.                     _sum = vmlaq_f32(_sum, _r20, _k6789);

  13403. 

  13404.                     _sum = vsetq_lane_f32(*outptr, _sum, 3);

  13405. 

  13406. #if __aarch64__

  13407.                     *outptr = vaddvq_f32(_sum);

  13408. #else

  13409.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  13410.                     _ss = vpadd_f32(_ss, _ss);

  13411. 

  13412.                     *outptr = vget_lane_f32(_ss, 0);

  13413. #endif // __aarch64__

  13414. #else

  13415.                     float sum = 0;

  13416. 

  13417.                     sum += r0[0] * ktmp[0];

  13418.                     sum += r0[1] * ktmp[1];

  13419.                     sum += r0[2] * ktmp[2];

  13420.                     sum += r1[0] * ktmp[3];

  13421.                     sum += r1[1] * ktmp[4];

  13422.                     sum += r1[2] * ktmp[5];

  13423.                     sum += r2[0] * ktmp[6];

  13424.                     sum += r2[1] * ktmp[7];

  13425.                     sum += r2[2] * ktmp[8];

  13426. 

  13427.                     *outptr += sum;

  13428. #endif // __ARM_NEON

  13429. 

  13430.                     r0 += 2;

  13431.                     r1 += 2;

  13432.                     r2 += 2;

  13433.                     outptr++;

  13434.                 }

  13435. 

  13436.                 r0 += tailstep;

  13437.                 r1 += tailstep;

  13438.                 r2 += tailstep;

  13439.             }

  13440. 

  13441.             ktmp += 9;

  13442.         }

  13443.     }

  13444. }