nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 6ebb774e37
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '6ebb774e37'
${ noResults }
ncnn/src/layer/arm/convolution_7x7.h

1396 lines
69 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. static void conv7x7s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  16. {

  17.     int w = bottom_blob.w;

  18.     int inch = bottom_blob.c;

  19. 

  20.     int outw = top_blob.w;

  21.     int outh = top_blob.h;

  22.     int outch = top_blob.c;

  23. 

  24.     const float* kernel = _kernel;

  25.     const float* bias = _bias;

  26. 

  27.     #pragma omp parallel for num_threads(opt.num_threads)

  28.     for (int p = 0; p < outch; p++)

  29.     {

  30.         Mat out = top_blob.channel(p);

  31. 

  32.         const float bias0 = bias ? bias[p] : 0.f;

  33. 

  34.         out.fill(bias0);

  35. 

  36.         for (int q = 0; q < inch; q++)

  37.         {

  38.             float* outptr = out;

  39. 

  40.             const float* img0 = bottom_blob.channel(q);

  41. 

  42.             const float* kernel0 = kernel + p * inch * 49 + q * 49;

  43. 

  44.             const float* r0 = img0;

  45.             const float* r1 = img0 + w;

  46.             const float* r2 = img0 + w * 2;

  47.             const float* r3 = img0 + w * 3;

  48.             const float* r4 = img0 + w * 4;

  49.             const float* r5 = img0 + w * 5;

  50.             const float* r6 = img0 + w * 6;

  51. 

  52.             const float* k0 = kernel0;

  53.             const float* k1 = kernel0 + 7;

  54.             const float* k2 = kernel0 + 14;

  55.             const float* k3 = kernel0 + 21;

  56.             const float* k4 = kernel0 + 28;

  57.             const float* k5 = kernel0 + 35;

  58.             const float* k6 = kernel0 + 42;

  59. 

  60.             int i = 0;

  61. 

  62.             for (; i < outh; i++)

  63.             {

  64. #if __ARM_NEON

  65.                 int nn = outw >> 2;

  66.                 int remain = outw - (nn << 2);

  67. #else

  68.                 int remain = outw;

  69. #endif // __ARM_NEON

  70. 

  71. #if __ARM_NEON

  72. #if __aarch64__

  73.                 float32x4_t _k0123 = vld1q_f32(k0);

  74.                 float32x4_t _k4567 = vld1q_f32(k0 + 4);

  75.                 float32x4_t _k78910 = vld1q_f32(k1);

  76.                 float32x4_t _k11121314 = vld1q_f32(k1 + 4);

  77.                 float32x4_t _k14151617 = vld1q_f32(k2);

  78.                 float32x4_t _k18192021 = vld1q_f32(k2 + 4);

  79.                 float32x4_t _k21222324 = vld1q_f32(k3);

  80.                 float32x4_t _k25262728 = vld1q_f32(k3 + 4);

  81.                 float32x4_t _k28293031 = vld1q_f32(k4);

  82.                 float32x4_t _k32333435 = vld1q_f32(k4 + 4);

  83.                 float32x4_t _k35363738 = vld1q_f32(k5);

  84.                 float32x4_t _k39404142 = vld1q_f32(k5 + 4);

  85.                 float32x4_t _k42434445 = vld1q_f32(k6);

  86.                 float32x4_t _k46474849 = vld1q_f32(k6 + 4);

  87. #ifdef __clang__ // __ARM_NEON && __aarch64__ && __clang__

  88.                 if (nn > 0)

  89.                 {

  90.                     asm volatile(

  91.                         // v0:  input / final output

  92.                         // v1 v2 v3: = ri0 ri4 ri0n , i <-  1-7

  93.                         // v4 = ri1 / ri3 / ri6

  94.                         // v5 = ri2 / ri5

  95.                         // v9 = intermediate sum register

  96.                         "0:                                        \n"

  97.                         "prfm       pldl1keep, [%1, #128]          \n"

  98.                         "ld1        {v0.4s}, [%1]                  \n"

  99. 

  100.                         //i = 1

  101.                         "prfm       pldl1keep, [%2, #384]          \n"

  102.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%2]    \n"

  103.                         "add        %2, %2, #16                    \n"

  104.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  105.                         "fmul       v9.4s, v1.4s, %18.s[0]         \n"

  106.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  107.                         "fmla       v0.4s, v4.4s, %18.s[1]         \n"

  108.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  109.                         "fmla       v9.4s, v5.4s, %18.s[2]         \n"

  110.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  111.                         "fmla       v0.4s, v4.4s, %18.s[3]         \n"

  112.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  113.                         "fmla       v9.4s, v2.4s, %19.s[0]         \n"

  114.                         "fmla       v0.4s, v5.4s, %19.s[1]         \n"

  115.                         "fmla       v9.4s, v4.4s, %19.s[2]         \n"

  116. 

  117.                         //i = 2

  118.                         "prfm       pldl1keep, [%3, #384]          \n"

  119.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%3]    \n" // v1 v2 v3: = r20 r24 r20n

  120.                         "add        %3, %3, #16                    \n"

  121.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n" // v4 = r21

  122.                         "fmla       v9.4s, v1.4s, %20.s[0]         \n" // *+ r10

  123.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n" // v5 = r22

  124.                         "fmla       v0.4s, v4.4s, %20.s[1]         \n" // *+ r11

  125.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n" // v4 = r23

  126.                         "fmla       v9.4s, v5.4s, %20.s[2]         \n" // *+ r1

  127.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n" // v5 = r25

  128.                         "fmla       v0.4s, v4.4s, %20.s[3]         \n" // *+ r13

  129.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n" // v4 = r26

  130.                         "fmla       v9.4s, v2.4s, %21.s[0]         \n" // *+ r14

  131.                         "fmla       v0.4s, v5.4s, %21.s[1]         \n" // *+ r15

  132.                         "fmla       v9.4s, v4.4s, %21.s[2]         \n" // *+ r16

  133. 

  134.                         //i = 3

  135.                         "prfm       pldl1keep, [%4, #384]          \n"

  136.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%4]    \n"

  137.                         "add        %4, %4, #16                    \n"

  138.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  139.                         "fmla       v9.4s, v1.4s, %22.s[0]         \n"

  140.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  141.                         "fmla       v0.4s, v4.4s, %22.s[1]         \n"

  142.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  143.                         "fmla       v9.4s, v5.4s, %22.s[2]         \n"

  144.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  145.                         "fmla       v0.4s, v4.4s, %22.s[3]         \n"

  146.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  147.                         "fmla       v9.4s, v2.4s, %23.s[0]         \n"

  148.                         "fmla       v0.4s, v5.4s, %23.s[1]         \n"

  149.                         "fmla       v9.4s, v4.4s, %23.s[2]         \n"

  150. 

  151.                         //i = 4

  152.                         "prfm       pldl1keep, [%5, #384]          \n"

  153.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%5]    \n"

  154.                         "add        %5, %5, #16                    \n"

  155.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  156.                         "fmla       v9.4s, v1.4s, %24.s[0]         \n"

  157.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  158.                         "fmla       v0.4s, v4.4s, %24.s[1]         \n"

  159.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  160.                         "fmla       v9.4s, v5.4s, %24.s[2]         \n"

  161.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  162.                         "fmla       v0.4s, v4.4s, %24.s[3]         \n"

  163.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  164.                         "fmla       v9.4s, v2.4s, %25.s[0]         \n"

  165.                         "fmla       v0.4s, v5.4s, %25.s[1]         \n"

  166.                         "fmla       v9.4s, v4.4s, %25.s[2]         \n"

  167. 

  168.                         //i = 5

  169.                         "prfm       pldl1keep, [%6, #384]          \n"

  170.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%6]    \n"

  171.                         "add        %6, %6, #16                    \n"

  172.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  173.                         "fmla       v9.4s, v1.4s, %26.s[0]         \n"

  174.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  175.                         "fmla       v0.4s, v4.4s, %26.s[1]         \n"

  176.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  177.                         "fmla       v9.4s, v5.4s, %26.s[2]         \n"

  178.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  179.                         "fmla       v0.4s, v4.4s, %26.s[3]         \n"

  180.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  181.                         "fmla       v9.4s, v2.4s, %27.s[0]         \n"

  182.                         "fmla       v0.4s, v5.4s, %27.s[1]         \n"

  183.                         "fmla       v9.4s, v4.4s, %27.s[2]         \n"

  184. 

  185.                         //i = 6

  186.                         "prfm       pldl1keep, [%7, #384]          \n"

  187.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%7]    \n"

  188.                         "add        %7, %7, #16                    \n"

  189.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  190.                         "fmla       v9.4s, v1.4s, %28.s[0]         \n"

  191.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  192.                         "fmla       v0.4s, v4.4s, %28.s[1]         \n"

  193.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  194.                         "fmla       v9.4s, v5.4s, %28.s[2]         \n"

  195.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  196.                         "fmla       v0.4s, v4.4s, %28.s[3]         \n"

  197.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  198.                         "fmla       v9.4s, v2.4s, %29.s[0]         \n"

  199.                         "fmla       v0.4s, v5.4s, %29.s[1]         \n"

  200.                         "fmla       v9.4s, v4.4s, %29.s[2]         \n"

  201. 

  202.                         //i = 7

  203.                         "prfm       pldl1keep, [%8, #384]          \n"

  204.                         "ld1        {v1.4s, v2.4s, v3.4s}, [%8]    \n"

  205.                         "add        %8, %8, #16                    \n"

  206.                         "ext        v4.16b, v1.16b, v2.16b, #4     \n"

  207.                         "fmla       v9.4s, v1.4s, %30.s[0]         \n"

  208.                         "ext        v5.16b, v1.16b, v2.16b, #8     \n"

  209.                         "fmla       v0.4s, v4.4s, %30.s[1]         \n"

  210.                         "ext        v4.16b, v1.16b, v2.16b, #12    \n"

  211.                         "fmla       v9.4s, v5.4s, %30.s[2]         \n"

  212.                         "ext        v5.16b, v2.16b, v3.16b, #4     \n"

  213.                         "fmla       v0.4s, v4.4s, %30.s[3]         \n"

  214.                         "ext        v4.16b, v2.16b, v3.16b, #8     \n"

  215.                         "fmla       v9.4s, v2.4s, %31.s[0]         \n"

  216.                         "fmla       v0.4s, v5.4s, %31.s[1]         \n"

  217.                         "fmla       v9.4s, v4.4s, %31.s[2]         \n"

  218. 

  219.                         "fadd       v0.4s, v0.4s, v9.4s            \n"

  220.                         "st1        {v0.4s}, [%1], #16             \n"

  221.                         "subs       %w0, %w0, #1                   \n"

  222.                         "bne        0b                             \n"

  223. 

  224.                         : "=r"(nn),     // %0

  225.                         "=r"(outptr), // %1

  226.                         "=r"(r0),     // %2

  227.                         "=r"(r1),     // %3

  228.                         "=r"(r2),     // %4

  229.                         "=r"(r3),     // %5

  230.                         "=r"(r4),     // %6

  231.                         "=r"(r5),     // %7

  232.                         "=r"(r6)      // %8

  233.                         : "0"(nn),

  234.                         "1"(outptr),

  235.                         "2"(r0),

  236.                         "3"(r1),

  237.                         "4"(r2),

  238.                         "5"(r3),

  239.                         "6"(r4),

  240.                         "7"(r5),

  241.                         "8"(r6),

  242.                         "w"(_k0123),     // %18

  243.                         "w"(_k4567),     // %19

  244.                         "w"(_k78910),    // %20

  245.                         "w"(_k11121314), // %21

  246.                         "w"(_k14151617), // %22

  247.                         "w"(_k18192021), // %23

  248.                         "w"(_k21222324), // %24

  249.                         "w"(_k25262728), // %25

  250.                         "w"(_k28293031), // %26

  251.                         "w"(_k32333435), // %27

  252.                         "w"(_k35363738), // %28

  253.                         "w"(_k39404142), // %29

  254.                         "w"(_k42434445), // %30

  255.                         "w"(_k46474849)  // %31

  256.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v9");

  257.                 }

  258. #else  // __ARM_NEON && __aarch64__ defined, but __clang__ not defined \

  259. // When compiled with gcc, gcc does not accept over 30 operands

  260.                 for (; nn > 0; nn--)

  261.                 {

  262.                     float32x4_t _sum = vld1q_f32(outptr);

  263. 

  264.                     float32x4_t _r00 = vld1q_f32(r0);             // 0 1 2 3

  265.                     float32x4_t _r04 = vld1q_f32(r0 + 4);         // 4 5 6 7

  266.                     float32x4_t _r00n = vld1q_f32(r0 + 8);        // 8 9 10 11

  267.                     float32x4_t _r01 = vextq_f32(_r00, _r04, 1);  // 1 2 3 4

  268.                     float32x4_t _r02 = vextq_f32(_r00, _r04, 2);  // 2 3 4 5

  269.                     float32x4_t _r03 = vextq_f32(_r00, _r04, 3);  // 3 4 5 6

  270.                     float32x4_t _r05 = vextq_f32(_r04, _r00n, 1); // 5 6 7 8

  271.                     float32x4_t _r06 = vextq_f32(_r04, _r00n, 2); // 6 7 8 9

  272. 

  273.                     _sum = vfmaq_laneq_f32(_sum, _r00, _k0123, 0);

  274.                     _sum = vfmaq_laneq_f32(_sum, _r01, _k0123, 1);

  275.                     _sum = vfmaq_laneq_f32(_sum, _r02, _k0123, 2);

  276.                     _sum = vfmaq_laneq_f32(_sum, _r03, _k0123, 3);

  277.                     _sum = vfmaq_laneq_f32(_sum, _r04, _k4567, 0);

  278.                     _sum = vfmaq_laneq_f32(_sum, _r05, _k4567, 1);

  279.                     _sum = vfmaq_laneq_f32(_sum, _r06, _k4567, 2);

  280. 

  281.                     float32x4_t _r10 = vld1q_f32(r1);

  282.                     float32x4_t _r14 = vld1q_f32(r1 + 4);

  283.                     float32x4_t _r10n = vld1q_f32(r1 + 8);

  284.                     float32x4_t _r11 = vextq_f32(_r10, _r14, 1);

  285.                     float32x4_t _r12 = vextq_f32(_r10, _r14, 2);

  286.                     float32x4_t _r13 = vextq_f32(_r10, _r14, 3);

  287.                     float32x4_t _r15 = vextq_f32(_r14, _r10n, 1);

  288.                     float32x4_t _r16 = vextq_f32(_r14, _r10n, 2);

  289. 

  290.                     _sum = vfmaq_laneq_f32(_sum, _r10, _k78910, 0);

  291.                     _sum = vfmaq_laneq_f32(_sum, _r11, _k78910, 1);

  292.                     _sum = vfmaq_laneq_f32(_sum, _r12, _k78910, 2);

  293.                     _sum = vfmaq_laneq_f32(_sum, _r13, _k78910, 3);

  294.                     _sum = vfmaq_laneq_f32(_sum, _r14, _k11121314, 0);

  295.                     _sum = vfmaq_laneq_f32(_sum, _r15, _k11121314, 1);

  296.                     _sum = vfmaq_laneq_f32(_sum, _r16, _k11121314, 2);

  297. 

  298.                     float32x4_t _r20 = vld1q_f32(r2);

  299.                     float32x4_t _r24 = vld1q_f32(r2 + 4);

  300.                     float32x4_t _r20n = vld1q_f32(r2 + 8);

  301.                     float32x4_t _r21 = vextq_f32(_r20, _r24, 1);

  302.                     float32x4_t _r22 = vextq_f32(_r20, _r24, 2);

  303.                     float32x4_t _r23 = vextq_f32(_r20, _r24, 3);

  304.                     float32x4_t _r25 = vextq_f32(_r24, _r20n, 1);

  305.                     float32x4_t _r26 = vextq_f32(_r24, _r20n, 2);

  306. 

  307.                     _sum = vfmaq_laneq_f32(_sum, _r20, _k14151617, 0);

  308.                     _sum = vfmaq_laneq_f32(_sum, _r21, _k14151617, 1);

  309.                     _sum = vfmaq_laneq_f32(_sum, _r22, _k14151617, 2);

  310.                     _sum = vfmaq_laneq_f32(_sum, _r23, _k14151617, 3);

  311.                     _sum = vfmaq_laneq_f32(_sum, _r24, _k18192021, 0);

  312.                     _sum = vfmaq_laneq_f32(_sum, _r25, _k18192021, 1);

  313.                     _sum = vfmaq_laneq_f32(_sum, _r26, _k18192021, 2);

  314. 

  315.                     float32x4_t _r30 = vld1q_f32(r3);

  316.                     float32x4_t _r34 = vld1q_f32(r3 + 4);

  317.                     float32x4_t _r30n = vld1q_f32(r3 + 8);

  318.                     float32x4_t _r31 = vextq_f32(_r30, _r34, 1);

  319.                     float32x4_t _r32 = vextq_f32(_r30, _r34, 2);

  320.                     float32x4_t _r33 = vextq_f32(_r30, _r34, 3);

  321.                     float32x4_t _r35 = vextq_f32(_r34, _r30n, 1);

  322.                     float32x4_t _r36 = vextq_f32(_r34, _r30n, 2);

  323. 

  324.                     _sum = vfmaq_laneq_f32(_sum, _r30, _k21222324, 0);

  325.                     _sum = vfmaq_laneq_f32(_sum, _r31, _k21222324, 1);

  326.                     _sum = vfmaq_laneq_f32(_sum, _r32, _k21222324, 2);

  327.                     _sum = vfmaq_laneq_f32(_sum, _r33, _k21222324, 3);

  328.                     _sum = vfmaq_laneq_f32(_sum, _r34, _k25262728, 0);

  329.                     _sum = vfmaq_laneq_f32(_sum, _r35, _k25262728, 1);

  330.                     _sum = vfmaq_laneq_f32(_sum, _r36, _k25262728, 2);

  331. 

  332.                     float32x4_t _r40 = vld1q_f32(r4);

  333.                     float32x4_t _r44 = vld1q_f32(r4 + 4);

  334.                     float32x4_t _r40n = vld1q_f32(r4 + 8);

  335.                     float32x4_t _r41 = vextq_f32(_r40, _r44, 1);

  336.                     float32x4_t _r42 = vextq_f32(_r40, _r44, 2);

  337.                     float32x4_t _r43 = vextq_f32(_r40, _r44, 3);

  338.                     float32x4_t _r45 = vextq_f32(_r44, _r40n, 1);

  339.                     float32x4_t _r46 = vextq_f32(_r44, _r40n, 2);

  340. 

  341.                     _sum = vfmaq_laneq_f32(_sum, _r40, _k28293031, 0);

  342.                     _sum = vfmaq_laneq_f32(_sum, _r41, _k28293031, 1);

  343.                     _sum = vfmaq_laneq_f32(_sum, _r42, _k28293031, 2);

  344.                     _sum = vfmaq_laneq_f32(_sum, _r43, _k28293031, 3);

  345.                     _sum = vfmaq_laneq_f32(_sum, _r44, _k32333435, 0);

  346.                     _sum = vfmaq_laneq_f32(_sum, _r45, _k32333435, 1);

  347.                     _sum = vfmaq_laneq_f32(_sum, _r46, _k32333435, 2);

  348. 

  349.                     float32x4_t _r50 = vld1q_f32(r5);

  350.                     float32x4_t _r54 = vld1q_f32(r5 + 4);

  351.                     float32x4_t _r50n = vld1q_f32(r5 + 8);

  352.                     float32x4_t _r51 = vextq_f32(_r50, _r54, 1);

  353.                     float32x4_t _r52 = vextq_f32(_r50, _r54, 2);

  354.                     float32x4_t _r53 = vextq_f32(_r50, _r54, 3);

  355.                     float32x4_t _r55 = vextq_f32(_r54, _r50n, 1);

  356.                     float32x4_t _r56 = vextq_f32(_r54, _r50n, 2);

  357. 

  358.                     _sum = vfmaq_laneq_f32(_sum, _r50, _k35363738, 0);

  359.                     _sum = vfmaq_laneq_f32(_sum, _r51, _k35363738, 1);

  360.                     _sum = vfmaq_laneq_f32(_sum, _r52, _k35363738, 2);

  361.                     _sum = vfmaq_laneq_f32(_sum, _r53, _k35363738, 3);

  362.                     _sum = vfmaq_laneq_f32(_sum, _r54, _k39404142, 0);

  363.                     _sum = vfmaq_laneq_f32(_sum, _r55, _k39404142, 1);

  364.                     _sum = vfmaq_laneq_f32(_sum, _r56, _k39404142, 2);

  365. 

  366.                     float32x4_t _r60 = vld1q_f32(r6);

  367.                     float32x4_t _r64 = vld1q_f32(r6 + 4);

  368.                     float32x4_t _r60n = vld1q_f32(r6 + 8);

  369.                     float32x4_t _r61 = vextq_f32(_r60, _r64, 1);

  370.                     float32x4_t _r62 = vextq_f32(_r60, _r64, 2);

  371.                     float32x4_t _r63 = vextq_f32(_r60, _r64, 3);

  372.                     float32x4_t _r65 = vextq_f32(_r64, _r60n, 1);

  373.                     float32x4_t _r66 = vextq_f32(_r64, _r60n, 2);

  374. 

  375.                     _sum = vfmaq_laneq_f32(_sum, _r60, _k42434445, 0);

  376.                     _sum = vfmaq_laneq_f32(_sum, _r61, _k42434445, 1);

  377.                     _sum = vfmaq_laneq_f32(_sum, _r62, _k42434445, 2);

  378.                     _sum = vfmaq_laneq_f32(_sum, _r63, _k42434445, 3);

  379.                     _sum = vfmaq_laneq_f32(_sum, _r64, _k46474849, 0);

  380.                     _sum = vfmaq_laneq_f32(_sum, _r65, _k46474849, 1);

  381.                     _sum = vfmaq_laneq_f32(_sum, _r66, _k46474849, 2);

  382. 

  383.                     vst1q_f32(outptr, _sum);

  384. 

  385.                     r0 += 4;

  386.                     r1 += 4;

  387.                     r2 += 4;

  388.                     r3 += 4;

  389.                     r4 += 4;

  390.                     r5 += 4;

  391.                     r6 += 4;

  392.                     outptr += 4;

  393.                 }

  394. #endif // __clang__

  395. #else  //__aarch32__

  396.                 if (nn > 0)

  397.                 {

  398.                     asm volatile(

  399.                         "0:                             \n"

  400. 

  401.                         "pld        [%1, #256]          \n"

  402.                         "vld1.f32   {d24-d25}, [%1]     \n" // _sum

  403.                         //                     "veor       q13, q13            \n"// _sum2 = 0;

  404.                         //                     "veor       q14, q14            \n"// _sum3 = 0;

  405.                         //                     "veor       q15, q15            \n"// _sum4 = 0;

  406. 

  407.                         "pld        [%9, #256]          \n"

  408.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k0123 k4567

  409.                         "add        %9, #28             \n"

  410. 

  411.                         "pld        [%2, #128]          \n"

  412.                         "vld1.f32   {d0-d1}, [%2]!      \n" // q0 = 0  1  2  3

  413.                         "vmla.f32   q12, q0, d8[0]      \n"

  414. 

  415.                         "pld        [%2, #256]          \n"

  416.                         "vld1.f32   {d4-d7}, [%2]       \n" // q2 = 4  5  6  7  q3 = 8  9 10 11

  417.                         "vmul.f32   q13, q2, d10[0]     \n"

  418. 

  419.                         "vext.32    q1, q0, q2, #1      \n" // q1 = 1  2  3  4

  420.                         "vext.32    q10, q2, q3, #1     \n" // q10= 5  6  7  8

  421.                         "vmul.f32   q14, q1, d8[1]      \n"

  422.                         "vmul.f32   q15, q10, d10[1]    \n"

  423. 

  424.                         "vext.32    q8, q0, q2, #2      \n" // q8 = 2  3  4  5

  425.                         "vext.32    q11, q2, q3, #2     \n" // q11= 6  7  8  9

  426.                         "vmla.f32   q12, q8, d9[0]      \n"

  427.                         "vmla.f32   q13, q11, d11[0]    \n"

  428. 

  429.                         "vext.32    q9, q0, q2, #3      \n" // q9 = 3  4  5  6

  430.                         "vmla.f32   q14, q9, d9[1]      \n"

  431. 

  432.                         "pld        [%9, #256]          \n"

  433.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k78910 k11121314

  434.                         "add        %9, #28             \n"

  435. 

  436.                         "pld        [%3, #128]          \n"

  437.                         "vld1.f32   {d0-d1}, [%3]!      \n"

  438.                         "vmla.f32   q15, q0, d12[0]     \n"

  439. 

  440.                         "pld        [%3, #256]          \n"

  441.                         "vld1.f32   {d4-d7}, [%3]       \n"

  442.                         "vmla.f32   q12, q2, d14[0]     \n"

  443. 

  444.                         "vext.32    q1, q0, q2, #1      \n"

  445.                         "vext.32    q10, q2, q3, #1     \n"

  446.                         "vmla.f32   q13, q1, d12[1]     \n"

  447.                         "vmla.f32   q14, q10, d14[1]    \n"

  448. 

  449.                         "vext.32    q8, q0, q2, #2      \n"

  450.                         "vext.32    q11, q2, q3, #2     \n"

  451.                         "vmla.f32   q15, q8, d13[0]     \n"

  452.                         "vmla.f32   q12, q11, d15[0]    \n"

  453. 

  454.                         "vext.32    q9, q0, q2, #3      \n"

  455.                         "vmla.f32   q13, q9, d13[1]     \n"

  456. 

  457.                         "pld        [%9, #256]          \n"

  458.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k14151617 k18192021

  459.                         "add        %9, #28             \n"

  460. 

  461.                         "pld        [%4, #128]          \n"

  462.                         "vld1.f32   {d0-d1}, [%4]!      \n"

  463.                         "vmla.f32   q14, q0, d8[0]      \n"

  464. 

  465.                         "pld        [%4, #256]          \n"

  466.                         "vld1.f32   {d4-d7}, [%4]       \n"

  467.                         "vmla.f32   q15, q2, d10[0]     \n"

  468. 

  469.                         "vext.32    q1, q0, q2, #1      \n"

  470.                         "vext.32    q10, q2, q3, #1     \n"

  471.                         "vmla.f32   q12, q1, d8[1]      \n"

  472.                         "vmla.f32   q13, q10, d10[1]    \n"

  473. 

  474.                         "vext.32    q8, q0, q2, #2      \n"

  475.                         "vext.32    q11, q2, q3, #2     \n"

  476.                         "vmla.f32   q14, q8, d9[0]      \n"

  477.                         "vmla.f32   q15, q11, d11[0]    \n"

  478. 

  479.                         "vext.32    q9, q0, q2, #3      \n"

  480.                         "vmla.f32   q12, q9, d9[1]      \n"

  481. 

  482.                         "pld        [%9, #256]          \n"

  483.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k21222324 k25262728

  484.                         "add        %9, #28             \n"

  485. 

  486.                         "pld        [%5, #128]          \n"

  487.                         "vld1.f32   {d0-d1}, [%5]!      \n"

  488.                         "vmla.f32   q13, q0, d12[0]     \n"

  489. 

  490.                         "pld        [%5, #256]          \n"

  491.                         "vld1.f32   {d4-d7}, [%5]       \n"

  492.                         "vmla.f32   q14, q2, d14[0]     \n"

  493. 

  494.                         "vext.32    q1, q0, q2, #1      \n"

  495.                         "vext.32    q10, q2, q3, #1     \n"

  496.                         "vmla.f32   q15, q1, d12[1]     \n"

  497.                         "vmla.f32   q12, q10, d14[1]    \n"

  498. 

  499.                         "vext.32    q8, q0, q2, #2      \n"

  500.                         "vext.32    q11, q2, q3, #2     \n"

  501.                         "vmla.f32   q13, q8, d13[0]     \n"

  502.                         "vmla.f32   q14, q11, d15[0]    \n"

  503. 

  504.                         "vext.32    q9, q0, q2, #3      \n"

  505.                         "vmla.f32   q15, q9, d13[1]     \n"

  506. 

  507.                         "pld        [%9, #256]          \n"

  508.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k28293031 k32333435

  509.                         "add        %9, #28             \n"

  510. 

  511.                         "pld        [%6, #128]          \n"

  512.                         "vld1.f32   {d0-d1}, [%6]!      \n"

  513.                         "vmla.f32   q12, q0, d8[0]      \n"

  514. 

  515.                         "pld        [%6, #256]          \n"

  516.                         "vld1.f32   {d4-d7}, [%6]       \n"

  517.                         "vmla.f32   q13, q2, d10[0]     \n"

  518. 

  519.                         "vext.32    q1, q0, q2, #1      \n"

  520.                         "vext.32    q10, q2, q3, #1     \n"

  521.                         "vmla.f32   q14, q1, d8[1]      \n"

  522.                         "vmla.f32   q15, q10, d10[1]    \n"

  523. 

  524.                         "vext.32    q8, q0, q2, #2      \n"

  525.                         "vext.32    q11, q2, q3, #2     \n"

  526.                         "vmla.f32   q12, q8, d9[0]      \n"

  527.                         "vmla.f32   q13, q11, d11[0]    \n"

  528. 

  529.                         "vext.32    q9, q0, q2, #3      \n"

  530.                         "vmla.f32   q14, q9, d9[1]      \n"

  531. 

  532.                         "pld        [%9, #256]          \n"

  533.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k35363738 k39404142

  534.                         "add        %9, #28             \n"

  535. 

  536.                         "pld        [%7, #128]          \n"

  537.                         "vld1.f32   {d0-d1}, [%7]!      \n"

  538.                         "vmla.f32   q15, q0, d12[0]     \n"

  539. 

  540.                         "pld        [%7, #256]          \n"

  541.                         "vld1.f32   {d4-d7}, [%7]       \n"

  542.                         "vmla.f32   q12, q2, d14[0]     \n"

  543. 

  544.                         "vext.32    q1, q0, q2, #1      \n"

  545.                         "vext.32    q10, q2, q3, #1     \n"

  546.                         "vmla.f32   q13, q1, d12[1]     \n"

  547.                         "vmla.f32   q14, q10, d14[1]    \n"

  548. 

  549.                         "vext.32    q8, q0, q2, #2      \n"

  550.                         "vext.32    q11, q2, q3, #2     \n"

  551.                         "vmla.f32   q15, q8, d13[0]     \n"

  552.                         "vmla.f32   q12, q11, d15[0]    \n"

  553. 

  554.                         "vext.32    q9, q0, q2, #3      \n"

  555.                         "vmla.f32   q13, q9, d13[1]     \n"

  556. 

  557.                         "pld        [%9, #256]          \n"

  558.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k42434445 k46474849

  559.                         "sub        %9, #168            \n" // restore k0

  560. 

  561.                         "pld        [%8, #128]          \n"

  562.                         "vld1.f32   {d0-d1}, [%8]!      \n"

  563.                         "vmla.f32   q14, q0, d8[0]      \n"

  564. 

  565.                         "pld        [%8, #256]          \n"

  566.                         "vld1.f32   {d4-d7}, [%8]       \n"

  567.                         "vmla.f32   q15, q2, d10[0]     \n"

  568. 

  569.                         "vext.32    q1, q0, q2, #1      \n"

  570.                         "vext.32    q10, q2, q3, #1     \n"

  571.                         "vmla.f32   q12, q1, d8[1]      \n"

  572.                         "vmla.f32   q13, q10, d10[1]    \n"

  573. 

  574.                         "vext.32    q8, q0, q2, #2      \n"

  575.                         "vext.32    q11, q2, q3, #2     \n"

  576.                         "vmla.f32   q14, q8, d9[0]      \n"

  577.                         "vmla.f32   q15, q11, d11[0]    \n"

  578. 

  579.                         "vext.32    q9, q0, q2, #3      \n"

  580.                         "vmla.f32   q12, q9, d9[1]      \n"

  581. 

  582.                         "vadd.f32   q13, q13, q14       \n"

  583.                         "vadd.f32   q13, q13, q15       \n"

  584.                         "vadd.f32   q12, q12, q13       \n"

  585. 

  586.                         "vst1.f32   {d24-d25}, [%1]!    \n"

  587. 

  588.                         "subs       %0, #1              \n"

  589.                         "bne        0b                  \n"

  590.                         : "=r"(nn),     // %0

  591.                         "=r"(outptr), // %1

  592.                         "=r"(r0),     // %2

  593.                         "=r"(r1),     // %3

  594.                         "=r"(r2),     // %4

  595.                         "=r"(r3),     // %5

  596.                         "=r"(r4),     // %6

  597.                         "=r"(r5),     // %7

  598.                         "=r"(r6),     // %8

  599.                         "=r"(k0)      // %9

  600.                         : "0"(nn),

  601.                         "1"(outptr),

  602.                         "2"(r0),

  603.                         "3"(r1),

  604.                         "4"(r2),

  605.                         "5"(r3),

  606.                         "6"(r4),

  607.                         "7"(r5),

  608.                         "8"(r6),

  609.                         "9"(k0)

  610.                         : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  611.                 }

  612. #endif // __aarch64__

  613. #endif // __ARM_NEON

  614. 

  615.                 for (; remain > 0; remain--)

  616.                 {

  617.                     float sum = 0;

  618. 

  619.                     sum += r0[0] * k0[0];

  620.                     sum += r0[1] * k0[1];

  621.                     sum += r0[2] * k0[2];

  622.                     sum += r0[3] * k0[3];

  623.                     sum += r0[4] * k0[4];

  624.                     sum += r0[5] * k0[5];

  625.                     sum += r0[6] * k0[6];

  626. 

  627.                     sum += r1[0] * k1[0];

  628.                     sum += r1[1] * k1[1];

  629.                     sum += r1[2] * k1[2];

  630.                     sum += r1[3] * k1[3];

  631.                     sum += r1[4] * k1[4];

  632.                     sum += r1[5] * k1[5];

  633.                     sum += r1[6] * k1[6];

  634. 

  635.                     sum += r2[0] * k2[0];

  636.                     sum += r2[1] * k2[1];

  637.                     sum += r2[2] * k2[2];

  638.                     sum += r2[3] * k2[3];

  639.                     sum += r2[4] * k2[4];

  640.                     sum += r2[5] * k2[5];

  641.                     sum += r2[6] * k2[6];

  642. 

  643.                     sum += r3[0] * k3[0];

  644.                     sum += r3[1] * k3[1];

  645.                     sum += r3[2] * k3[2];

  646.                     sum += r3[3] * k3[3];

  647.                     sum += r3[4] * k3[4];

  648.                     sum += r3[5] * k3[5];

  649.                     sum += r3[6] * k3[6];

  650. 

  651.                     sum += r4[0] * k4[0];

  652.                     sum += r4[1] * k4[1];

  653.                     sum += r4[2] * k4[2];

  654.                     sum += r4[3] * k4[3];

  655.                     sum += r4[4] * k4[4];

  656.                     sum += r4[5] * k4[5];

  657.                     sum += r4[6] * k4[6];

  658. 

  659.                     sum += r5[0] * k5[0];

  660.                     sum += r5[1] * k5[1];

  661.                     sum += r5[2] * k5[2];

  662.                     sum += r5[3] * k5[3];

  663.                     sum += r5[4] * k5[4];

  664.                     sum += r5[5] * k5[5];

  665.                     sum += r5[6] * k5[6];

  666. 

  667.                     sum += r6[0] * k6[0];

  668.                     sum += r6[1] * k6[1];

  669.                     sum += r6[2] * k6[2];

  670.                     sum += r6[3] * k6[3];

  671.                     sum += r6[4] * k6[4];

  672.                     sum += r6[5] * k6[5];

  673.                     sum += r6[6] * k6[6];

  674. 

  675.                     *outptr += sum;

  676. 

  677.                     r0++;

  678.                     r1++;

  679.                     r2++;

  680.                     r3++;

  681.                     r4++;

  682.                     r5++;

  683.                     r6++;

  684.                     outptr++;

  685.                 }

  686. 

  687.                 r0 += 6;

  688.                 r1 += 6;

  689.                 r2 += 6;

  690.                 r3 += 6;

  691.                 r4 += 6;

  692.                 r5 += 6;

  693.                 r6 += 6;

  694.             }

  695.         }

  696.     }

  697. }

  698. 

  699. static void conv7x7s2_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  700. {

  701.     int w = bottom_blob.w;

  702.     int inch = bottom_blob.c;

  703. 

  704.     int outw = top_blob.w;

  705.     int outh = top_blob.h;

  706.     int outch = top_blob.c;

  707. 

  708.     const int tailstep = w - 2 * outw + w;

  709. 

  710.     const float* kernel = _kernel;

  711.     const float* bias = _bias;

  712. 

  713.     #pragma omp parallel for num_threads(opt.num_threads)

  714.     for (int p = 0; p < outch; p++)

  715.     {

  716.         Mat out = top_blob.channel(p);

  717. 

  718.         const float bias0 = bias ? bias[p] : 0.f;

  719. 

  720.         out.fill(bias0);

  721. 

  722.         for (int q = 0; q < inch; q++)

  723.         {

  724.             float* outptr = out;

  725. 

  726.             const float* img0 = bottom_blob.channel(q);

  727. 

  728.             const float* kernel0 = kernel + p * inch * 49 + q * 49;

  729. 

  730.             const float* r0 = img0;

  731.             const float* r1 = img0 + w;

  732.             const float* r2 = img0 + w * 2;

  733.             const float* r3 = img0 + w * 3;

  734.             const float* r4 = img0 + w * 4;

  735.             const float* r5 = img0 + w * 5;

  736.             const float* r6 = img0 + w * 6;

  737. 

  738.             const float* k0 = kernel0;

  739.             const float* k1 = kernel0 + 7;

  740.             const float* k2 = kernel0 + 14;

  741.             const float* k3 = kernel0 + 21;

  742.             const float* k4 = kernel0 + 28;

  743.             const float* k5 = kernel0 + 35;

  744.             const float* k6 = kernel0 + 42;

  745. 

  746.             int i = 0;

  747. 

  748.             for (; i < outh; i++)

  749.             {

  750. #if __ARM_NEON

  751.                 int nn = outw >> 2;

  752.                 int remain = outw - (nn << 2);

  753. #else

  754.                 int remain = outw;

  755. #endif // __ARM_NEON

  756. 

  757. #if __ARM_NEON

  758. #if __aarch64__

  759.                 float32x4_t _k0123 = vld1q_f32(k0);

  760.                 float32x4_t _k4567 = vld1q_f32(k0 + 4);

  761.                 float32x4_t _k78910 = vld1q_f32(k1);

  762.                 float32x4_t _k11121314 = vld1q_f32(k1 + 4);

  763.                 float32x4_t _k14151617 = vld1q_f32(k2);

  764.                 float32x4_t _k18192021 = vld1q_f32(k2 + 4);

  765.                 float32x4_t _k21222324 = vld1q_f32(k3);

  766.                 float32x4_t _k25262728 = vld1q_f32(k3 + 4);

  767.                 float32x4_t _k28293031 = vld1q_f32(k4);

  768.                 float32x4_t _k32333435 = vld1q_f32(k4 + 4);

  769.                 float32x4_t _k35363738 = vld1q_f32(k5);

  770.                 float32x4_t _k39404142 = vld1q_f32(k5 + 4);

  771.                 float32x4_t _k42434445 = vld1q_f32(k6);

  772.                 float32x4_t _k46474849 = vld1q_f32(k6 + 4);

  773. #ifdef __clang__ // __ARM_NEON && __aarch64__ && __clang__

  774.                 if (nn > 0)

  775.                 {

  776.                     asm volatile(

  777.                         // v0:  input / final output

  778.                         // v1 v2: = _ri0/_ri1  first

  779.                         // v3 v4: =                  then _r0_8101214/_r0_9111315

  780.                         // v5 = ri2 / ri4 / ri6

  781.                         // v6 = ri3 / ri5

  782.                         // v9 = intermediate sum register

  783.                         "0:                                        \n"

  784.                         "prfm       pldl1keep, [%1, #128]          \n"

  785.                         "ld1        {v0.4s}, [%1]                  \n"

  786. 

  787.                         //i = 1

  788.                         "prfm       pldl1keep, [%2, #512]          \n"

  789.                         "ld2        {v1.4s, v2.4s}, [%2]           \n" // v1  v2 = _r00  _r01

  790.                         "add        %2, %2, #32                    \n"

  791.                         "ld2        {v3.4s, v4.4s}, [%2]           \n" // v3  v4 = _r0_8101214 / _r0_9111315

  792.                         "fmul       v9.4s, v1.4s, %18.s[0]         \n" // *+ _r00

  793.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n" // v5 = _r02

  794.                         "fmla       v0.4s, v2.4s, %18.s[1]         \n" // *+ _r01

  795.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n" // v6 = _r03

  796.                         "fmla       v9.4s, v5.4s, %18.s[2]         \n" // *+ _r02

  797.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n" // v5 = _r04

  798.                         "fmla       v0.4s, v6.4s, %18.s[3]         \n" // *+ _r03

  799.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n" // v6 = _r05

  800.                         "fmla       v9.4s, v5.4s, %19.s[0]         \n" // *+ _r04

  801.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n" // v5 = _r06

  802.                         "fmla       v0.4s, v6.4s, %19.s[1]         \n" // *+ _r05

  803.                         "fmla       v9.4s, v5.4s, %19.s[2]         \n" // *+ _r06

  804. 

  805.                         //i = 2

  806.                         "prfm       pldl1keep, [%3, #512]          \n"

  807.                         "ld2        {v1.4s, v2.4s}, [%3]           \n"

  808.                         "add        %3, %3, #32                    \n"

  809.                         "ld2        {v3.4s, v4.4s}, [%3]           \n"

  810.                         "fmla       v9.4s, v1.4s, %20.s[0]         \n"

  811.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  812.                         "fmla       v0.4s, v2.4s, %20.s[1]         \n"

  813.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  814.                         "fmla       v9.4s, v5.4s, %20.s[2]         \n"

  815.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  816.                         "fmla       v0.4s, v6.4s, %20.s[3]         \n"

  817.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  818.                         "fmla       v9.4s, v5.4s, %21.s[0]         \n"

  819.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  820.                         "fmla       v0.4s, v6.4s, %21.s[1]         \n"

  821.                         "fmla       v9.4s, v5.4s, %21.s[2]         \n"

  822. 

  823.                         //i = 3

  824.                         "prfm       pldl1keep, [%4, #512]          \n"

  825.                         "ld2        {v1.4s, v2.4s}, [%4]           \n"

  826.                         "add        %4, %4, #32                    \n"

  827.                         "ld2        {v3.4s, v4.4s}, [%4]           \n"

  828.                         "fmla       v9.4s, v1.4s, %22.s[0]         \n"

  829.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  830.                         "fmla       v0.4s, v2.4s, %22.s[1]         \n"

  831.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  832.                         "fmla       v9.4s, v5.4s, %22.s[2]         \n"

  833.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  834.                         "fmla       v0.4s, v6.4s, %22.s[3]         \n"

  835.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  836.                         "fmla       v9.4s, v5.4s, %23.s[0]         \n"

  837.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  838.                         "fmla       v0.4s, v6.4s, %23.s[1]         \n"

  839.                         "fmla       v9.4s, v5.4s, %23.s[2]         \n"

  840. 

  841.                         //i = 4

  842.                         "prfm       pldl1keep, [%5, #512]          \n"

  843.                         "ld2        {v1.4s, v2.4s}, [%5]           \n"

  844.                         "add        %5, %5, #32                    \n"

  845.                         "ld2        {v3.4s, v4.4s}, [%5]           \n"

  846.                         "fmla       v9.4s, v1.4s, %24.s[0]         \n"

  847.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  848.                         "fmla       v0.4s, v2.4s, %24.s[1]         \n"

  849.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  850.                         "fmla       v9.4s, v5.4s, %24.s[2]         \n"

  851.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  852.                         "fmla       v0.4s, v6.4s, %24.s[3]         \n"

  853.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  854.                         "fmla       v9.4s, v5.4s, %25.s[0]         \n"

  855.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  856.                         "fmla       v0.4s, v6.4s, %25.s[1]         \n"

  857.                         "fmla       v9.4s, v5.4s, %25.s[2]         \n"

  858. 

  859.                         //i = 5

  860.                         "prfm       pldl1keep, [%6, #512]          \n"

  861.                         "ld2        {v1.4s, v2.4s}, [%6]           \n"

  862.                         "add        %6, %6, #32                    \n"

  863.                         "ld2        {v3.4s, v4.4s}, [%6]           \n"

  864.                         "fmla       v9.4s, v1.4s, %26.s[0]         \n"

  865.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  866.                         "fmla       v0.4s, v2.4s, %26.s[1]         \n"

  867.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  868.                         "fmla       v9.4s, v5.4s, %26.s[2]         \n"

  869.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  870.                         "fmla       v0.4s, v6.4s, %26.s[3]         \n"

  871.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  872.                         "fmla       v9.4s, v5.4s, %27.s[0]         \n"

  873.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  874.                         "fmla       v0.4s, v6.4s, %27.s[1]         \n"

  875.                         "fmla       v9.4s, v5.4s, %27.s[2]         \n"

  876. 

  877.                         //i = 6

  878.                         "prfm       pldl1keep, [%7, #512]          \n"

  879.                         "ld2        {v1.4s, v2.4s}, [%7]           \n"

  880.                         "add        %7, %7, #32                    \n"

  881.                         "ld2        {v3.4s, v4.4s}, [%7]           \n"

  882.                         "fmla       v9.4s, v1.4s, %28.s[0]         \n"

  883.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  884.                         "fmla       v0.4s, v2.4s, %28.s[1]         \n"

  885.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  886.                         "fmla       v9.4s, v5.4s, %28.s[2]         \n"

  887.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  888.                         "fmla       v0.4s, v6.4s, %28.s[3]         \n"

  889.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  890.                         "fmla       v9.4s, v5.4s, %29.s[0]         \n"

  891.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  892.                         "fmla       v0.4s, v6.4s, %29.s[1]         \n"

  893.                         "fmla       v9.4s, v5.4s, %29.s[2]         \n"

  894. 

  895.                         //i = 7

  896.                         "prfm       pldl1keep, [%8, #512]          \n"

  897.                         "ld2        {v1.4s, v2.4s}, [%8]           \n"

  898.                         "add        %8, %8, #32                    \n"

  899.                         "ld2        {v3.4s, v4.4s}, [%8]           \n"

  900.                         "fmla       v9.4s, v1.4s, %30.s[0]         \n"

  901.                         "ext        v5.16b, v1.16b, v3.16b, #4     \n"

  902.                         "fmla       v0.4s, v2.4s, %30.s[1]         \n"

  903.                         "ext        v6.16b, v2.16b, v4.16b, #4     \n"

  904.                         "fmla       v9.4s, v5.4s, %30.s[2]         \n"

  905.                         "ext        v5.16b, v1.16b, v3.16b, #8     \n"

  906.                         "fmla       v0.4s, v6.4s, %30.s[3]         \n"

  907.                         "ext        v6.16b, v2.16b, v4.16b, #8     \n"

  908.                         "fmla       v9.4s, v5.4s, %31.s[0]         \n"

  909.                         "ext        v5.16b, v1.16b, v3.16b, #12    \n"

  910.                         "fmla       v0.4s, v6.4s, %31.s[1]         \n"

  911.                         "fmla       v9.4s, v5.4s, %31.s[2]         \n"

  912. 

  913.                         "fadd       v0.4s, v0.4s, v9.4s            \n"

  914.                         "st1        {v0.4s}, [%1], #16             \n"

  915.                         "subs       %w0, %w0, #1                   \n"

  916.                         "bne        0b                             \n"

  917.                         : "=r"(nn),     // %0

  918.                         "=r"(outptr), // %1

  919.                         "=r"(r0),     // %2

  920.                         "=r"(r1),     // %3

  921.                         "=r"(r2),     // %4

  922.                         "=r"(r3),     // %5

  923.                         "=r"(r4),     // %6

  924.                         "=r"(r5),     // %7

  925.                         "=r"(r6)      // %8

  926.                         : "0"(nn),

  927.                         "1"(outptr),

  928.                         "2"(r0),

  929.                         "3"(r1),

  930.                         "4"(r2),

  931.                         "5"(r3),

  932.                         "6"(r4),

  933.                         "7"(r5),

  934.                         "8"(r6),

  935.                         "w"(_k0123),     // %18

  936.                         "w"(_k4567),     // %19

  937.                         "w"(_k78910),    // %20

  938.                         "w"(_k11121314), // %21

  939.                         "w"(_k14151617), // %22

  940.                         "w"(_k18192021), // %23

  941.                         "w"(_k21222324), // %24

  942.                         "w"(_k25262728), // %25

  943.                         "w"(_k28293031), // %26

  944.                         "w"(_k32333435), // %27

  945.                         "w"(_k35363738), // %28

  946.                         "w"(_k39404142), // %29

  947.                         "w"(_k42434445), // %30

  948.                         "w"(_k46474849)  // %31

  949.                         : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v9");

  950.                 }

  951. #else  // __ARM_NEON && __aarch64__ defined, but __clang__ not defined \

  952. // When compiled with gcc, gcc does not accept over 30 operands

  953.                 for (; nn > 0; nn--)

  954.                 {

  955.                     float32x4_t _sum = vld1q_f32(outptr);

  956. 

  957.                     float32x4x2_t _r00_02461357 = vld2q_f32(r0);

  958.                     float32x4x2_t _r00nx2 = vld2q_f32(r0 + 8);

  959.                     float32x4_t _r0_8101214 = _r00nx2.val[0];           // 8 10 12 14

  960.                     float32x4_t _r0_9111315 = _r00nx2.val[1];           // 9 11 13 15

  961.                     float32x4_t _r00 = _r00_02461357.val[0];            // 0 2 4 6

  962.                     float32x4_t _r01 = _r00_02461357.val[1];            // 1 3 5 7

  963.                     float32x4_t _r02 = vextq_f32(_r00, _r0_8101214, 1); // 2 4 6 8

  964.                     float32x4_t _r03 = vextq_f32(_r01, _r0_9111315, 1); // 3 5 7 9

  965.                     float32x4_t _r04 = vextq_f32(_r00, _r0_8101214, 2); // 4 6 8 10

  966.                     float32x4_t _r05 = vextq_f32(_r01, _r0_9111315, 2); // 5 7 9 11

  967.                     float32x4_t _r06 = vextq_f32(_r00, _r0_8101214, 3); // 6 8 10 12

  968. 

  969.                     _sum = vfmaq_laneq_f32(_sum, _r00, _k0123, 0);

  970.                     _sum = vfmaq_laneq_f32(_sum, _r01, _k0123, 1);

  971.                     _sum = vfmaq_laneq_f32(_sum, _r02, _k0123, 2);

  972.                     _sum = vfmaq_laneq_f32(_sum, _r03, _k0123, 3);

  973.                     _sum = vfmaq_laneq_f32(_sum, _r04, _k4567, 0);

  974.                     _sum = vfmaq_laneq_f32(_sum, _r05, _k4567, 1);

  975.                     _sum = vfmaq_laneq_f32(_sum, _r06, _k4567, 2);

  976. 

  977.                     float32x4x2_t _r10_02461357 = vld2q_f32(r1);

  978.                     float32x4x2_t _r10nx2 = vld2q_f32(r1 + 8);

  979.                     float32x4_t _r1_8101214 = _r10nx2.val[0];

  980.                     float32x4_t _r1_9111315 = _r10nx2.val[1];

  981.                     float32x4_t _r10 = _r10_02461357.val[0];

  982.                     float32x4_t _r11 = _r10_02461357.val[1];

  983.                     float32x4_t _r12 = vextq_f32(_r10, _r1_8101214, 1);

  984.                     float32x4_t _r13 = vextq_f32(_r11, _r1_9111315, 1);

  985.                     float32x4_t _r14 = vextq_f32(_r10, _r1_8101214, 2);

  986.                     float32x4_t _r15 = vextq_f32(_r11, _r1_9111315, 2);

  987.                     float32x4_t _r16 = vextq_f32(_r10, _r1_8101214, 3);

  988. 

  989.                     _sum = vfmaq_laneq_f32(_sum, _r10, _k78910, 0);

  990.                     _sum = vfmaq_laneq_f32(_sum, _r11, _k78910, 1);

  991.                     _sum = vfmaq_laneq_f32(_sum, _r12, _k78910, 2);

  992.                     _sum = vfmaq_laneq_f32(_sum, _r13, _k78910, 3);

  993.                     _sum = vfmaq_laneq_f32(_sum, _r14, _k11121314, 0);

  994.                     _sum = vfmaq_laneq_f32(_sum, _r15, _k11121314, 1);

  995.                     _sum = vfmaq_laneq_f32(_sum, _r16, _k11121314, 2);

  996. 

  997.                     float32x4x2_t _r20_02461357 = vld2q_f32(r2);

  998.                     float32x4x2_t _r20nx2 = vld2q_f32(r2 + 8);

  999.                     float32x4_t _r2_8101214 = _r20nx2.val[0];

  1000.                     float32x4_t _r2_9111315 = _r20nx2.val[1];

  1001.                     float32x4_t _r20 = _r20_02461357.val[0];

  1002.                     float32x4_t _r21 = _r20_02461357.val[1];

  1003.                     float32x4_t _r22 = vextq_f32(_r20, _r2_8101214, 1);

  1004.                     float32x4_t _r23 = vextq_f32(_r21, _r2_9111315, 1);

  1005.                     float32x4_t _r24 = vextq_f32(_r20, _r2_8101214, 2);

  1006.                     float32x4_t _r25 = vextq_f32(_r21, _r2_9111315, 2);

  1007.                     float32x4_t _r26 = vextq_f32(_r20, _r2_8101214, 3);

  1008. 

  1009.                     _sum = vfmaq_laneq_f32(_sum, _r20, _k14151617, 0);

  1010.                     _sum = vfmaq_laneq_f32(_sum, _r21, _k14151617, 1);

  1011.                     _sum = vfmaq_laneq_f32(_sum, _r22, _k14151617, 2);

  1012.                     _sum = vfmaq_laneq_f32(_sum, _r23, _k14151617, 3);

  1013.                     _sum = vfmaq_laneq_f32(_sum, _r24, _k18192021, 0);

  1014.                     _sum = vfmaq_laneq_f32(_sum, _r25, _k18192021, 1);

  1015.                     _sum = vfmaq_laneq_f32(_sum, _r26, _k18192021, 2);

  1016. 

  1017.                     float32x4x2_t _r30_02461357 = vld2q_f32(r3);

  1018.                     float32x4x2_t _r30nx2 = vld2q_f32(r3 + 8);

  1019.                     float32x4_t _r3_8101214 = _r30nx2.val[0];

  1020.                     float32x4_t _r3_9111315 = _r30nx2.val[1];

  1021.                     float32x4_t _r30 = _r30_02461357.val[0];

  1022.                     float32x4_t _r31 = _r30_02461357.val[1];

  1023.                     float32x4_t _r32 = vextq_f32(_r30, _r3_8101214, 1);

  1024.                     float32x4_t _r33 = vextq_f32(_r31, _r3_9111315, 1);

  1025.                     float32x4_t _r34 = vextq_f32(_r30, _r3_8101214, 2);

  1026.                     float32x4_t _r35 = vextq_f32(_r31, _r3_9111315, 2);

  1027.                     float32x4_t _r36 = vextq_f32(_r30, _r3_8101214, 3);

  1028. 

  1029.                     _sum = vfmaq_laneq_f32(_sum, _r30, _k21222324, 0);

  1030.                     _sum = vfmaq_laneq_f32(_sum, _r31, _k21222324, 1);

  1031.                     _sum = vfmaq_laneq_f32(_sum, _r32, _k21222324, 2);

  1032.                     _sum = vfmaq_laneq_f32(_sum, _r33, _k21222324, 3);

  1033.                     _sum = vfmaq_laneq_f32(_sum, _r34, _k25262728, 0);

  1034.                     _sum = vfmaq_laneq_f32(_sum, _r35, _k25262728, 1);

  1035.                     _sum = vfmaq_laneq_f32(_sum, _r36, _k25262728, 2);

  1036. 

  1037.                     float32x4x2_t _r40_02461357 = vld2q_f32(r4);

  1038.                     float32x4x2_t _r40nx2 = vld2q_f32(r4 + 8);

  1039.                     float32x4_t _r4_8101214 = _r40nx2.val[0];

  1040.                     float32x4_t _r4_9111315 = _r40nx2.val[1];

  1041.                     float32x4_t _r40 = _r40_02461357.val[0];

  1042.                     float32x4_t _r41 = _r40_02461357.val[1];

  1043.                     float32x4_t _r42 = vextq_f32(_r40, _r4_8101214, 1);

  1044.                     float32x4_t _r43 = vextq_f32(_r41, _r4_9111315, 1);

  1045.                     float32x4_t _r44 = vextq_f32(_r40, _r4_8101214, 2);

  1046.                     float32x4_t _r45 = vextq_f32(_r41, _r4_9111315, 2);

  1047.                     float32x4_t _r46 = vextq_f32(_r40, _r4_8101214, 3);

  1048. 

  1049.                     _sum = vfmaq_laneq_f32(_sum, _r40, _k28293031, 0);

  1050.                     _sum = vfmaq_laneq_f32(_sum, _r41, _k28293031, 1);

  1051.                     _sum = vfmaq_laneq_f32(_sum, _r42, _k28293031, 2);

  1052.                     _sum = vfmaq_laneq_f32(_sum, _r43, _k28293031, 3);

  1053.                     _sum = vfmaq_laneq_f32(_sum, _r44, _k32333435, 0);

  1054.                     _sum = vfmaq_laneq_f32(_sum, _r45, _k32333435, 1);

  1055.                     _sum = vfmaq_laneq_f32(_sum, _r46, _k32333435, 2);

  1056. 

  1057.                     float32x4x2_t _r50_02461357 = vld2q_f32(r5);

  1058.                     float32x4x2_t _r50nx2 = vld2q_f32(r5 + 8);

  1059.                     float32x4_t _r5_8101214 = _r50nx2.val[0];

  1060.                     float32x4_t _r5_9111315 = _r50nx2.val[1];

  1061.                     float32x4_t _r50 = _r50_02461357.val[0];

  1062.                     float32x4_t _r51 = _r50_02461357.val[1];

  1063.                     float32x4_t _r52 = vextq_f32(_r50, _r5_8101214, 1);

  1064.                     float32x4_t _r53 = vextq_f32(_r51, _r5_9111315, 1);

  1065.                     float32x4_t _r54 = vextq_f32(_r50, _r5_8101214, 2);

  1066.                     float32x4_t _r55 = vextq_f32(_r51, _r5_9111315, 2);

  1067.                     float32x4_t _r56 = vextq_f32(_r50, _r5_8101214, 3);

  1068. 

  1069.                     _sum = vfmaq_laneq_f32(_sum, _r50, _k35363738, 0);

  1070.                     _sum = vfmaq_laneq_f32(_sum, _r51, _k35363738, 1);

  1071.                     _sum = vfmaq_laneq_f32(_sum, _r52, _k35363738, 2);

  1072.                     _sum = vfmaq_laneq_f32(_sum, _r53, _k35363738, 3);

  1073.                     _sum = vfmaq_laneq_f32(_sum, _r54, _k39404142, 0);

  1074.                     _sum = vfmaq_laneq_f32(_sum, _r55, _k39404142, 1);

  1075.                     _sum = vfmaq_laneq_f32(_sum, _r56, _k39404142, 2);

  1076. 

  1077.                     float32x4x2_t _r60_02461357 = vld2q_f32(r6);

  1078.                     float32x4x2_t _r60nx2 = vld2q_f32(r6 + 8);

  1079.                     float32x4_t _r6_8101214 = _r60nx2.val[0];

  1080.                     float32x4_t _r6_9111315 = _r60nx2.val[1];

  1081.                     float32x4_t _r60 = _r60_02461357.val[0];

  1082.                     float32x4_t _r61 = _r60_02461357.val[1];

  1083.                     float32x4_t _r62 = vextq_f32(_r60, _r6_8101214, 1);

  1084.                     float32x4_t _r63 = vextq_f32(_r61, _r6_9111315, 1);

  1085.                     float32x4_t _r64 = vextq_f32(_r60, _r6_8101214, 2);

  1086.                     float32x4_t _r65 = vextq_f32(_r61, _r6_9111315, 2);

  1087.                     float32x4_t _r66 = vextq_f32(_r60, _r6_8101214, 3);

  1088. 

  1089.                     _sum = vfmaq_laneq_f32(_sum, _r60, _k42434445, 0);

  1090.                     _sum = vfmaq_laneq_f32(_sum, _r61, _k42434445, 1);

  1091.                     _sum = vfmaq_laneq_f32(_sum, _r62, _k42434445, 2);

  1092.                     _sum = vfmaq_laneq_f32(_sum, _r63, _k42434445, 3);

  1093.                     _sum = vfmaq_laneq_f32(_sum, _r64, _k46474849, 0);

  1094.                     _sum = vfmaq_laneq_f32(_sum, _r65, _k46474849, 1);

  1095.                     _sum = vfmaq_laneq_f32(_sum, _r66, _k46474849, 2);

  1096. 

  1097.                     vst1q_f32(outptr, _sum);

  1098. 

  1099.                     r0 += 8;

  1100.                     r1 += 8;

  1101.                     r2 += 8;

  1102.                     r3 += 8;

  1103.                     r4 += 8;

  1104.                     r5 += 8;

  1105.                     r6 += 8;

  1106.                     outptr += 4;

  1107.                 }

  1108. #endif // __clang__

  1109. #else

  1110.                 if (nn > 0)

  1111.                 {

  1112.                     asm volatile(

  1113.                         "0:                             \n"

  1114. 

  1115.                         "pld        [%1, #256]          \n"

  1116.                         "vld1.f32   {d26-d27}, [%1]     \n" // _sum

  1117.                         //                     "veor       q14, q14            \n"// _sum2 = 0;

  1118.                         //                     "veor       q15, q15            \n"// _sum3 = 0;

  1119. 

  1120.                         "pld        [%9, #256]          \n"

  1121.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k0123 k4567

  1122.                         "add        %9, #28             \n"

  1123. 

  1124.                         "pld        [%2, #512]          \n"

  1125.                         "vld2.f32   {d0-d3}, [%2]!      \n" // q0 = 0  2  4  6  q1 = 1  3  5  7

  1126.                         "vmla.f32   q13, q0, d8[0]      \n"

  1127.                         "vmul.f32   q14, q1, d8[1]      \n"

  1128. 

  1129.                         "vld2.f32   {d4-d7}, [%2]       \n" // q2 = 8 10 12 14  q3 = 9 11 13 15

  1130.                         "vext.32    q8, q0, q2, #1      \n" // q8 = 2  4  6  8

  1131.                         "vext.32    q9, q1, q3, #1      \n" // q9 = 3  5  7  9

  1132.                         "vmul.f32   q15, q8, d9[0]      \n"

  1133.                         "vmla.f32   q13, q9, d9[1]      \n"

  1134. 

  1135.                         "vext.32    q10, q0, q2, #2     \n" // q10= 4  6  8 10

  1136.                         "vext.32    q11, q1, q3, #2     \n" // q11= 5  7  9 11

  1137.                         "vmla.f32   q14, q10, d10[0]    \n"

  1138.                         "vmla.f32   q15, q11, d10[1]    \n"

  1139. 

  1140.                         "vext.32    q12, q0, q2, #3     \n" // q12= 6  8 10 12

  1141.                         "vmla.f32   q13, q12, d11[0]    \n"

  1142. 

  1143.                         "pld        [%9, #256]          \n"

  1144.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k78910 k11121314

  1145.                         "add        %9, #28             \n"

  1146. 

  1147.                         "pld        [%3, #512]          \n"

  1148.                         "vld2.f32   {d0-d3}, [%3]!      \n"

  1149.                         "vmla.f32   q14, q0, d12[0]     \n"

  1150.                         "vmla.f32   q15, q1, d12[1]     \n"

  1151. 

  1152.                         "vld2.f32   {d4-d7}, [%3]       \n"

  1153.                         "vext.32    q8, q0, q2, #1      \n"

  1154.                         "vext.32    q9, q1, q3, #1      \n"

  1155.                         "vmla.f32   q13, q8, d13[0]     \n"

  1156.                         "vmla.f32   q14, q9, d13[1]     \n"

  1157. 

  1158.                         "vext.32    q10, q0, q2, #2     \n"

  1159.                         "vext.32    q11, q1, q3, #2     \n"

  1160.                         "vmla.f32   q15, q10, d14[0]    \n"

  1161.                         "vmla.f32   q13, q11, d14[1]    \n"

  1162. 

  1163.                         "vext.32    q12, q0, q2, #3     \n"

  1164.                         "vmla.f32   q14, q12, d15[0]    \n"

  1165. 

  1166.                         "pld        [%9, #256]          \n"

  1167.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k14151617 k18192021

  1168.                         "add        %9, #28             \n"

  1169. 

  1170.                         "pld        [%4, #512]          \n"

  1171.                         "vld2.f32   {d0-d3}, [%4]!      \n"

  1172.                         "vmla.f32   q15, q0, d8[0]      \n"

  1173.                         "vmla.f32   q13, q1, d8[1]      \n"

  1174. 

  1175.                         "vld2.f32   {d4-d7}, [%4]       \n"

  1176.                         "vext.32    q8, q0, q2, #1      \n"

  1177.                         "vext.32    q9, q1, q3, #1      \n"

  1178.                         "vmla.f32   q14, q8, d9[0]      \n"

  1179.                         "vmla.f32   q15, q9, d9[1]      \n"

  1180. 

  1181.                         "vext.32    q10, q0, q2, #2     \n"

  1182.                         "vext.32    q11, q1, q3, #2     \n"

  1183.                         "vmla.f32   q13, q10, d10[0]    \n"

  1184.                         "vmla.f32   q14, q11, d10[1]    \n"

  1185. 

  1186.                         "vext.32    q12, q0, q2, #3     \n"

  1187.                         "vmla.f32   q15, q12, d11[0]    \n"

  1188. 

  1189.                         "pld        [%9, #256]          \n"

  1190.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k21222324 k25262728

  1191.                         "add        %9, #28             \n"

  1192. 

  1193.                         "pld        [%5, #512]          \n"

  1194.                         "vld2.f32   {d0-d3}, [%5]!      \n"

  1195.                         "vmla.f32   q13, q0, d12[0]     \n"

  1196.                         "vmla.f32   q14, q1, d12[1]     \n"

  1197. 

  1198.                         "vld2.f32   {d4-d7}, [%5]       \n"

  1199.                         "vext.32    q8, q0, q2, #1      \n"

  1200.                         "vext.32    q9, q1, q3, #1      \n"

  1201.                         "vmla.f32   q15, q8, d13[0]     \n"

  1202.                         "vmla.f32   q13, q9, d13[1]     \n"

  1203. 

  1204.                         "vext.32    q10, q0, q2, #2     \n"

  1205.                         "vext.32    q11, q1, q3, #2     \n"

  1206.                         "vmla.f32   q14, q10, d14[0]    \n"

  1207.                         "vmla.f32   q15, q11, d14[1]    \n"

  1208. 

  1209.                         "vext.32    q12, q0, q2, #3     \n"

  1210.                         "vmla.f32   q13, q12, d15[0]    \n"

  1211. 

  1212.                         "pld        [%9, #256]          \n"

  1213.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k28293031 k32333435

  1214.                         "add        %9, #28             \n"

  1215. 

  1216.                         "pld        [%6, #512]          \n"

  1217.                         "vld2.f32   {d0-d3}, [%6]!      \n"

  1218.                         "vmla.f32   q14, q0, d8[0]      \n"

  1219.                         "vmla.f32   q15, q1, d8[1]      \n"

  1220. 

  1221.                         "vld2.f32   {d4-d7}, [%6]       \n"

  1222.                         "vext.32    q8, q0, q2, #1      \n"

  1223.                         "vext.32    q9, q1, q3, #1      \n"

  1224.                         "vmla.f32   q13, q8, d9[0]      \n"

  1225.                         "vmla.f32   q14, q9, d9[1]      \n"

  1226. 

  1227.                         "vext.32    q10, q0, q2, #2     \n"

  1228.                         "vext.32    q11, q1, q3, #2     \n"

  1229.                         "vmla.f32   q15, q10, d10[0]    \n"

  1230.                         "vmla.f32   q13, q11, d10[1]    \n"

  1231. 

  1232.                         "vext.32    q12, q0, q2, #3     \n"

  1233.                         "vmla.f32   q14, q12, d11[0]    \n"

  1234. 

  1235.                         "pld        [%9, #256]          \n"

  1236.                         "vld1.f32   {d12-d15}, [%9]     \n" // q6 q7 = k35363738 k39404142

  1237.                         "add        %9, #28             \n"

  1238. 

  1239.                         "pld        [%7, #512]          \n"

  1240.                         "vld2.f32   {d0-d3}, [%7]!      \n"

  1241.                         "vmla.f32   q15, q0, d12[0]     \n"

  1242.                         "vmla.f32   q13, q1, d12[1]     \n"

  1243. 

  1244.                         "vld2.f32   {d4-d7}, [%7]       \n"

  1245.                         "vext.32    q8, q0, q2, #1      \n"

  1246.                         "vext.32    q9, q1, q3, #1      \n"

  1247.                         "vmla.f32   q14, q8, d13[0]     \n"

  1248.                         "vmla.f32   q15, q9, d13[1]     \n"

  1249. 

  1250.                         "vext.32    q10, q0, q2, #2     \n"

  1251.                         "vext.32    q11, q1, q3, #2     \n"

  1252.                         "vmla.f32   q13, q10, d14[0]    \n"

  1253.                         "vmla.f32   q14, q11, d14[1]    \n"

  1254. 

  1255.                         "vext.32    q12, q0, q2, #3     \n"

  1256.                         "vmla.f32   q15, q12, d15[0]    \n"

  1257. 

  1258.                         "pld        [%9, #256]          \n"

  1259.                         "vld1.f32   {d8-d11}, [%9]      \n" // q4 q5 = k42434445 k46474849

  1260.                         "sub        %9, #168            \n" // restore k0

  1261. 

  1262.                         "pld        [%8, #512]          \n"

  1263.                         "vld2.f32   {d0-d3}, [%8]!      \n"

  1264.                         "vmla.f32   q13, q0, d8[0]      \n"

  1265.                         "vmla.f32   q14, q1, d8[1]      \n"

  1266. 

  1267.                         "vld2.f32   {d4-d7}, [%8]       \n"

  1268.                         "vext.32    q8, q0, q2, #1      \n"

  1269.                         "vext.32    q9, q1, q3, #1      \n"

  1270.                         "vmla.f32   q15, q8, d9[0]      \n"

  1271.                         "vmla.f32   q13, q9, d9[1]      \n"

  1272. 

  1273.                         "vext.32    q10, q0, q2, #2     \n"

  1274.                         "vext.32    q11, q1, q3, #2     \n"

  1275.                         "vmla.f32   q14, q10, d10[0]    \n"

  1276.                         "vmla.f32   q15, q11, d10[1]    \n"

  1277. 

  1278.                         "vext.32    q12, q0, q2, #3     \n"

  1279.                         "vmla.f32   q13, q12, d11[0]    \n"

  1280. 

  1281.                         "vadd.f32   q14, q14, q15       \n"

  1282.                         "vadd.f32   q13, q13, q14       \n"

  1283. 

  1284.                         "vst1.f32   {d26-d27}, [%1]!    \n"

  1285. 

  1286.                         "subs       %0, #1              \n"

  1287.                         "bne        0b                  \n"

  1288.                         : "=r"(nn),     // %0

  1289.                         "=r"(outptr), // %1

  1290.                         "=r"(r0),     // %2

  1291.                         "=r"(r1),     // %3

  1292.                         "=r"(r2),     // %4

  1293.                         "=r"(r3),     // %5

  1294.                         "=r"(r4),     // %6

  1295.                         "=r"(r5),     // %7

  1296.                         "=r"(r6),     // %8

  1297.                         "=r"(k0)      // %9

  1298.                         : "0"(nn),

  1299.                         "1"(outptr),

  1300.                         "2"(r0),

  1301.                         "3"(r1),

  1302.                         "4"(r2),

  1303.                         "5"(r3),

  1304.                         "6"(r4),

  1305.                         "7"(r5),

  1306.                         "8"(r6),

  1307.                         "9"(k0)

  1308.                         : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");

  1309.                 }

  1310. #endif // __aarch64__

  1311. #endif // __ARM_NEON

  1312. 

  1313.                 for (; remain > 0; remain--)

  1314.                 {

  1315.                     float sum = 0;

  1316. 

  1317.                     sum += r0[0] * k0[0];

  1318.                     sum += r0[1] * k0[1];

  1319.                     sum += r0[2] * k0[2];

  1320.                     sum += r0[3] * k0[3];

  1321.                     sum += r0[4] * k0[4];

  1322.                     sum += r0[5] * k0[5];

  1323.                     sum += r0[6] * k0[6];

  1324. 

  1325.                     sum += r1[0] * k1[0];

  1326.                     sum += r1[1] * k1[1];

  1327.                     sum += r1[2] * k1[2];

  1328.                     sum += r1[3] * k1[3];

  1329.                     sum += r1[4] * k1[4];

  1330.                     sum += r1[5] * k1[5];

  1331.                     sum += r1[6] * k1[6];

  1332. 

  1333.                     sum += r2[0] * k2[0];

  1334.                     sum += r2[1] * k2[1];

  1335.                     sum += r2[2] * k2[2];

  1336.                     sum += r2[3] * k2[3];

  1337.                     sum += r2[4] * k2[4];

  1338.                     sum += r2[5] * k2[5];

  1339.                     sum += r2[6] * k2[6];

  1340. 

  1341.                     sum += r3[0] * k3[0];

  1342.                     sum += r3[1] * k3[1];

  1343.                     sum += r3[2] * k3[2];

  1344.                     sum += r3[3] * k3[3];

  1345.                     sum += r3[4] * k3[4];

  1346.                     sum += r3[5] * k3[5];

  1347.                     sum += r3[6] * k3[6];

  1348. 

  1349.                     sum += r4[0] * k4[0];

  1350.                     sum += r4[1] * k4[1];

  1351.                     sum += r4[2] * k4[2];

  1352.                     sum += r4[3] * k4[3];

  1353.                     sum += r4[4] * k4[4];

  1354.                     sum += r4[5] * k4[5];

  1355.                     sum += r4[6] * k4[6];

  1356. 

  1357.                     sum += r5[0] * k5[0];

  1358.                     sum += r5[1] * k5[1];

  1359.                     sum += r5[2] * k5[2];

  1360.                     sum += r5[3] * k5[3];

  1361.                     sum += r5[4] * k5[4];

  1362.                     sum += r5[5] * k5[5];

  1363.                     sum += r5[6] * k5[6];

  1364. 

  1365.                     sum += r6[0] * k6[0];

  1366.                     sum += r6[1] * k6[1];

  1367.                     sum += r6[2] * k6[2];

  1368.                     sum += r6[3] * k6[3];

  1369.                     sum += r6[4] * k6[4];

  1370.                     sum += r6[5] * k6[5];

  1371.                     sum += r6[6] * k6[6];

  1372. 

  1373.                     *outptr += sum;

  1374. 

  1375.                     r0 += 2;

  1376.                     r1 += 2;

  1377.                     r2 += 2;

  1378.                     r3 += 2;

  1379.                     r4 += 2;

  1380.                     r5 += 2;

  1381.                     r6 += 2;

  1382.                     outptr++;

  1383.                 }

  1384. 

  1385.                 r0 += tailstep;

  1386.                 r1 += tailstep;

  1387.                 r2 += tailstep;

  1388.                 r3 += tailstep;

  1389.                 r4 += tailstep;

  1390.                 r5 += tailstep;

  1391.                 r6 += tailstep;

  1392.             }

  1393.         }

  1394.     }

  1395. }