nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: b7db8be4f6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'b7db8be4f6'
${ noResults }
ncnn/src/layer/arm/convolution_5x5.h

1252 lines
49 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #if __ARM_NEON

  16. #include <arm_neon.h>

  17. #endif // __ARM_NEON

  18. 

  19. static void conv5x5s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias)

  20. {

  21.     int w = bottom_blob.w;

  22.     int h = bottom_blob.h;

  23.     int inch = bottom_blob.c;

  24. 

  25.     int outw = top_blob.w;

  26.     int outh = top_blob.h;

  27.     int outch = top_blob.c;

  28. 

  29.     const float* kernel = _kernel;

  30.     const float* bias = _bias;

  31. 

  32.     #pragma omp parallel for

  33.     for (int p=0; p<outch; p++)

  34.     {

  35.         Mat out = top_blob.channel(p);

  36. 

  37.         const float bias0 = bias ? bias[p] : 0.f;

  38. 

  39.         out.fill(bias0);

  40. 

  41.         for (int q=0; q<inch; q++)

  42.         {

  43.             float* outptr = out;

  44.             float* outptr2 = outptr + outw;

  45. 

  46.             const float* img0 = bottom_blob.channel(q);

  47. 

  48.             const float* kernel0 = kernel + p*inch*25  + q*25;

  49. 

  50.             const float* r0 = img0;

  51.             const float* r1 = img0 + w;

  52.             const float* r2 = img0 + w*2;

  53.             const float* r3 = img0 + w*3;

  54.             const float* r4 = img0 + w*4;

  55.             const float* r5 = img0 + w*5;

  56. 

  57.             const float* k0 = kernel0;

  58.             const float* k1 = kernel0 + 5;

  59.             const float* k2 = kernel0 + 10;

  60.             const float* k3 = kernel0 + 15;

  61.             const float* k4 = kernel0 + 20;

  62. 

  63. #if __ARM_NEON

  64.             float32x4_t _k0123 = vld1q_f32(kernel0);

  65.             float32x4_t _k4567 = vld1q_f32(kernel0+4);

  66.             float32x4_t _k891011 = vld1q_f32(kernel0+8);

  67.             float32x4_t _k12131415 = vld1q_f32(kernel0+12);

  68.             float32x4_t _k16171819 = vld1q_f32(kernel0+16);

  69.             float32x4_t _k20212223 = vld1q_f32(kernel0+20);

  70.             float32x4_t _k24242424 = vdupq_n_f32(kernel0[24]);

  71. #endif // __ARM_NEON

  72. 

  73.             int i = 0;

  74. 

  75.             for (; i+1 < outh; i+=2)

  76.             {

  77. 

  78. #if __ARM_NEON

  79.                 int nn = outw >> 2;

  80.                 int remain = outw - (nn << 2);

  81. #else

  82.                 int remain = outw;

  83. #endif // __ARM_NEON

  84. 

  85. #if __ARM_NEON

  86. #if __aarch64__

  87.                 for (; nn>0; nn--)

  88.                 {

  89.                     float32x4_t _sum = vdupq_n_f32(0.f);

  90.                     float32x4_t _sum2 = vdupq_n_f32(0.f);

  91. 

  92.                     float32x4_t _r00 = vld1q_f32(r0);

  93.                     float32x4_t _r04 = vld1q_f32(r0 + 4);

  94.                     float32x4_t _r01 = vextq_f32(_r00, _r04, 1);

  95.                     float32x4_t _r02 = vextq_f32(_r00, _r04, 2);

  96.                     float32x4_t _r03 = vextq_f32(_r00, _r04, 3);

  97. 

  98.                     float32x4_t _r10 = vld1q_f32(r1);

  99.                     float32x4_t _r14 = vld1q_f32(r1 + 4);

  100.                     float32x4_t _r11 = vextq_f32(_r10, _r14, 1);

  101.                     float32x4_t _r12 = vextq_f32(_r10, _r14, 2);

  102.                     float32x4_t _r13 = vextq_f32(_r10, _r14, 3);

  103. 

  104.                     float32x4_t _r20 = vld1q_f32(r2);

  105.                     float32x4_t _r24 = vld1q_f32(r2 + 4);

  106.                     float32x4_t _r21 = vextq_f32(_r20, _r24, 1);

  107.                     float32x4_t _r22 = vextq_f32(_r20, _r24, 2);

  108.                     float32x4_t _r23 = vextq_f32(_r20, _r24, 3);

  109. 

  110.                     float32x4_t _r30 = vld1q_f32(r3);

  111.                     float32x4_t _r34 = vld1q_f32(r3 + 4);

  112.                     float32x4_t _r31 = vextq_f32(_r30, _r34, 1);

  113.                     float32x4_t _r32 = vextq_f32(_r30, _r34, 2);

  114.                     float32x4_t _r33 = vextq_f32(_r30, _r34, 3);

  115. 

  116.                     float32x4_t _r40 = vld1q_f32(r4);

  117.                     float32x4_t _r44 = vld1q_f32(r4 + 4);

  118.                     float32x4_t _r41 = vextq_f32(_r40, _r44, 1);

  119.                     float32x4_t _r42 = vextq_f32(_r40, _r44, 2);

  120.                     float32x4_t _r43 = vextq_f32(_r40, _r44, 3);

  121. 

  122.                     float32x4_t _r50 = vld1q_f32(r5);

  123.                     float32x4_t _r54 = vld1q_f32(r5 + 4);

  124.                     float32x4_t _r51 = vextq_f32(_r50, _r54, 1);

  125.                     float32x4_t _r52 = vextq_f32(_r50, _r54, 2);

  126.                     float32x4_t _r53 = vextq_f32(_r50, _r54, 3);

  127. 

  128.                     _sum = vfmaq_laneq_f32(_sum, _r00, _k0123, 0);

  129.                     _sum = vfmaq_laneq_f32(_sum, _r01, _k0123, 1);

  130.                     _sum = vfmaq_laneq_f32(_sum, _r02, _k0123, 2);

  131.                     _sum = vfmaq_laneq_f32(_sum, _r03, _k0123, 3);

  132.                     _sum = vfmaq_laneq_f32(_sum, _r04, _k4567, 0);

  133. 

  134.                     _sum = vfmaq_laneq_f32(_sum, _r10, _k4567, 1);

  135.                     _sum = vfmaq_laneq_f32(_sum, _r11, _k4567, 2);

  136.                     _sum = vfmaq_laneq_f32(_sum, _r12, _k4567, 3);

  137.                     _sum = vfmaq_laneq_f32(_sum, _r13, _k891011, 0);

  138.                     _sum = vfmaq_laneq_f32(_sum, _r14, _k891011, 1);

  139. 

  140.                     _sum = vfmaq_laneq_f32(_sum, _r20, _k891011, 2);

  141.                     _sum = vfmaq_laneq_f32(_sum, _r21, _k891011, 3);

  142.                     _sum = vfmaq_laneq_f32(_sum, _r22, _k12131415, 0);

  143.                     _sum = vfmaq_laneq_f32(_sum, _r23, _k12131415, 1);

  144.                     _sum = vfmaq_laneq_f32(_sum, _r24, _k12131415, 2);

  145. 

  146.                     _sum = vfmaq_laneq_f32(_sum, _r30, _k12131415, 3);

  147.                     _sum = vfmaq_laneq_f32(_sum, _r31, _k16171819, 0);

  148.                     _sum = vfmaq_laneq_f32(_sum, _r32, _k16171819, 1);

  149.                     _sum = vfmaq_laneq_f32(_sum, _r33, _k16171819, 2);

  150.                     _sum = vfmaq_laneq_f32(_sum, _r34, _k16171819, 3);

  151. 

  152.                     _sum = vfmaq_laneq_f32(_sum, _r40, _k20212223, 0);

  153.                     _sum = vfmaq_laneq_f32(_sum, _r41, _k20212223, 1);

  154.                     _sum = vfmaq_laneq_f32(_sum, _r42, _k20212223, 2);

  155.                     _sum = vfmaq_laneq_f32(_sum, _r43, _k20212223, 3);

  156.                     _sum = vfmaq_laneq_f32(_sum, _r44, _k24242424, 0);

  157. 

  158.                     _sum2 = vfmaq_laneq_f32(_sum2, _r10, _k0123, 0);

  159.                     _sum2 = vfmaq_laneq_f32(_sum2, _r11, _k0123, 1);

  160.                     _sum2 = vfmaq_laneq_f32(_sum2, _r12, _k0123, 2);

  161.                     _sum2 = vfmaq_laneq_f32(_sum2, _r13, _k0123, 3);

  162.                     _sum2 = vfmaq_laneq_f32(_sum2, _r14, _k4567, 0);

  163. 

  164.                     _sum2 = vfmaq_laneq_f32(_sum2, _r20, _k4567, 1);

  165.                     _sum2 = vfmaq_laneq_f32(_sum2, _r21, _k4567, 2);

  166.                     _sum2 = vfmaq_laneq_f32(_sum2, _r22, _k4567, 3);

  167.                     _sum2 = vfmaq_laneq_f32(_sum2, _r23, _k891011, 0);

  168.                     _sum2 = vfmaq_laneq_f32(_sum2, _r24, _k891011, 1);

  169. 

  170.                     _sum2 = vfmaq_laneq_f32(_sum2, _r30, _k891011, 2);

  171.                     _sum2 = vfmaq_laneq_f32(_sum2, _r31, _k891011, 3);

  172.                     _sum2 = vfmaq_laneq_f32(_sum2, _r32, _k12131415, 0);

  173.                     _sum2 = vfmaq_laneq_f32(_sum2, _r33, _k12131415, 1);

  174.                     _sum2 = vfmaq_laneq_f32(_sum2, _r34, _k12131415, 2);

  175. 

  176.                     _sum2 = vfmaq_laneq_f32(_sum2, _r40, _k12131415, 3);

  177.                     _sum2 = vfmaq_laneq_f32(_sum2, _r41, _k16171819, 0);

  178.                     _sum2 = vfmaq_laneq_f32(_sum2, _r42, _k16171819, 1);

  179.                     _sum2 = vfmaq_laneq_f32(_sum2, _r43, _k16171819, 2);

  180.                     _sum2 = vfmaq_laneq_f32(_sum2, _r44, _k16171819, 3);

  181. 

  182.                     _sum2 = vfmaq_laneq_f32(_sum2, _r50, _k20212223, 0);

  183.                     _sum2 = vfmaq_laneq_f32(_sum2, _r51, _k20212223, 1);

  184.                     _sum2 = vfmaq_laneq_f32(_sum2, _r52, _k20212223, 2);

  185.                     _sum2 = vfmaq_laneq_f32(_sum2, _r53, _k20212223, 3);

  186.                     _sum2 = vfmaq_laneq_f32(_sum2, _r54, _k24242424, 0);

  187. 

  188.                     vst1q_f32(outptr, _sum);

  189.                     vst1q_f32(outptr2, _sum2);

  190. 

  191.                     r0 += 4;

  192.                     r1 += 4;

  193.                     r2 += 4;

  194.                     r3 += 4;

  195.                     r4 += 4;

  196.                     r5 += 4;

  197.                     outptr += 4;

  198.                     outptr2 += 4;

  199.                 }

  200. #else

  201.                 if (nn > 0)

  202.                 {

  203.                 asm volatile(

  204. //                     "veor       q13, q13            \n"

  205. //                     "veor       q14, q14            \n"

  206. 

  207.                     "pld        [%1, #128]          \n"

  208. 

  209.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = out

  210. 

  211.                     "0:                             \n"

  212. 

  213.                     // q11 = rx1 / rx3

  214.                     // q12 = rx2

  215. 

  216.                     // q13 q14 = intermediate sum register

  217. 

  218.                     "pld        [%2, #128]          \n"

  219. 

  220.                     "vld1.f32   {d16-d17}, [%2]     \n"// q8 = out2

  221. 

  222. 

  223.                     "pld        [%4, #256]          \n"

  224. 

  225.                     // r1

  226.                     "vld1.f32   {d18-d21}, [%4]     \n"// q9 q10 = r10 r14

  227.                     "add        %4, #16             \n"

  228. 

  229.                     "vext.32    q11, q9, q10, #1    \n"// r11

  230.                     "vmul.f32   q13, q9, %e19[1]    \n"

  231.                     "vmla.f32   q8, q9, %e18[0]     \n"

  232. 

  233.                     "vext.32    q12, q9, q10, #2    \n"// r12

  234.                     "vmla.f32   q7, q11, %f19[0]    \n"

  235.                     "vmul.f32   q14, q11, %e18[1]   \n"

  236. 

  237.                     "vext.32    q11, q9, q10, #3    \n"// r13

  238.                     "vmla.f32   q13, q12, %f19[1]   \n"

  239.                     "vmla.f32   q8, q12, %f18[0]    \n"

  240. 

  241.                     "vmla.f32   q7, q11, %e20[0]    \n"

  242.                     "vmla.f32   q14, q11, %f18[1]   \n"

  243. 

  244.                     "pld        [%5, #256]          \n"

  245. 

  246.                     "vmla.f32   q13, q10, %e20[1]   \n"

  247.                     "vmla.f32   q8, q10, %e19[0]    \n"

  248. 

  249.                     // r2

  250.                     "vld1.f32   {d18-d21}, [%5]     \n"// q9 q10 = r20 r24

  251.                     "add        %5, #16             \n"

  252. 

  253.                     "vext.32    q11, q9, q10, #1    \n"// r21

  254.                     "vmla.f32   q7, q9, %f20[0]     \n"

  255.                     "vmla.f32   q14, q9, %e19[1]    \n"

  256. 

  257.                     "vext.32    q12, q9, q10, #2    \n"// r22

  258.                     "vmla.f32   q13, q11, %f20[1]   \n"

  259.                     "vmla.f32   q8, q11, %f19[0]    \n"

  260. 

  261.                     "vext.32    q11, q9, q10, #3    \n"// r23

  262.                     "vmla.f32   q7, q12, %e21[0]    \n"

  263.                     "vmla.f32   q14, q12, %f19[1]   \n"

  264. 

  265.                     "vmla.f32   q13, q11, %e21[1]   \n"

  266.                     "vmla.f32   q8, q11, %e20[0]    \n"

  267. 

  268.                     "pld        [%6, #256]          \n"

  269. 

  270.                     "vmla.f32   q7, q10, %f21[0]    \n"

  271.                     "vmla.f32   q14, q10, %e20[1]   \n"

  272. 

  273.                     // r3

  274.                     "vld1.f32   {d18-d21}, [%6]     \n"// q9 q10 = r30 r34

  275.                     "add        %6, #16             \n"

  276. 

  277.                     "vext.32    q11, q9, q10, #1    \n"// r31

  278.                     "vmla.f32   q13, q9, %f21[1]    \n"

  279.                     "vmla.f32   q8, q9, %f20[0]     \n"

  280. 

  281.                     "vext.32    q12, q9, q10, #2    \n"// r32

  282.                     "vmla.f32   q7, q11, %e22[0]    \n"

  283.                     "vmla.f32   q14, q11, %f20[1]   \n"

  284. 

  285.                     "vext.32    q11, q9, q10, #3    \n"// r33

  286.                     "vmla.f32   q13, q12, %e22[1]   \n"

  287.                     "vmla.f32   q8, q12, %e21[0]    \n"

  288. 

  289.                     "vmla.f32   q7, q11, %f22[0]    \n"

  290.                     "vmla.f32   q14, q11, %e21[1]   \n"

  291. 

  292.                     "pld        [%7, #256]          \n"

  293. 

  294.                     "vmla.f32   q13, q10, %f22[1]   \n"

  295.                     "vmla.f32   q8, q10, %f21[0]    \n"

  296. 

  297.                     // r4

  298.                     "vld1.f32   {d18-d21}, [%7]     \n"// q9 q10 = r40 r44

  299.                     "add        %7, #16             \n"

  300. 

  301.                     "vext.32    q11, q9, q10, #1    \n"// r41

  302.                     "vmla.f32   q7, q9, %e23[0]     \n"

  303.                     "vmla.f32   q14, q9, %f21[1]    \n"

  304. 

  305.                     "vext.32    q12, q9, q10, #2    \n"// r42

  306.                     "vmla.f32   q13, q11, %e23[1]   \n"

  307.                     "vmla.f32   q8, q11, %e22[0]    \n"

  308. 

  309.                     "vext.32    q11, q9, q10, #3    \n"// r43

  310.                     "vmla.f32   q7, q12, %f23[0]    \n"

  311.                     "vmla.f32   q14, q12, %e22[1]   \n"

  312. 

  313.                     "vmla.f32   q13, q11, %f23[1]   \n"

  314.                     "vmla.f32   q8, q11, %f22[0]    \n"

  315. 

  316.                     "pld        [%3, #256]          \n"

  317. 

  318.                     "vmla.f32   q7, q10, %e24[0]    \n"

  319.                     "vmla.f32   q14, q10, %f22[1]   \n"

  320. 

  321.                     // r0 and r5

  322.                     "vld1.f32   {d18-d21}, [%3]     \n"// q9 q10 = r00 r04

  323.                     "add        %3, #16             \n"

  324. 

  325.                     "vext.32    q11, q9, q10, #1    \n"// r01

  326.                     "vmla.f32   q13, q11, %e18[1]   \n"

  327. 

  328.                     "vext.32    q12, q9, q10, #2    \n"// r02

  329.                     "vmla.f32   q7, q12, %f18[0]    \n"

  330. 

  331.                     "vext.32    q11, q9, q10, #3    \n"// r03

  332. 

  333.                     "pld        [%8, #256]          \n"

  334. 

  335.                     "vmla.f32   q13, q11, %f18[1]   \n"

  336. 

  337.                     // r5

  338.                     "vld1.f32   {d22-d25}, [%8]     \n"// q11 q12 = r50 r54

  339.                     "add        %8, #16             \n"

  340. 

  341.                     "vmla.f32   q8, q11, %e23[0]    \n"

  342.                     "vmla.f32   q14, q12, %e24[0]   \n"

  343. 

  344.                     "vmla.f32   q7, q9, %e18[0]     \n"

  345.                     "vmla.f32   q13, q10, %e19[0]   \n"

  346. 

  347.                     "vext.32    q9, q11, q12, #1    \n"// r51

  348.                     "vext.32    q10, q11, q12, #2   \n"// r52

  349. 

  350.                     "vmla.f32   q14, q9, %e23[1]    \n"

  351. 

  352.                     "vext.32    q9, q11, q12, #3    \n"// r53

  353.                     "vmla.f32   q8, q10, %f23[0]    \n"

  354. 

  355.                     "vmla.f32   q14, q9, %f23[1]    \n"

  356. 

  357.                     "vadd.f32   q7, q7, q13         \n"

  358. 

  359. //                     "veor       q13, q13            \n"

  360. 

  361.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  362. 

  363.                     "vadd.f32   q8, q8, q14         \n"

  364. 

  365.                     "pld        [%1, #128]          \n"

  366. 

  367.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = out

  368. 

  369. //                     "veor       q14, q14            \n"

  370. 

  371.                     "vst1.f32   {d16-d17}, [%2]!    \n"

  372. 

  373.                     "subs       %0, #1              \n"

  374.                     "bne        0b                  \n"

  375.                     : "=r"(nn),         // %0

  376.                       "=r"(outptr),     // %1

  377.                       "=r"(outptr2),    // %2

  378.                       "=r"(r0),         // %3

  379.                       "=r"(r1),         // %4

  380.                       "=r"(r2),         // %5

  381.                       "=r"(r3),         // %6

  382.                       "=r"(r4),         // %7

  383.                       "=r"(r5)          // %8

  384.                     : "0"(nn),

  385.                       "1"(outptr),

  386.                       "2"(outptr2),

  387.                       "3"(r0),

  388.                       "4"(r1),

  389.                       "5"(r2),

  390.                       "6"(r3),

  391.                       "7"(r4),

  392.                       "8"(r5),

  393.                       "w"(_k0123),      // %18

  394.                       "w"(_k4567),      // %19

  395.                       "w"(_k891011),    // %20

  396.                       "w"(_k12131415),  // %21

  397.                       "w"(_k16171819),  // %22

  398.                       "w"(_k20212223),  // %23

  399.                       "w"(_k24242424)   // %24

  400.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  401.                 );

  402.                 }

  403. #endif // __aarch64__

  404. #endif // __ARM_NEON

  405.                 for (; remain>0; remain--)

  406.                 {

  407.                     float sum = 0;

  408.                     float sum2 = 0;

  409. #if __ARM_NEON

  410.                     float32x4_t _r1 = vld1q_f32(r1);

  411.                     float32x4_t _k1 = vld1q_f32(k1);

  412.                     float32x4_t _sum = vmulq_f32(_r1, _k1);

  413.                     float32x4_t _sum2 = vmulq_f32(_r1, _k0123);

  414. 

  415.                     float32x4_t _r2 = vld1q_f32(r2);

  416.                     float32x4_t _k2 = vld1q_f32(k2);

  417.                     _sum = vmlaq_f32(_sum, _r2, _k2);

  418.                     _sum2 = vmlaq_f32(_sum2, _r2, _k1);

  419. 

  420.                     float32x4_t _r3 = vld1q_f32(r3);

  421.                     _sum = vmlaq_f32(_sum, _r3, _k20212223);

  422.                     _sum2 = vmlaq_f32(_sum2, _r3, _k2);

  423. 

  424.                     float32x4_t _r4 = vld1q_f32(r4);

  425.                     float32x4_t _k4 = vld1q_f32(k4);

  426.                     _sum = vmlaq_f32(_sum, _r4, _k4);

  427.                     _sum2 = vmlaq_f32(_sum2, _r4, _k20212223);

  428. 

  429.                     float32x4_t _r0 = vld1q_f32(r0);

  430.                     _sum = vmlaq_f32(_sum, _r0, _k0123);

  431.                     float32x4_t _r5 = vld1q_f32(r5);

  432.                     _sum2 = vmlaq_f32(_sum2, _r5, _k4);

  433. 

  434.                     float32x4_t _k_t4;

  435.                     _k_t4 = vsetq_lane_f32(k0[4], _k_t4, 0);

  436.                     _k_t4 = vsetq_lane_f32(k1[4], _k_t4, 1);

  437.                     _k_t4 = vsetq_lane_f32(k2[4], _k_t4, 2);

  438.                     _k_t4 = vsetq_lane_f32(k3[4], _k_t4, 3);

  439. 

  440.                     float32x4_t _r_t4;

  441. 

  442.                     _r_t4 = vsetq_lane_f32(r0[4], _r_t4, 0);

  443.                     _r_t4 = vsetq_lane_f32(r1[4], _r_t4, 1);

  444.                     _r_t4 = vsetq_lane_f32(r2[4], _r_t4, 2);

  445.                     _r_t4 = vsetq_lane_f32(r3[4], _r_t4, 3);

  446.                     _sum = vmlaq_f32(_sum, _r_t4, _k_t4);

  447. 

  448.                     sum = r4[4] * k4[4];

  449. 

  450.                     _r_t4 = vextq_f32(_r_t4, _r_t4, 1);

  451.                     _r_t4 = vsetq_lane_f32(r4[4], _r_t4, 3);

  452.                     _sum2 = vmlaq_f32(_sum2, _r_t4, _k_t4);

  453. 

  454.                     sum2 = r5[4] * k4[4];

  455. 

  456.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  457.                     float32x2_t _ss2 = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2));

  458.                     float32x2_t _ss_ss2 = vpadd_f32(_ss, _ss2);

  459. 

  460.                     sum += vget_lane_f32(_ss_ss2, 0);

  461.                     sum2 += vget_lane_f32(_ss_ss2, 1);

  462. #else

  463.                     sum += r0[0] * k0[0];

  464.                     sum += r0[1] * k0[1];

  465.                     sum += r0[2] * k0[2];

  466.                     sum += r0[3] * k0[3];

  467.                     sum += r0[4] * k0[4];

  468. 

  469.                     sum += r1[0] * k1[0];

  470.                     sum += r1[1] * k1[1];

  471.                     sum += r1[2] * k1[2];

  472.                     sum += r1[3] * k1[3];

  473.                     sum += r1[4] * k1[4];

  474. 

  475.                     sum += r2[0] * k2[0];

  476.                     sum += r2[1] * k2[1];

  477.                     sum += r2[2] * k2[2];

  478.                     sum += r2[3] * k2[3];

  479.                     sum += r2[4] * k2[4];

  480. 

  481.                     sum += r3[0] * k3[0];

  482.                     sum += r3[1] * k3[1];

  483.                     sum += r3[2] * k3[2];

  484.                     sum += r3[3] * k3[3];

  485.                     sum += r3[4] * k3[4];

  486. 

  487.                     sum += r4[0] * k4[0];

  488.                     sum += r4[1] * k4[1];

  489.                     sum += r4[2] * k4[2];

  490.                     sum += r4[3] * k4[3];

  491.                     sum += r4[4] * k4[4];

  492. 

  493.                     sum2 += r1[0] * k0[0];

  494.                     sum2 += r1[1] * k0[1];

  495.                     sum2 += r1[2] * k0[2];

  496.                     sum2 += r1[3] * k0[3];

  497.                     sum2 += r1[4] * k0[4];

  498. 

  499.                     sum2 += r2[0] * k1[0];

  500.                     sum2 += r2[1] * k1[1];

  501.                     sum2 += r2[2] * k1[2];

  502.                     sum2 += r2[3] * k1[3];

  503.                     sum2 += r2[4] * k1[4];

  504. 

  505.                     sum2 += r3[0] * k2[0];

  506.                     sum2 += r3[1] * k2[1];

  507.                     sum2 += r3[2] * k2[2];

  508.                     sum2 += r3[3] * k2[3];

  509.                     sum2 += r3[4] * k2[4];

  510. 

  511.                     sum2 += r4[0] * k3[0];

  512.                     sum2 += r4[1] * k3[1];

  513.                     sum2 += r4[2] * k3[2];

  514.                     sum2 += r4[3] * k3[3];

  515.                     sum2 += r4[4] * k3[4];

  516. 

  517.                     sum2 += r5[0] * k4[0];

  518.                     sum2 += r5[1] * k4[1];

  519.                     sum2 += r5[2] * k4[2];

  520.                     sum2 += r5[3] * k4[3];

  521.                     sum2 += r5[4] * k4[4];

  522. #endif // __ARM_NEON

  523.                     *outptr += sum;

  524.                     *outptr2 += sum2;

  525. 

  526.                     r0++;

  527.                     r1++;

  528.                     r2++;

  529.                     r3++;

  530.                     r4++;

  531.                     r5++;

  532.                     outptr++;

  533.                     outptr2++;

  534.                 }

  535. 

  536.                 r0 += 4 + w;

  537.                 r1 += 4 + w;

  538.                 r2 += 4 + w;

  539.                 r3 += 4 + w;

  540.                 r4 += 4 + w;

  541.                 r5 += 4 + w;

  542. 

  543.                 outptr += outw;

  544.                 outptr2 += outw;

  545.             }

  546. 

  547.             for (; i < outh; i++)

  548.             {

  549. 

  550. #if __ARM_NEON

  551.                 int nn = outw >> 2;

  552.                 int remain = outw - (nn << 2);

  553. #else

  554.                 int remain = outw;

  555. #endif // __ARM_NEON

  556. 

  557. #if __ARM_NEON

  558. #if __aarch64__

  559.                 for (; nn>0; nn--)

  560.                 {

  561.                     float32x4_t _sum = vdupq_n_f32(0.f);

  562. 

  563.                     float32x4_t _r00 = vld1q_f32(r0);

  564.                     float32x4_t _r04 = vld1q_f32(r0 + 4);

  565.                     float32x4_t _r01 = vextq_f32(_r00, _r04, 1);

  566.                     float32x4_t _r02 = vextq_f32(_r00, _r04, 2);

  567.                     float32x4_t _r03 = vextq_f32(_r00, _r04, 3);

  568. 

  569.                     float32x4_t _r10 = vld1q_f32(r1);

  570.                     float32x4_t _r14 = vld1q_f32(r1 + 4);

  571.                     float32x4_t _r11 = vextq_f32(_r10, _r14, 1);

  572.                     float32x4_t _r12 = vextq_f32(_r10, _r14, 2);

  573.                     float32x4_t _r13 = vextq_f32(_r10, _r14, 3);

  574. 

  575.                     float32x4_t _r20 = vld1q_f32(r2);

  576.                     float32x4_t _r24 = vld1q_f32(r2 + 4);

  577.                     float32x4_t _r21 = vextq_f32(_r20, _r24, 1);

  578.                     float32x4_t _r22 = vextq_f32(_r20, _r24, 2);

  579.                     float32x4_t _r23 = vextq_f32(_r20, _r24, 3);

  580. 

  581.                     float32x4_t _r30 = vld1q_f32(r3);

  582.                     float32x4_t _r34 = vld1q_f32(r3 + 4);

  583.                     float32x4_t _r31 = vextq_f32(_r30, _r34, 1);

  584.                     float32x4_t _r32 = vextq_f32(_r30, _r34, 2);

  585.                     float32x4_t _r33 = vextq_f32(_r30, _r34, 3);

  586. 

  587.                     float32x4_t _r40 = vld1q_f32(r4);

  588.                     float32x4_t _r44 = vld1q_f32(r4 + 4);

  589.                     float32x4_t _r41 = vextq_f32(_r40, _r44, 1);

  590.                     float32x4_t _r42 = vextq_f32(_r40, _r44, 2);

  591.                     float32x4_t _r43 = vextq_f32(_r40, _r44, 3);

  592. 

  593.                     _sum = vfmaq_laneq_f32(_sum, _r00, _k0123, 0);

  594.                     _sum = vfmaq_laneq_f32(_sum, _r01, _k0123, 1);

  595.                     _sum = vfmaq_laneq_f32(_sum, _r02, _k0123, 2);

  596.                     _sum = vfmaq_laneq_f32(_sum, _r03, _k0123, 3);

  597.                     _sum = vfmaq_laneq_f32(_sum, _r04, _k4567, 0);

  598. 

  599.                     _sum = vfmaq_laneq_f32(_sum, _r10, _k4567, 1);

  600.                     _sum = vfmaq_laneq_f32(_sum, _r11, _k4567, 2);

  601.                     _sum = vfmaq_laneq_f32(_sum, _r12, _k4567, 3);

  602.                     _sum = vfmaq_laneq_f32(_sum, _r13, _k891011, 0);

  603.                     _sum = vfmaq_laneq_f32(_sum, _r14, _k891011, 1);

  604. 

  605.                     _sum = vfmaq_laneq_f32(_sum, _r20, _k891011, 2);

  606.                     _sum = vfmaq_laneq_f32(_sum, _r21, _k891011, 3);

  607.                     _sum = vfmaq_laneq_f32(_sum, _r22, _k12131415, 0);

  608.                     _sum = vfmaq_laneq_f32(_sum, _r23, _k12131415, 1);

  609.                     _sum = vfmaq_laneq_f32(_sum, _r24, _k12131415, 2);

  610. 

  611.                     _sum = vfmaq_laneq_f32(_sum, _r30, _k12131415, 3);

  612.                     _sum = vfmaq_laneq_f32(_sum, _r31, _k16171819, 0);

  613.                     _sum = vfmaq_laneq_f32(_sum, _r32, _k16171819, 1);

  614.                     _sum = vfmaq_laneq_f32(_sum, _r33, _k16171819, 2);

  615.                     _sum = vfmaq_laneq_f32(_sum, _r34, _k16171819, 3);

  616. 

  617.                     _sum = vfmaq_laneq_f32(_sum, _r40, _k20212223, 0);

  618.                     _sum = vfmaq_laneq_f32(_sum, _r41, _k20212223, 1);

  619.                     _sum = vfmaq_laneq_f32(_sum, _r42, _k20212223, 2);

  620.                     _sum = vfmaq_laneq_f32(_sum, _r43, _k20212223, 3);

  621.                     _sum = vfmaq_laneq_f32(_sum, _r44, _k24242424, 0);

  622. 

  623.                     vst1q_f32(outptr, _sum);

  624. 

  625.                     r0 += 4;

  626.                     r1 += 4;

  627.                     r2 += 4;

  628.                     r3 += 4;

  629.                     r4 += 4;

  630.                     outptr += 4;

  631.                 }

  632. #else

  633.                 if (nn > 0)

  634.                 {

  635.                 asm volatile(

  636. //                     "veor       q15, q15            \n"// _sum3 = 0;

  637. 

  638.                     "pld        [%1, #128]          \n"

  639. 

  640.                     "pld        [%2, #256]          \n"

  641. 

  642.                     "vld1.f32   {d16-d19}, [%2]     \n"// _r00 = vld1q_f32(r0+j);

  643.                     "add        %2, #16             \n"

  644. 

  645.                     "0:                             \n"

  646. 

  647.                     "vld1.f32   {d14-d15}, [%1]     \n"// _sum = vld1q_f32(outptr+j);

  648.                     "veor       q13, q13            \n"// _sum2 = 0;

  649.                     "veor       q14, q14            \n"// _sum3 = 0;

  650. 

  651.                     "vext.32    q10, q8, q9, #1     \n"// _r01

  652.                     "vext.32    q11, q8, q9, #2     \n"// _r02

  653.                     "vext.32    q12, q8, q9, #3     \n"// _r03

  654. 

  655.                     "vmla.f32   q7, q8, %e14[0]     \n"

  656.                     "vmla.f32   q13, q10, %e14[1]   \n"

  657. 

  658.                     "pld        [%3, #256]          \n"

  659. 

  660.                     "vmla.f32   q14, q11, %f14[0]   \n"

  661.                     "vmul.f32   q15, q12, %f14[1]   \n"

  662.                     "vmla.f32   q7, q9, %e15[0]     \n"

  663. 

  664.                     "vld1.f32   {d16-d19}, [%3]     \n"

  665.                     "add        %3, #16             \n"

  666.                     "vext.32    q10, q8, q9, #1     \n"

  667.                     "vext.32    q11, q8, q9, #2     \n"

  668.                     "vext.32    q12, q8, q9, #3     \n"

  669. 

  670.                     "vmla.f32   q7, q8, %e15[1]     \n"

  671.                     "vmla.f32   q13, q10, %f15[0]   \n"

  672. 

  673.                     "pld        [%4, #256]          \n"

  674. 

  675.                     "vmla.f32   q14, q11, %f15[1]   \n"

  676.                     "vmla.f32   q15, q12, %e16[0]   \n"

  677.                     "vmla.f32   q7, q9, %e16[1]     \n"

  678. 

  679.                     "vld1.f32   {d16-d19}, [%4]     \n"

  680.                     "add        %4, #16             \n"

  681.                     "vext.32    q10, q8, q9, #1     \n"

  682.                     "vext.32    q11, q8, q9, #2     \n"

  683.                     "vext.32    q12, q8, q9, #3     \n"

  684. 

  685.                     "vmla.f32   q7, q8, %f16[0]     \n"

  686.                     "vmla.f32   q13, q10, %f16[1]   \n"

  687. 

  688.                     "pld        [%5, #256]          \n"

  689. 

  690.                     "vmla.f32   q14, q11, %e17[0]   \n"

  691.                     "vmla.f32   q15, q12, %e17[1]   \n"

  692.                     "vmla.f32   q7, q9, %f17[0]     \n"

  693. 

  694.                     "vld1.f32   {d16-d19}, [%5]     \n"

  695.                     "add        %5, #16             \n"

  696.                     "vext.32    q10, q8, q9, #1     \n"

  697.                     "vext.32    q11, q8, q9, #2     \n"

  698.                     "vext.32    q12, q8, q9, #3     \n"

  699. 

  700.                     "vmla.f32   q7, q8, %f17[1]     \n"

  701.                     "vmla.f32   q13, q10, %e18[0]   \n"

  702. 

  703.                     "pld        [%6, #256]          \n"

  704. 

  705.                     "vmla.f32   q14, q11, %e18[1]   \n"

  706.                     "vmla.f32   q15, q12, %f18[0]   \n"

  707.                     "vmla.f32   q7, q9, %f18[1]     \n"

  708. 

  709.                     "vld1.f32   {d16-d19}, [%6]     \n"

  710.                     "add        %6, #16             \n"

  711.                     "vext.32    q10, q8, q9, #1     \n"

  712.                     "vext.32    q11, q8, q9, #2     \n"

  713.                     "vext.32    q12, q8, q9, #3     \n"

  714. 

  715.                     "vmla.f32   q7, q8, %e19[0]     \n"

  716.                     "vmla.f32   q13, q10, %e19[1]   \n"

  717.                     "vmla.f32   q14, q11, %f19[0]   \n"

  718.                     "vmla.f32   q15, q12, %f19[1]   \n"

  719.                     "vmla.f32   q7, q9, %e20[0]     \n"

  720. 

  721.                     "vadd.f32   q14, q14, q15       \n"

  722.                     "vadd.f32   q7, q7, q13         \n"

  723. //                     "veor       q15, q15            \n"// _sum3 = 0;

  724. 

  725.                     "pld        [%2, #256]          \n"

  726. 

  727.                     "vadd.f32   q7, q7, q14         \n"

  728. 

  729.                     "vld1.f32   {d16-d19}, [%2]     \n"// _r00 = vld1q_f32(r0+j);

  730.                     "add        %2, #16             \n"

  731. 

  732.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  733. 

  734.                     "pld        [%1, #128]          \n"

  735. 

  736.                     "subs       %0, #1              \n"

  737.                     "bne        0b                  \n"

  738. 

  739.                     "sub        %2, #16             \n"

  740.                     : "=r"(nn),         // %0

  741.                       "=r"(outptr),     // %1

  742.                       "=r"(r0),         // %2

  743.                       "=r"(r1),         // %3

  744.                       "=r"(r2),         // %4

  745.                       "=r"(r3),         // %5

  746.                       "=r"(r4)          // %6

  747.                     : "0"(nn),

  748.                       "1"(outptr),

  749.                       "2"(r0),

  750.                       "3"(r1),

  751.                       "4"(r2),

  752.                       "5"(r3),

  753.                       "6"(r4),

  754.                       "w"(_k0123),      // %14

  755.                       "w"(_k4567),      // %15

  756.                       "w"(_k891011),    // %16

  757.                       "w"(_k12131415),  // %17

  758.                       "w"(_k16171819),  // %18

  759.                       "w"(_k20212223),  // %19

  760.                       "w"(_k24242424)   // %20

  761.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  762.                 );

  763.                 }

  764. #endif // __aarch64__

  765. #endif // __ARM_NEON

  766.                 for (; remain>0; remain--)

  767.                 {

  768.                     float sum = 0;

  769. #if __ARM_NEON

  770.                     float32x4_t _r0 = vld1q_f32(r0);

  771.                     float32x4_t _sum = vmulq_f32(_r0, _k0123);

  772. 

  773.                     float32x4_t _r1 = vld1q_f32(r1);

  774.                     _sum = vmlaq_f32(_sum, _r1, vld1q_f32(k1));

  775. 

  776.                     float32x4_t _r2 = vld1q_f32(r2);

  777.                     _sum = vmlaq_f32(_sum, _r2, vld1q_f32(k2));

  778. 

  779.                     float32x4_t _r3 = vld1q_f32(r3);

  780.                     _sum = vmlaq_f32(_sum, _r3, _k20212223);

  781. 

  782.                     float32x4_t _r4 = vld1q_f32(r4);

  783.                     _sum = vmlaq_f32(_sum, _r4, vld1q_f32(k4));

  784. 

  785.                     float32x4_t _k_t4;

  786.                     _k_t4 = vsetq_lane_f32(k0[4], _k_t4, 0);

  787.                     _k_t4 = vsetq_lane_f32(k1[4], _k_t4, 1);

  788.                     _k_t4 = vsetq_lane_f32(k2[4], _k_t4, 2);

  789.                     _k_t4 = vsetq_lane_f32(k3[4], _k_t4, 3);

  790. 

  791.                     float32x4_t _r_t4;

  792. 

  793.                     _r_t4 = vsetq_lane_f32(r0[4], _r_t4, 0);

  794.                     _r_t4 = vsetq_lane_f32(r1[4], _r_t4, 1);

  795.                     _r_t4 = vsetq_lane_f32(r2[4], _r_t4, 2);

  796.                     _r_t4 = vsetq_lane_f32(r3[4], _r_t4, 3);

  797.                     _sum = vmlaq_f32(_sum, _r_t4, _k_t4);

  798. 

  799.                     sum = r4[4] * k4[4];

  800. 

  801.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  802.                     _ss = vpadd_f32(_ss, _ss);

  803. 

  804.                     sum += vget_lane_f32(_ss, 0);

  805. #else

  806.                     sum += r0[0] * k0[0];

  807.                     sum += r0[1] * k0[1];

  808.                     sum += r0[2] * k0[2];

  809.                     sum += r0[3] * k0[3];

  810.                     sum += r0[4] * k0[4];

  811. 

  812.                     sum += r1[0] * k1[0];

  813.                     sum += r1[1] * k1[1];

  814.                     sum += r1[2] * k1[2];

  815.                     sum += r1[3] * k1[3];

  816.                     sum += r1[4] * k1[4];

  817. 

  818.                     sum += r2[0] * k2[0];

  819.                     sum += r2[1] * k2[1];

  820.                     sum += r2[2] * k2[2];

  821.                     sum += r2[3] * k2[3];

  822.                     sum += r2[4] * k2[4];

  823. 

  824.                     sum += r3[0] * k3[0];

  825.                     sum += r3[1] * k3[1];

  826.                     sum += r3[2] * k3[2];

  827.                     sum += r3[3] * k3[3];

  828.                     sum += r3[4] * k3[4];

  829. 

  830.                     sum += r4[0] * k4[0];

  831.                     sum += r4[1] * k4[1];

  832.                     sum += r4[2] * k4[2];

  833.                     sum += r4[3] * k4[3];

  834.                     sum += r4[4] * k4[4];

  835. #endif

  836.                     *outptr += sum;

  837. 

  838.                     r0++;

  839.                     r1++;

  840.                     r2++;

  841.                     r3++;

  842.                     r4++;

  843.                     outptr++;

  844.                 }

  845. 

  846.                 r0 += 4;

  847.                 r1 += 4;

  848.                 r2 += 4;

  849.                 r3 += 4;

  850.                 r4 += 4;

  851. 

  852.             }

  853. 

  854.         }

  855.     }

  856. 

  857. }

  858. 

  859. static void conv5x5s2_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias)

  860. {

  861.     int w = bottom_blob.w;

  862.     int h = bottom_blob.h;

  863.     int inch = bottom_blob.c;

  864. 

  865.     int outw = top_blob.w;

  866.     int outh = top_blob.h;

  867.     int outch = top_blob.c;

  868. 

  869.     const int tailstep = w - 2*outw + w;

  870. 

  871.     const float* kernel = _kernel;

  872.     const float* bias = _bias;

  873. 

  874.     #pragma omp parallel for

  875.     for (int p=0; p<outch; p++)

  876.     {

  877.         Mat out = top_blob.channel(p);

  878. 

  879.         const float bias0 = bias ? bias[p] : 0.f;

  880. 

  881.         out.fill(bias0);

  882. 

  883.         for (int q=0; q<inch; q++)

  884.         {

  885.             float* outptr = out;

  886. 

  887.             const float* img0 = bottom_blob.channel(q);

  888. 

  889.             const float* kernel0 = kernel + p*inch*25  + q*25;

  890. 

  891.             const float* r0 = img0;

  892.             const float* r1 = img0 + w;

  893.             const float* r2 = img0 + w*2;

  894.             const float* r3 = img0 + w*3;

  895.             const float* r4 = img0 + w*4;

  896. 

  897.             const float* k0 = kernel0;

  898.             const float* k1 = kernel0 + 5;

  899.             const float* k2 = kernel0 + 10;

  900.             const float* k3 = kernel0 + 15;

  901.             const float* k4 = kernel0 + 20;

  902. 

  903. #if __ARM_NEON

  904.             float32x4_t _k0123 = vld1q_f32(kernel0);

  905.             float32x4_t _k4567 = vld1q_f32(kernel0+4);

  906.             float32x4_t _k891011 = vld1q_f32(kernel0+8);

  907.             float32x4_t _k12131415 = vld1q_f32(kernel0+12);

  908.             float32x4_t _k16171819 = vld1q_f32(kernel0+16);

  909.             float32x4_t _k20212223 = vld1q_f32(kernel0+20);

  910.             float32x4_t _k24242424 = vdupq_n_f32(kernel0[24]);

  911. #endif // __ARM_NEON

  912. 

  913.             for (int i = 0; i < outh; i++)

  914.             {

  915. 

  916. #if __ARM_NEON

  917.                 int nn = outw >> 2;

  918.                 int remain = outw - (nn << 2);

  919. #else

  920.                 int remain = outw;

  921. #endif // __ARM_NEON

  922. 

  923. #if __ARM_NEON

  924. #if __aarch64__

  925.                 for (; nn>0; nn--)

  926.                 {

  927.                     float32x4_t _sum = vdupq_n_f32(0.f);

  928. 

  929.                     float32x4x2_t _r00_02461357 = vld2q_f32(r0);

  930.                     float32x4x2_t _r00nx2 = vld2q_f32(r0 + 8);

  931.                     float32x4_t _r0_8101214 = _r00nx2.val[0];// 8 10 12 14

  932.                     float32x4_t _r0_9111315 = _r00nx2.val[1];// 9 11 13 15

  933.                     float32x4_t _r00 = _r00_02461357.val[0];// 0 2 4 6

  934.                     float32x4_t _r01 = _r00_02461357.val[1];// 1 3 5 7

  935.                     float32x4_t _r02 = vextq_f32(_r00, _r0_8101214, 1);// 2 4 6 8

  936.                     float32x4_t _r03 = vextq_f32(_r01, _r0_9111315, 1);// 3 5 7 9

  937.                     float32x4_t _r04 = vextq_f32(_r00, _r0_8101214, 2);// 4 6 8 10

  938. 

  939.                     float32x4x2_t _r10_02461357 = vld2q_f32(r1);

  940.                     float32x4x2_t _r10nx2 = vld2q_f32(r1 + 8);

  941.                     float32x4_t _r1_8101214 = _r10nx2.val[0];

  942.                     float32x4_t _r1_9111315 = _r10nx2.val[1];

  943.                     float32x4_t _r10 = _r10_02461357.val[0];

  944.                     float32x4_t _r11 = _r10_02461357.val[1];

  945.                     float32x4_t _r12 = vextq_f32(_r10, _r1_8101214, 1);

  946.                     float32x4_t _r13 = vextq_f32(_r11, _r1_9111315, 1);

  947.                     float32x4_t _r14 = vextq_f32(_r10, _r1_8101214, 2);

  948. 

  949.                     float32x4x2_t _r20_02461357 = vld2q_f32(r2);

  950.                     float32x4x2_t _r20nx2 = vld2q_f32(r2 + 8);

  951.                     float32x4_t _r2_8101214 = _r20nx2.val[0];

  952.                     float32x4_t _r2_9111315 = _r20nx2.val[1];

  953.                     float32x4_t _r20 = _r20_02461357.val[0];

  954.                     float32x4_t _r21 = _r20_02461357.val[1];

  955.                     float32x4_t _r22 = vextq_f32(_r20, _r2_8101214, 1);

  956.                     float32x4_t _r23 = vextq_f32(_r21, _r2_9111315, 1);

  957.                     float32x4_t _r24 = vextq_f32(_r20, _r2_8101214, 2);

  958. 

  959.                     float32x4x2_t _r30_02461357 = vld2q_f32(r3);

  960.                     float32x4x2_t _r30nx2 = vld2q_f32(r3 + 8);

  961.                     float32x4_t _r3_8101214 = _r30nx2.val[0];

  962.                     float32x4_t _r3_9111315 = _r30nx2.val[1];

  963.                     float32x4_t _r30 = _r30_02461357.val[0];

  964.                     float32x4_t _r31 = _r30_02461357.val[1];

  965.                     float32x4_t _r32 = vextq_f32(_r30, _r3_8101214, 1);

  966.                     float32x4_t _r33 = vextq_f32(_r31, _r3_9111315, 1);

  967.                     float32x4_t _r34 = vextq_f32(_r30, _r3_8101214, 2);

  968. 

  969.                     float32x4x2_t _r40_02461357 = vld2q_f32(r4);

  970.                     float32x4x2_t _r40nx2 = vld2q_f32(r4 + 8);

  971.                     float32x4_t _r4_8101214 = _r40nx2.val[0];

  972.                     float32x4_t _r4_9111315 = _r40nx2.val[1];

  973.                     float32x4_t _r40 = _r40_02461357.val[0];

  974.                     float32x4_t _r41 = _r40_02461357.val[1];

  975.                     float32x4_t _r42 = vextq_f32(_r40, _r4_8101214, 1);

  976.                     float32x4_t _r43 = vextq_f32(_r41, _r4_9111315, 1);

  977.                     float32x4_t _r44 = vextq_f32(_r40, _r4_8101214, 2);

  978. 

  979.                     _sum = vfmaq_laneq_f32(_sum, _r00, _k0123, 0);

  980.                     _sum = vfmaq_laneq_f32(_sum, _r01, _k0123, 1);

  981.                     _sum = vfmaq_laneq_f32(_sum, _r02, _k0123, 2);

  982.                     _sum = vfmaq_laneq_f32(_sum, _r03, _k0123, 3);

  983.                     _sum = vfmaq_laneq_f32(_sum, _r04, _k4567, 0);

  984. 

  985.                     _sum = vfmaq_laneq_f32(_sum, _r10, _k4567, 1);

  986.                     _sum = vfmaq_laneq_f32(_sum, _r11, _k4567, 2);

  987.                     _sum = vfmaq_laneq_f32(_sum, _r12, _k4567, 3);

  988.                     _sum = vfmaq_laneq_f32(_sum, _r13, _k891011, 0);

  989.                     _sum = vfmaq_laneq_f32(_sum, _r14, _k891011, 1);

  990. 

  991.                     _sum = vfmaq_laneq_f32(_sum, _r20, _k891011, 2);

  992.                     _sum = vfmaq_laneq_f32(_sum, _r21, _k891011, 3);

  993.                     _sum = vfmaq_laneq_f32(_sum, _r22, _k12131415, 0);

  994.                     _sum = vfmaq_laneq_f32(_sum, _r23, _k12131415, 1);

  995.                     _sum = vfmaq_laneq_f32(_sum, _r24, _k12131415, 2);

  996. 

  997.                     _sum = vfmaq_laneq_f32(_sum, _r30, _k12131415, 3);

  998.                     _sum = vfmaq_laneq_f32(_sum, _r31, _k16171819, 0);

  999.                     _sum = vfmaq_laneq_f32(_sum, _r32, _k16171819, 1);

  1000.                     _sum = vfmaq_laneq_f32(_sum, _r33, _k16171819, 2);

  1001.                     _sum = vfmaq_laneq_f32(_sum, _r34, _k16171819, 3);

  1002. 

  1003.                     _sum = vfmaq_laneq_f32(_sum, _r40, _k20212223, 0);

  1004.                     _sum = vfmaq_laneq_f32(_sum, _r41, _k20212223, 1);

  1005.                     _sum = vfmaq_laneq_f32(_sum, _r42, _k20212223, 2);

  1006.                     _sum = vfmaq_laneq_f32(_sum, _r43, _k20212223, 3);

  1007.                     _sum = vfmaq_laneq_f32(_sum, _r44, _k24242424, 0);

  1008. 

  1009.                     vst1q_f32(outptr, _sum);

  1010. 

  1011.                     r0 += 8;

  1012.                     r1 += 8;

  1013.                     r2 += 8;

  1014.                     r3 += 8;

  1015.                     r4 += 8;

  1016.                     outptr += 4;

  1017.                 }

  1018. #else

  1019.                 if (nn > 0)

  1020.                 {

  1021.                 asm volatile(

  1022.                     "veor       q15, q15            \n"// _sump3 = 0;

  1023.                     "pld        [%1, #128]          \n"

  1024.                     "veor       q13, q13            \n"// _sump2 = 0;

  1025.                     "pld        [%2, #256]          \n"

  1026.                     "veor       q14, q14            \n"// _sump3 = 0;

  1027. 

  1028.                     "vld2.f32   {d16-d19}, [%2]!    \n"// q8  = 0  2  4  6   q9  = 1  3  5  7

  1029. 

  1030.                     "pld        [%2, #256]          \n"

  1031. 

  1032.                     "vld2.f32   {d20-d23}, [%2]     \n"// q10 = 8 10 12 14   q11 = 9 11 13 15

  1033. 

  1034.                     "0:                             \n"

  1035. 

  1036.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = outptr

  1037. 

  1038.                     "vext.32    q12, q8, q10, #1    \n"// q12 = 2 4 6 8

  1039.                     "vext.32    q11, q9, q11, #1    \n"// q11 = 3 5 7 9

  1040.                     "vext.32    q10, q8, q10, #2    \n"// q10 = 4 6 8 10

  1041. 

  1042.                     "vmla.f32   q7, q8, %e14[0]     \n"

  1043.                     "vmla.f32   q13, q9, %e14[1]    \n"

  1044. 

  1045.                     "pld        [%3, #256]          \n"

  1046. 

  1047.                     "vmla.f32   q14, q12, %f14[0]   \n"

  1048.                     "vmla.f32   q15, q11, %f14[1]   \n"

  1049.                     "vmla.f32   q7, q10, %e15[0]    \n"

  1050. 

  1051.                     "vld2.f32   {d16-d19}, [%3]!    \n"

  1052. 

  1053.                     "pld        [%3, #256]          \n"

  1054. 

  1055.                     "vld2.f32   {d20-d23}, [%3]     \n"

  1056.                     "vext.32    q12, q8, q10, #1    \n"

  1057.                     "vext.32    q11, q9, q11, #1    \n"

  1058.                     "vext.32    q10, q8, q10, #2    \n"

  1059. 

  1060.                     "vmla.f32   q7, q8, %e15[1]     \n"

  1061.                     "vmla.f32   q13, q9, %f15[0]    \n"

  1062. 

  1063.                     "pld        [%4, #256]          \n"

  1064. 

  1065.                     "vmla.f32   q14, q12, %f15[1]   \n"

  1066.                     "vmla.f32   q15, q11, %e16[0]   \n"

  1067.                     "vmla.f32   q7, q10, %e16[1]    \n"

  1068. 

  1069.                     "vld2.f32   {d16-d19}, [%4]!    \n"

  1070. 

  1071.                     "pld        [%4, #256]          \n"

  1072. 

  1073.                     "vld2.f32   {d20-d23}, [%4]     \n"

  1074.                     "vext.32    q12, q8, q10, #1    \n"

  1075.                     "vext.32    q11, q9, q11, #1    \n"

  1076.                     "vext.32    q10, q8, q10, #2    \n"

  1077. 

  1078.                     "vmla.f32   q7, q8, %f16[0]     \n"

  1079.                     "vmla.f32   q13, q9, %f16[1]    \n"

  1080. 

  1081.                     "pld        [%5, #256]          \n"

  1082. 

  1083.                     "vmla.f32   q14, q12, %e17[0]   \n"

  1084.                     "vmla.f32   q15, q11, %e17[1]   \n"

  1085.                     "vmla.f32   q7, q10, %f17[0]    \n"

  1086. 

  1087.                     "vld2.f32   {d16-d19}, [%5]!    \n"

  1088. 

  1089.                     "pld        [%5, #256]          \n"

  1090. 

  1091.                     "vld2.f32   {d20-d23}, [%5]     \n"

  1092.                     "vext.32    q12, q8, q10, #1    \n"

  1093.                     "vext.32    q11, q9, q11, #1    \n"

  1094.                     "vext.32    q10, q8, q10, #2    \n"

  1095. 

  1096.                     "vmla.f32   q7, q8, %f17[1]     \n"

  1097.                     "vmla.f32   q13, q9, %e18[0]    \n"

  1098. 

  1099.                     "pld        [%6, #256]          \n"

  1100. 

  1101.                     "vmla.f32   q14, q12, %e18[1]   \n"

  1102.                     "vmla.f32   q15, q11, %f18[0]   \n"

  1103.                     "vmla.f32   q7, q10, %f18[1]    \n"

  1104. 

  1105.                     "vld2.f32   {d16-d19}, [%6]!    \n"

  1106. 

  1107.                     "pld        [%6, #256]          \n"

  1108. 

  1109.                     "vld2.f32   {d20-d23}, [%6]     \n"

  1110.                     "vext.32    q12, q8, q10, #1    \n"

  1111.                     "vext.32    q11, q9, q11, #1    \n"

  1112.                     "vext.32    q10, q8, q10, #2    \n"

  1113. 

  1114.                     "vmla.f32   q7, q8, %e19[0]     \n"

  1115.                     "vmla.f32   q13, q9, %e19[1]    \n"

  1116.                     "vmla.f32   q14, q12, %f19[0]   \n"

  1117.                     "vmla.f32   q15, q11, %f19[1]   \n"

  1118.                     "vmla.f32   q7, q10, %e20[0]    \n"

  1119. 

  1120.                     "pld        [%2, #256]          \n"

  1121. 

  1122.                     "vld2.f32   {d16-d19}, [%2]!    \n"// q8  = 0  2  4  6   q9  = 1  3  5  7

  1123. 

  1124.                     "vadd.f32   q14, q14, q15       \n"

  1125.                     "vadd.f32   q7, q7, q13         \n"

  1126.                     "veor       q15, q15            \n"// _sump3 = 0;

  1127.                     "veor       q13, q13            \n"// _sump2 = 0;

  1128. 

  1129.                     "pld        [%2, #256]          \n"

  1130. 

  1131.                     "vadd.f32   q7, q7, q14         \n"

  1132. 

  1133.                     "vld2.f32   {d20-d23}, [%2]     \n"// q10 = 8 10 12 14   q11 = 9 11 13 15

  1134. 

  1135.                     "veor       q14, q14            \n"// _sump3 = 0;

  1136. 

  1137.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  1138. 

  1139.                     "pld        [%1, #128]          \n"

  1140. 

  1141.                     "subs       %0, #1              \n"

  1142.                     "bne        0b                  \n"

  1143. 

  1144.                     "sub        %2, #32             \n"

  1145.                     : "=r"(nn),         // %0

  1146.                       "=r"(outptr),     // %1

  1147.                       "=r"(r0),         // %2

  1148.                       "=r"(r1),         // %3

  1149.                       "=r"(r2),         // %4

  1150.                       "=r"(r3),         // %5

  1151.                       "=r"(r4)          // %6

  1152.                     : "0"(nn),

  1153.                       "1"(outptr),

  1154.                       "2"(r0),

  1155.                       "3"(r1),

  1156.                       "4"(r2),

  1157.                       "5"(r3),

  1158.                       "6"(r4),

  1159.                       "w"(_k0123),      // %14

  1160.                       "w"(_k4567),      // %15

  1161.                       "w"(_k891011),    // %16

  1162.                       "w"(_k12131415),  // %17

  1163.                       "w"(_k16171819),  // %18

  1164.                       "w"(_k20212223),  // %19

  1165.                       "w"(_k24242424)   // %20

  1166.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  1167.                 );

  1168.                 }

  1169. #endif // __aarch64__

  1170. #endif // __ARM_NEON

  1171.                 for (; remain>0; remain--)

  1172.                 {

  1173.                     float sum = 0;

  1174. #if __ARM_NEON

  1175.                     float32x4_t _r0 = vld1q_f32(r0);

  1176.                     float32x4_t _sum = vmulq_f32(_r0, _k0123);

  1177. 

  1178.                     float32x4_t _r1 = vld1q_f32(r1);

  1179.                     _sum = vmlaq_f32(_sum, _r1, vld1q_f32(k1));

  1180. 

  1181.                     float32x4_t _r2 = vld1q_f32(r2);

  1182.                     _sum = vmlaq_f32(_sum, _r2, vld1q_f32(k2));

  1183. 

  1184.                     float32x4_t _r3 = vld1q_f32(r3);

  1185.                     _sum = vmlaq_f32(_sum, _r3, vld1q_f32(k3));

  1186. 

  1187.                     float32x4_t _r4 = vld1q_f32(r4);

  1188.                     _sum = vmlaq_f32(_sum, _r4, _k20212223);

  1189. 

  1190.                     sum += r0[4] * k0[4];

  1191.                     sum += r1[4] * k1[4];

  1192.                     sum += r2[4] * k2[4];

  1193.                     sum += r3[4] * k3[4];

  1194.                     sum += r4[4] * k4[4];

  1195. 

  1196.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  1197.                     _ss = vpadd_f32(_ss, _ss);

  1198. 

  1199.                     sum += vget_lane_f32(_ss, 0);

  1200. #else

  1201.                     sum += r0[0] * k0[0];

  1202.                     sum += r0[1] * k0[1];

  1203.                     sum += r0[2] * k0[2];

  1204.                     sum += r0[3] * k0[3];

  1205.                     sum += r0[4] * k0[4];

  1206. 

  1207.                     sum += r1[0] * k1[0];

  1208.                     sum += r1[1] * k1[1];

  1209.                     sum += r1[2] * k1[2];

  1210.                     sum += r1[3] * k1[3];

  1211.                     sum += r1[4] * k1[4];

  1212. 

  1213.                     sum += r2[0] * k2[0];

  1214.                     sum += r2[1] * k2[1];

  1215.                     sum += r2[2] * k2[2];

  1216.                     sum += r2[3] * k2[3];

  1217.                     sum += r2[4] * k2[4];

  1218. 

  1219.                     sum += r3[0] * k3[0];

  1220.                     sum += r3[1] * k3[1];

  1221.                     sum += r3[2] * k3[2];

  1222.                     sum += r3[3] * k3[3];

  1223.                     sum += r3[4] * k3[4];

  1224. 

  1225.                     sum += r4[0] * k4[0];

  1226.                     sum += r4[1] * k4[1];

  1227.                     sum += r4[2] * k4[2];

  1228.                     sum += r4[3] * k4[3];

  1229.                     sum += r4[4] * k4[4];

  1230. #endif

  1231.                     *outptr += sum;

  1232. 

  1233.                     r0 += 2;

  1234.                     r1 += 2;

  1235.                     r2 += 2;

  1236.                     r3 += 2;

  1237.                     r4 += 2;

  1238.                     outptr++;

  1239.                 }

  1240. 

  1241.                 r0 += tailstep;

  1242.                 r1 += tailstep;

  1243.                 r2 += tailstep;

  1244.                 r3 += tailstep;

  1245.                 r4 += tailstep;

  1246.             }

  1247. 

  1248.         }

  1249.     }

  1250. 

  1251. }