nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: e213605cd4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e213605cd4'
${ noResults }
ncnn/src/layer/arm/convolution_5x5.h

1424 lines
56 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #if __ARM_NEON

  16. #include <arm_neon.h>

  17. #endif // __ARM_NEON

  18. 

  19. static void conv5x5s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  20. {

  21.     int w = bottom_blob.w;

  22.     int inch = bottom_blob.c;

  23. 

  24.     int outw = top_blob.w;

  25.     int outh = top_blob.h;

  26.     int outch = top_blob.c;

  27. 

  28.     const float* kernel = _kernel;

  29.     const float* bias = _bias;

  30. 

  31.     #pragma omp parallel for num_threads(opt.num_threads)

  32.     for (int p=0; p<outch; p++)

  33.     {

  34.         Mat out = top_blob.channel(p);

  35. 

  36.         const float bias0 = bias ? bias[p] : 0.f;

  37. 

  38.         out.fill(bias0);

  39. 

  40.         for (int q=0; q<inch; q++)

  41.         {

  42.             float* outptr = out;

  43.             float* outptr2 = outptr + outw;

  44. 

  45.             const float* img0 = bottom_blob.channel(q);

  46. 

  47.             const float* kernel0 = kernel + p*inch*25  + q*25;

  48. 

  49.             const float* r0 = img0;

  50.             const float* r1 = img0 + w;

  51.             const float* r2 = img0 + w*2;

  52.             const float* r3 = img0 + w*3;

  53.             const float* r4 = img0 + w*4;

  54.             const float* r5 = img0 + w*5;

  55. 

  56.             const float* k0 = kernel0;

  57.             const float* k1 = kernel0 + 5;

  58.             const float* k2 = kernel0 + 10;

  59.             const float* k3 = kernel0 + 15;

  60.             const float* k4 = kernel0 + 20;

  61. 

  62. #if __ARM_NEON

  63.             float32x4_t _k0123 = vld1q_f32(kernel0);

  64.             float32x4_t _k4567 = vld1q_f32(kernel0+4);

  65.             float32x4_t _k891011 = vld1q_f32(kernel0+8);

  66.             float32x4_t _k12131415 = vld1q_f32(kernel0+12);

  67.             float32x4_t _k16171819 = vld1q_f32(kernel0+16);

  68.             float32x4_t _k20212223 = vld1q_f32(kernel0+20);

  69.             float32x4_t _k24242424 = vdupq_n_f32(kernel0[24]);

  70. #endif // __ARM_NEON

  71. 

  72.             int i = 0;

  73. 

  74.             for (; i+1 < outh; i+=2)

  75.             {

  76. 

  77. #if __ARM_NEON

  78.                 int nn = outw >> 2;

  79.                 int remain = outw - (nn << 2);

  80. #else

  81.                 int remain = outw;

  82. #endif // __ARM_NEON

  83. 

  84. #if __ARM_NEON

  85. #if __aarch64__

  86.                 if (nn > 0)

  87.                 {

  88.                 asm volatile(

  89.                     // v11 = rx1 / rx3

  90.                     // v12 = rx2

  91.                     // v13 v14 = intermediate sum register

  92.                     

  93.                     "prfm       pldl1keep, [%1, #128]          \n"

  94.                     "ld1        {v7.4s}, [%1]                  \n"// v7 = out

  95. 

  96.                     "0:                                        \n"

  97. 

  98.                     "prfm       pldl1keep, [%2, #128]          \n"

  99.                     "ld1        {v8.4s}, [%2]                  \n"// v8 = out2

  100. 

  101.                     // r1

  102.                     "prfm       pldl1keep, [%4, #256]          \n"

  103.                     "ld1        {v9.4s, v10.4s}, [%4]          \n"// v9 v10 = r10 r14

  104.                     "add        %4, %4, #16                    \n"

  105. 

  106.                     "ext        v11.16b, v9.16b, v10.16b, #4   \n" //r11

  107.                     "fmul       v13.4s, v9.4s, %19.s[1]        \n"

  108.                     "fmla       v8.4s,  v9.4s, %18.s[0]        \n"

  109. 

  110.                     "ext        v12.16b, v9.16b, v10.16b, #8   \n" //r12

  111.                     "fmla       v7.4s,  v11.4s, %19.s[2]       \n"

  112.                     "fmul       v14.4s, v11.4s, %18.s[1]       \n"

  113. 

  114.                     "ext        v11.16b, v9.16b, v10.16b, #12  \n" //r13

  115.                     "fmla       v13.4s, v12.4s, %19.s[3]       \n"

  116.                     "fmla       v8.4s,  v12.4s, %18.s[2]       \n"

  117. 

  118.                     "fmla       v7.4s,  v11.4s, %20.s[0]       \n"

  119.                     "fmla       v14.4s, v11.4s, %18.s[3]       \n"

  120. 

  121.                     "prfm       pldl1keep, [%5, #256]          \n"

  122. 

  123.                     "fmla       v13.4s, v10.4s, %20.s[1]       \n"

  124.                     "fmla       v8.4s,  v10.4s, %19.s[0]       \n"

  125. 

  126.                     // r2

  127.                     "ld1        {v9.4s, v10.4s}, [%5]          \n"// v9 v10 = r20 r24

  128.                     "add        %5, %5, #16                    \n"

  129. 

  130.                     "ext        v11.16b, v9.16b, v10.16b, #4   \n" //r21

  131.                     "fmla       v7.4s,  v9.4s, %20.s[2]        \n"

  132.                     "fmla       v14.4s, v9.4s, %19.s[1]        \n"                    

  133. 

  134.                     "ext        v12.16b, v9.16b, v10.16b, #8   \n" //r22

  135.                     "fmla       v13.4s, v11.4s, %20.s[3]       \n"

  136.                     "fmla       v8.4s,  v11.4s, %19.s[2]       \n"   

  137.                     

  138.                     "ext        v11.16b, v9.16b, v10.16b, #12  \n" //r23

  139.                     "fmla       v7.4s,  v12.4s, %21.s[0]       \n"

  140.                     "fmla       v14.4s, v12.4s, %19.s[3]       \n"   

  141. 

  142.                     "fmla       v13.4s, v11.4s, %21.s[1]       \n"

  143.                     "fmla       v8.4s,  v11.4s, %20.s[0]       \n" 

  144. 

  145.                     "prfm       pldl1keep, [%6, #256]          \n"

  146. 

  147.                     "fmla       v7.4s,  v10.4s, %21.s[2]       \n"

  148.                     "fmla       v14.4s, v10.4s, %20.s[1]       \n" 

  149.                     

  150.                     // r3

  151.                     "ld1        {v9.4s, v10.4s}, [%6]          \n"// v9 v10 = r30 r34

  152.                     "add        %6, %6, #16                    \n"

  153. 

  154.                     "ext        v11.16b, v9.16b, v10.16b, #4   \n" //r31

  155.                     "fmla       v13.4s, v9.4s, %21.s[3]        \n"

  156.                     "fmla       v8.4s,  v9.4s, %20.s[2]        \n"   

  157. 

  158.                     "ext        v12.16b, v9.16b, v10.16b, #8   \n" //r32

  159.                     "fmla       v7.4s,  v11.4s, %22.s[0]       \n"

  160.                     "fmla       v14.4s, v11.4s, %20.s[3]       \n"   

  161. 

  162.                     "ext        v11.16b, v9.16b, v10.16b, #12  \n" //r33

  163.                     "fmla       v13.4s, v12.4s, %22.s[1]       \n"

  164.                     "fmla       v8.4s,  v12.4s, %21.s[0]       \n"   

  165. 

  166.                     "fmla       v7.4s,  v11.4s, %22.s[2]       \n"

  167.                     "fmla       v14.4s, v11.4s, %21.s[1]       \n"   

  168. 

  169.                     "prfm       pldl1keep, [%7, #256]          \n"

  170. 

  171.                     "fmla       v13.4s, v10.4s, %22.s[3]       \n"

  172.                     "fmla       v8.4s,  v10.4s, %21.s[2]       \n"  

  173. 

  174.                     // r4

  175.                     "ld1        {v9.4s, v10.4s}, [%7]          \n"// v9 v10 = r40 r44

  176.                     "add        %7, %7, #16                    \n"

  177. 

  178.                     "ext        v11.16b, v9.16b, v10.16b, #4   \n" //r41

  179.                     "fmla       v7.4s,  v9.4s, %23.s[0]        \n"

  180.                     "fmla       v14.4s, v9.4s, %21.s[3]        \n"  

  181. 

  182.                     "ext        v12.16b, v9.16b, v10.16b, #8   \n" //r41

  183.                     "fmla       v13.4s, v11.4s, %23.s[1]       \n"

  184.                     "fmla       v8.4s,  v11.4s, %22.s[0]       \n"  

  185. 

  186.                     "ext        v11.16b, v9.16b, v10.16b, #12  \n" //r41

  187.                     "fmla       v7.4s,  v12.4s, %23.s[2]       \n"

  188.                     "fmla       v14.4s, v12.4s, %22.s[1]       \n"

  189. 

  190.                     "fmla       v13.4s, v11.4s, %23.s[3]       \n"

  191.                     "fmla       v8.4s,  v11.4s, %22.s[2]       \n"

  192. 

  193.                     "prfm       pldl1keep, [%3, #256]          \n"

  194. 

  195.                     "fmla       v7.4s,  v10.4s, %24.s[0]       \n"

  196.                     "fmla       v14.4s, v10.4s, %22.s[3]       \n"

  197. 

  198.                     // r0 and r5

  199.                     "ld1        {v9.4s, v10.4s}, [%3]          \n"// v9 v10 = r00 r04

  200.                     "add        %3, %3, #16                    \n"

  201. 

  202.                     "ext        v11.16b, v9.16b, v10.16b, #4   \n" //r01

  203.                     "fmla       v13.4s, v11.4s, %18.s[1]       \n"

  204. 

  205.                     "ext        v12.16b, v9.16b, v10.16b, #8   \n" //r02

  206.                     "fmla       v7.4s, v12.4s, %18.s[2]        \n"

  207. 

  208.                     "ext        v11.16b, v9.16b, v10.16b, #12  \n" //r03

  209. 

  210.                     "prfm       pldl1keep, [%8, #256]          \n"

  211. 

  212.                     "fmla       v13.4s, v11.4s, %18.s[3]       \n"

  213. 

  214.                     // r5

  215.                     "ld1        {v11.4s, v12.4s}, [%8]         \n"// v11 v12 = r50 r54

  216.                     "add        %8, %8, #16                    \n"

  217. 

  218.                     "fmla       v8.4s,  v11.4s, %23.s[0]       \n"

  219.                     "fmla       v14.4s, v12.4s, %24.s[0]       \n"

  220. 

  221.                     "fmla       v7.4s,  v9.4s,  %18.s[0]       \n"

  222.                     "fmla       v13.4s, v10.4s, %19.s[0]       \n"

  223. 

  224.                     "ext        v9.16b,  v11.16b, v12.16b, #4  \n" //r51

  225.                     "ext        v10.16b, v11.16b, v12.16b, #8  \n" //r52

  226. 

  227.                     "fmla       v14.4s, v9.4s, %23.s[1]        \n"

  228. 

  229.                     "ext        v9.16b, v11.16b, v12.16b, #12  \n" //r53

  230.                     "fmla       v8.4s, v10.4s, %23.s[2]        \n"

  231. 

  232.                     "fmla       v14.4s, v9.4s, %23.s[3]        \n"

  233. 

  234.                     "fadd       v7.4s, v7.4s, v13.4s           \n"

  235. 

  236.                     "st1        {v7.4s}, [%1], #16             \n"

  237. 

  238.                     "fadd       v8.4s, v8.4s, v14.4s           \n"

  239. 

  240.                     "prfm       pldl1keep, [%1, #128]          \n"

  241.                     "ld1        {v7.4s}, [%1]                  \n"// v7 = out

  242.                     "st1        {v8.4s}, [%2], #16             \n"

  243. 

  244.                     "subs       %w0, %w0, #1                   \n"

  245.                     "bne        0b                             \n"

  246.                     : "=r"(nn),         // %0

  247.                       "=r"(outptr),     // %1

  248.                       "=r"(outptr2),    // %2

  249.                       "=r"(r0),         // %3

  250.                       "=r"(r1),         // %4

  251.                       "=r"(r2),         // %5

  252.                       "=r"(r3),         // %6

  253.                       "=r"(r4),         // %7

  254.                       "=r"(r5)          // %8

  255.                     : "0"(nn),

  256.                       "1"(outptr),

  257.                       "2"(outptr2),

  258.                       "3"(r0),

  259.                       "4"(r1),

  260.                       "5"(r2),

  261.                       "6"(r3),

  262.                       "7"(r4),

  263.                       "8"(r5),

  264.                       "w"(_k0123),      // %18

  265.                       "w"(_k4567),      // %19

  266.                       "w"(_k891011),    // %20

  267.                       "w"(_k12131415),  // %21

  268.                       "w"(_k16171819),  // %22

  269.                       "w"(_k20212223),  // %23

  270.                       "w"(_k24242424)   // %24

  271.                     : "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  272.                 );

  273.                 }

  274. #else

  275.                 if (nn > 0)

  276.                 {

  277.                 asm volatile(

  278. //                     "veor       q13, q13            \n"

  279. //                     "veor       q14, q14            \n"

  280. 

  281.                     "pld        [%1, #128]          \n"

  282. 

  283.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = out

  284. 

  285.                     "0:                             \n"

  286. 

  287.                     // q11 = rx1 / rx3

  288.                     // q12 = rx2

  289. 

  290.                     // q13 q14 = intermediate sum register

  291. 

  292.                     "pld        [%2, #128]          \n"

  293. 

  294.                     "vld1.f32   {d16-d17}, [%2]     \n"// q8 = out2

  295. 

  296. 

  297.                     "pld        [%4, #256]          \n"

  298. 

  299.                     // r1

  300.                     "vld1.f32   {d18-d21}, [%4]     \n"// q9 q10 = r10 r14

  301.                     "add        %4, #16             \n"

  302. 

  303.                     "vext.32    q11, q9, q10, #1    \n"// r11

  304.                     "vmul.f32   q13, q9, %e19[1]    \n"

  305.                     "vmla.f32   q8, q9, %e18[0]     \n"

  306. 

  307.                     "vext.32    q12, q9, q10, #2    \n"// r12

  308.                     "vmla.f32   q7, q11, %f19[0]    \n"

  309.                     "vmul.f32   q14, q11, %e18[1]   \n"

  310. 

  311.                     "vext.32    q11, q9, q10, #3    \n"// r13

  312.                     "vmla.f32   q13, q12, %f19[1]   \n"

  313.                     "vmla.f32   q8, q12, %f18[0]    \n"

  314. 

  315.                     "vmla.f32   q7, q11, %e20[0]    \n"

  316.                     "vmla.f32   q14, q11, %f18[1]   \n"

  317. 

  318.                     "pld        [%5, #256]          \n"

  319. 

  320.                     "vmla.f32   q13, q10, %e20[1]   \n"

  321.                     "vmla.f32   q8, q10, %e19[0]    \n"

  322. 

  323.                     // r2

  324.                     "vld1.f32   {d18-d21}, [%5]     \n"// q9 q10 = r20 r24

  325.                     "add        %5, #16             \n"

  326. 

  327.                     "vext.32    q11, q9, q10, #1    \n"// r21

  328.                     "vmla.f32   q7, q9, %f20[0]     \n"

  329.                     "vmla.f32   q14, q9, %e19[1]    \n"

  330. 

  331.                     "vext.32    q12, q9, q10, #2    \n"// r22

  332.                     "vmla.f32   q13, q11, %f20[1]   \n"

  333.                     "vmla.f32   q8, q11, %f19[0]    \n"

  334. 

  335.                     "vext.32    q11, q9, q10, #3    \n"// r23

  336.                     "vmla.f32   q7, q12, %e21[0]    \n"

  337.                     "vmla.f32   q14, q12, %f19[1]   \n"

  338. 

  339.                     "vmla.f32   q13, q11, %e21[1]   \n"

  340.                     "vmla.f32   q8, q11, %e20[0]    \n"

  341. 

  342.                     "pld        [%6, #256]          \n"

  343. 

  344.                     "vmla.f32   q7, q10, %f21[0]    \n"

  345.                     "vmla.f32   q14, q10, %e20[1]   \n"

  346. 

  347.                     // r3

  348.                     "vld1.f32   {d18-d21}, [%6]     \n"// q9 q10 = r30 r34

  349.                     "add        %6, #16             \n"

  350. 

  351.                     "vext.32    q11, q9, q10, #1    \n"// r31

  352.                     "vmla.f32   q13, q9, %f21[1]    \n"

  353.                     "vmla.f32   q8, q9, %f20[0]     \n"

  354. 

  355.                     "vext.32    q12, q9, q10, #2    \n"// r32

  356.                     "vmla.f32   q7, q11, %e22[0]    \n"

  357.                     "vmla.f32   q14, q11, %f20[1]   \n"

  358. 

  359.                     "vext.32    q11, q9, q10, #3    \n"// r33

  360.                     "vmla.f32   q13, q12, %e22[1]   \n"

  361.                     "vmla.f32   q8, q12, %e21[0]    \n"

  362. 

  363.                     "vmla.f32   q7, q11, %f22[0]    \n"

  364.                     "vmla.f32   q14, q11, %e21[1]   \n"

  365. 

  366.                     "pld        [%7, #256]          \n"

  367. 

  368.                     "vmla.f32   q13, q10, %f22[1]   \n"

  369.                     "vmla.f32   q8, q10, %f21[0]    \n"

  370. 

  371.                     // r4

  372.                     "vld1.f32   {d18-d21}, [%7]     \n"// q9 q10 = r40 r44

  373.                     "add        %7, #16             \n"

  374. 

  375.                     "vext.32    q11, q9, q10, #1    \n"// r41

  376.                     "vmla.f32   q7, q9, %e23[0]     \n"

  377.                     "vmla.f32   q14, q9, %f21[1]    \n"

  378. 

  379.                     "vext.32    q12, q9, q10, #2    \n"// r42

  380.                     "vmla.f32   q13, q11, %e23[1]   \n"

  381.                     "vmla.f32   q8, q11, %e22[0]    \n"

  382. 

  383.                     "vext.32    q11, q9, q10, #3    \n"// r43

  384.                     "vmla.f32   q7, q12, %f23[0]    \n"

  385.                     "vmla.f32   q14, q12, %e22[1]   \n"

  386. 

  387.                     "vmla.f32   q13, q11, %f23[1]   \n"

  388.                     "vmla.f32   q8, q11, %f22[0]    \n"

  389. 

  390.                     "pld        [%3, #256]          \n"

  391. 

  392.                     "vmla.f32   q7, q10, %e24[0]    \n"

  393.                     "vmla.f32   q14, q10, %f22[1]   \n"

  394. 

  395.                     // r0 and r5

  396.                     "vld1.f32   {d18-d21}, [%3]     \n"// q9 q10 = r00 r04

  397.                     "add        %3, #16             \n"

  398. 

  399.                     "vext.32    q11, q9, q10, #1    \n"// r01

  400.                     "vmla.f32   q13, q11, %e18[1]   \n"

  401. 

  402.                     "vext.32    q12, q9, q10, #2    \n"// r02

  403.                     "vmla.f32   q7, q12, %f18[0]    \n"

  404. 

  405.                     "vext.32    q11, q9, q10, #3    \n"// r03

  406. 

  407.                     "pld        [%8, #256]          \n"

  408. 

  409.                     "vmla.f32   q13, q11, %f18[1]   \n"

  410. 

  411.                     // r5

  412.                     "vld1.f32   {d22-d25}, [%8]     \n"// q11 q12 = r50 r54

  413.                     "add        %8, #16             \n"

  414. 

  415.                     "vmla.f32   q8, q11, %e23[0]    \n"

  416.                     "vmla.f32   q14, q12, %e24[0]   \n"

  417. 

  418.                     "vmla.f32   q7, q9, %e18[0]     \n"

  419.                     "vmla.f32   q13, q10, %e19[0]   \n"

  420. 

  421.                     "vext.32    q9, q11, q12, #1    \n"// r51

  422.                     "vext.32    q10, q11, q12, #2   \n"// r52

  423. 

  424.                     "vmla.f32   q14, q9, %e23[1]    \n"

  425. 

  426.                     "vext.32    q9, q11, q12, #3    \n"// r53

  427.                     "vmla.f32   q8, q10, %f23[0]    \n"

  428. 

  429.                     "vmla.f32   q14, q9, %f23[1]    \n"

  430. 

  431.                     "vadd.f32   q7, q7, q13         \n"

  432. 

  433. //                     "veor       q13, q13            \n"

  434. 

  435.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  436. 

  437.                     "vadd.f32   q8, q8, q14         \n"

  438. 

  439.                     "pld        [%1, #128]          \n"

  440. 

  441.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = out

  442. 

  443. //                     "veor       q14, q14            \n"

  444. 

  445.                     "vst1.f32   {d16-d17}, [%2]!    \n"

  446. 

  447.                     "subs       %0, #1              \n"

  448.                     "bne        0b                  \n"

  449.                     : "=r"(nn),         // %0

  450.                       "=r"(outptr),     // %1

  451.                       "=r"(outptr2),    // %2

  452.                       "=r"(r0),         // %3

  453.                       "=r"(r1),         // %4

  454.                       "=r"(r2),         // %5

  455.                       "=r"(r3),         // %6

  456.                       "=r"(r4),         // %7

  457.                       "=r"(r5)          // %8

  458.                     : "0"(nn),

  459.                       "1"(outptr),

  460.                       "2"(outptr2),

  461.                       "3"(r0),

  462.                       "4"(r1),

  463.                       "5"(r2),

  464.                       "6"(r3),

  465.                       "7"(r4),

  466.                       "8"(r5),

  467.                       "w"(_k0123),      // %18

  468.                       "w"(_k4567),      // %19

  469.                       "w"(_k891011),    // %20

  470.                       "w"(_k12131415),  // %21

  471.                       "w"(_k16171819),  // %22

  472.                       "w"(_k20212223),  // %23

  473.                       "w"(_k24242424)   // %24

  474.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  475.                 );

  476.                 }

  477. #endif // __aarch64__

  478. #endif // __ARM_NEON

  479.                 for (; remain>0; remain--)

  480.                 {

  481.                     float sum = 0;

  482.                     float sum2 = 0;

  483. #if __ARM_NEON

  484.                     float32x4_t _r1 = vld1q_f32(r1);

  485.                     float32x4_t _k1 = vld1q_f32(k1);

  486.                     float32x4_t _sum = vmulq_f32(_r1, _k1);

  487.                     float32x4_t _sum2 = vmulq_f32(_r1, _k0123);

  488. 

  489.                     float32x4_t _r2 = vld1q_f32(r2);

  490.                     float32x4_t _k2 = vld1q_f32(k2);

  491.                     _sum = vmlaq_f32(_sum, _r2, _k2);

  492.                     _sum2 = vmlaq_f32(_sum2, _r2, _k1);

  493. 

  494.                     float32x4_t _r3 = vld1q_f32(r3);

  495.                     float32x4_t _k3 = vld1q_f32(k3);

  496.                     _sum = vmlaq_f32(_sum, _r3, _k3);

  497.                     _sum2 = vmlaq_f32(_sum2, _r3, _k2);

  498. 

  499.                     float32x4_t _r4 = vld1q_f32(r4);

  500.                     _sum = vmlaq_f32(_sum, _r4, _k20212223);

  501.                     _sum2 = vmlaq_f32(_sum2, _r4, _k3);

  502. 

  503.                     float32x4_t _r0 = vld1q_f32(r0);

  504.                     _sum = vmlaq_f32(_sum, _r0, _k0123);

  505.                     float32x4_t _r5 = vld1q_f32(r5);

  506.                     _sum2 = vmlaq_f32(_sum2, _r5, _k20212223);

  507. 

  508.                     float32x4_t _k_t4;

  509.                     _k_t4 = vsetq_lane_f32(k0[4], _k_t4, 0);

  510.                     _k_t4 = vsetq_lane_f32(k1[4], _k_t4, 1);

  511.                     _k_t4 = vsetq_lane_f32(k2[4], _k_t4, 2);

  512.                     _k_t4 = vsetq_lane_f32(k3[4], _k_t4, 3);

  513. 

  514.                     float32x4_t _r_t4;

  515. 

  516.                     _r_t4 = vsetq_lane_f32(r0[4], _r_t4, 0);

  517.                     _r_t4 = vsetq_lane_f32(r1[4], _r_t4, 1);

  518.                     _r_t4 = vsetq_lane_f32(r2[4], _r_t4, 2);

  519.                     _r_t4 = vsetq_lane_f32(r3[4], _r_t4, 3);

  520.                     _sum = vmlaq_f32(_sum, _r_t4, _k_t4);

  521. 

  522.                     sum = r4[4] * k4[4];

  523. 

  524.                     _r_t4 = vextq_f32(_r_t4, _r_t4, 1);

  525.                     _r_t4 = vsetq_lane_f32(r4[4], _r_t4, 3);

  526.                     _sum2 = vmlaq_f32(_sum2, _r_t4, _k_t4);

  527. 

  528.                     sum2 = r5[4] * k4[4];

  529. 

  530.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  531.                     float32x2_t _ss2 = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2));

  532.                     float32x2_t _ss_ss2 = vpadd_f32(_ss, _ss2);

  533. 

  534.                     sum += vget_lane_f32(_ss_ss2, 0);

  535.                     sum2 += vget_lane_f32(_ss_ss2, 1);

  536. #else

  537.                     sum += r0[0] * k0[0];

  538.                     sum += r0[1] * k0[1];

  539.                     sum += r0[2] * k0[2];

  540.                     sum += r0[3] * k0[3];

  541.                     sum += r0[4] * k0[4];

  542. 

  543.                     sum += r1[0] * k1[0];

  544.                     sum += r1[1] * k1[1];

  545.                     sum += r1[2] * k1[2];

  546.                     sum += r1[3] * k1[3];

  547.                     sum += r1[4] * k1[4];

  548. 

  549.                     sum += r2[0] * k2[0];

  550.                     sum += r2[1] * k2[1];

  551.                     sum += r2[2] * k2[2];

  552.                     sum += r2[3] * k2[3];

  553.                     sum += r2[4] * k2[4];

  554. 

  555.                     sum += r3[0] * k3[0];

  556.                     sum += r3[1] * k3[1];

  557.                     sum += r3[2] * k3[2];

  558.                     sum += r3[3] * k3[3];

  559.                     sum += r3[4] * k3[4];

  560. 

  561.                     sum += r4[0] * k4[0];

  562.                     sum += r4[1] * k4[1];

  563.                     sum += r4[2] * k4[2];

  564.                     sum += r4[3] * k4[3];

  565.                     sum += r4[4] * k4[4];

  566. 

  567.                     sum2 += r1[0] * k0[0];

  568.                     sum2 += r1[1] * k0[1];

  569.                     sum2 += r1[2] * k0[2];

  570.                     sum2 += r1[3] * k0[3];

  571.                     sum2 += r1[4] * k0[4];

  572. 

  573.                     sum2 += r2[0] * k1[0];

  574.                     sum2 += r2[1] * k1[1];

  575.                     sum2 += r2[2] * k1[2];

  576.                     sum2 += r2[3] * k1[3];

  577.                     sum2 += r2[4] * k1[4];

  578. 

  579.                     sum2 += r3[0] * k2[0];

  580.                     sum2 += r3[1] * k2[1];

  581.                     sum2 += r3[2] * k2[2];

  582.                     sum2 += r3[3] * k2[3];

  583.                     sum2 += r3[4] * k2[4];

  584. 

  585.                     sum2 += r4[0] * k3[0];

  586.                     sum2 += r4[1] * k3[1];

  587.                     sum2 += r4[2] * k3[2];

  588.                     sum2 += r4[3] * k3[3];

  589.                     sum2 += r4[4] * k3[4];

  590. 

  591.                     sum2 += r5[0] * k4[0];

  592.                     sum2 += r5[1] * k4[1];

  593.                     sum2 += r5[2] * k4[2];

  594.                     sum2 += r5[3] * k4[3];

  595.                     sum2 += r5[4] * k4[4];

  596. #endif // __ARM_NEON

  597.                     *outptr += sum;

  598.                     *outptr2 += sum2;

  599. 

  600.                     r0++;

  601.                     r1++;

  602.                     r2++;

  603.                     r3++;

  604.                     r4++;

  605.                     r5++;

  606.                     outptr++;

  607.                     outptr2++;

  608.                 }

  609. 

  610.                 r0 += 4 + w;

  611.                 r1 += 4 + w;

  612.                 r2 += 4 + w;

  613.                 r3 += 4 + w;

  614.                 r4 += 4 + w;

  615.                 r5 += 4 + w;

  616. 

  617.                 outptr += outw;

  618.                 outptr2 += outw;

  619.             }

  620. 

  621.             for (; i < outh; i++)

  622.             {

  623. 

  624. #if __ARM_NEON

  625.                 int nn = outw >> 2;

  626.                 int remain = outw - (nn << 2);

  627. #else

  628.                 int remain = outw;

  629. #endif // __ARM_NEON

  630. 

  631. #if __ARM_NEON

  632. #if __aarch64__

  633.                 if (nn > 0)

  634.                 {

  635.                 asm volatile(

  636.                     "prfm       pldl1keep, [%1, #128]          \n"

  637.                     "prfm       pldl1keep, [%2, #256]          \n"

  638. 

  639.                     "ld1        {v8.4s, v9.4s}, [%2]           \n"// _r00 = vld1q_f32(r0+j);

  640.                     "add        %2, %2, #16                    \n"

  641. 

  642.                     "0:                                        \n"

  643. 

  644.                     "ld1        {v7.4s}, [%1]                  \n"// _sum = vld1q_f32(outptr+j);

  645. 

  646.                     "ext        v10.16b, v8.16b, v9.16b, #4    \n" //_r01

  647.                     "ext        v11.16b, v8.16b, v9.16b, #8    \n" //_r02

  648.                     "ext        v12.16b, v8.16b, v9.16b, #12   \n" //_r03

  649. 

  650.                     "fmla       v7.4s,   v8.4s, %14.s[0]       \n"

  651.                     "fmul       v13.4s, v10.4s, %14.s[1]       \n"

  652. 

  653.                     "prfm       pldl1keep, [%3, #256]          \n"

  654. 

  655.                     "fmul       v14.4s, v11.4s, %14.s[2]       \n"

  656.                     "fmul       v15.4s, v12.4s, %14.s[3]       \n"

  657.                     "fmla       v7.4s,   v9.4s, %15.s[0]       \n"

  658. 

  659.                     "ld1        {v8.4s, v9.4s}, [%3]           \n"

  660.                     "add        %3, %3, #16                    \n"

  661.                     "ext        v10.16b, v8.16b, v9.16b, #4    \n" //_r11

  662.                     "ext        v11.16b, v8.16b, v9.16b, #8    \n" //_r12

  663.                     "ext        v12.16b, v8.16b, v9.16b, #12   \n" //_r13

  664. 

  665.                     "fmla       v7.4s,   v8.4s, %15.s[1]       \n"

  666.                     "fmla       v13.4s, v10.4s, %15.s[2]       \n"

  667. 

  668.                     "prfm       pldl1keep, [%4, #256]          \n"

  669. 

  670.                     "fmla       v14.4s, v11.4s, %15.s[3]       \n"

  671.                     "fmla       v15.4s, v12.4s, %16.s[0]       \n"

  672.                     "fmla       v7.4s,   v9.4s, %16.s[1]       \n"

  673. 

  674.                     "ld1        {v8.4s, v9.4s}, [%4]           \n"

  675.                     "add        %4, %4, #16                    \n"

  676.                     "ext        v10.16b, v8.16b, v9.16b, #4    \n" //_r21

  677.                     "ext        v11.16b, v8.16b, v9.16b, #8    \n" //_r22

  678.                     "ext        v12.16b, v8.16b, v9.16b, #12   \n" //_r23

  679. 

  680.                     "fmla       v7.4s,   v8.4s, %16.s[2]       \n"

  681.                     "fmla       v13.4s, v10.4s, %16.s[3]       \n"

  682. 

  683.                     "prfm       pldl1keep, [%5, #256]          \n"

  684. 

  685.                     "fmla       v14.4s, v11.4s, %17.s[0]       \n"

  686.                     "fmla       v15.4s, v12.4s, %17.s[1]       \n"

  687.                     "fmla       v7.4s,   v9.4s, %17.s[2]       \n"

  688. 

  689.                     "ld1        {v8.4s, v9.4s}, [%5]           \n"

  690.                     "add        %5, %5, #16                    \n"

  691.                     "ext        v10.16b, v8.16b, v9.16b, #4    \n" //_r31

  692.                     "ext        v11.16b, v8.16b, v9.16b, #8    \n" //_r32

  693.                     "ext        v12.16b, v8.16b, v9.16b, #12   \n" //_r33

  694. 

  695.                     "fmla       v7.4s,   v8.4s, %17.s[3]       \n"

  696.                     "fmla       v13.4s, v10.4s, %18.s[0]       \n"

  697. 

  698.                     "prfm       pldl1keep, [%6, #256]          \n"

  699. 

  700.                     "fmla       v14.4s, v11.4s, %18.s[1]       \n"

  701.                     "fmla       v15.4s, v12.4s, %18.s[2]       \n"

  702.                     "fmla       v7.4s,   v9.4s, %18.s[3]       \n"

  703. 

  704.                     "ld1        {v8.4s, v9.4s}, [%6]           \n"

  705.                     "add        %6, %6, #16                    \n"

  706.                     "ext        v10.16b, v8.16b, v9.16b, #4    \n" //_r41

  707.                     "ext        v11.16b, v8.16b, v9.16b, #8    \n" //_r42

  708.                     "ext        v12.16b, v8.16b, v9.16b, #12   \n" //_r43

  709. 

  710.                     "fmla       v7.4s,   v8.4s, %19.s[0]       \n"

  711.                     "fmla       v13.4s, v10.4s, %19.s[1]       \n"

  712.                     "fmla       v14.4s, v11.4s, %19.s[2]       \n"

  713.                     "fmla       v15.4s, v12.4s, %19.s[3]       \n"

  714.                     "fmla       v7.4s,   v9.4s, %20.s[0]       \n"

  715. 

  716.                     "fadd       v14.4s, v14.4s, v15.4s         \n"

  717.                     "fadd       v7.4s,   v7.4s, v13.4s         \n"                    

  718. 

  719.                     "prfm       pldl1keep, [%2, #256]          \n"

  720. 

  721.                     "fadd       v7.4s,   v7.4s, v14.4s         \n"     

  722. 

  723.                     "ld1        {v8.4s, v9.4s}, [%2]           \n"

  724.                     "add        %2, %2, #16                    \n"

  725. 

  726.                     "st1        {v7.4s}, [%1], #16             \n"

  727. 

  728.                     "prfm       pldl1keep, [%1, #128]          \n"

  729. 

  730.                     "subs       %w0, %w0, #1                   \n"

  731.                     "bne        0b                             \n"

  732. 

  733.                     "sub        %2, %2, #16                    \n"

  734.                     : "=r"(nn),         // %0

  735.                       "=r"(outptr),     // %1

  736.                       "=r"(r0),         // %2

  737.                       "=r"(r1),         // %3

  738.                       "=r"(r2),         // %4

  739.                       "=r"(r3),         // %5

  740.                       "=r"(r4)          // %6

  741.                     : "0"(nn),

  742.                       "1"(outptr),

  743.                       "2"(r0),

  744.                       "3"(r1),

  745.                       "4"(r2),

  746.                       "5"(r3),

  747.                       "6"(r4),

  748.                       "w"(_k0123),      // %14

  749.                       "w"(_k4567),      // %15

  750.                       "w"(_k891011),    // %16

  751.                       "w"(_k12131415),  // %17

  752.                       "w"(_k16171819),  // %18

  753.                       "w"(_k20212223),  // %19

  754.                       "w"(_k24242424)   // %20

  755.                     : "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  756.                 );

  757.                 }

  758. #else

  759.                 if (nn > 0)

  760.                 {

  761.                 asm volatile(

  762. //                     "veor       q15, q15            \n"// _sum3 = 0;

  763. 

  764.                     "pld        [%1, #128]          \n"

  765. 

  766.                     "pld        [%2, #256]          \n"

  767. 

  768.                     "vld1.f32   {d16-d19}, [%2]     \n"// _r00 = vld1q_f32(r0+j);

  769.                     "add        %2, #16             \n"

  770. 

  771.                     "0:                             \n"

  772. 

  773.                     "vld1.f32   {d14-d15}, [%1]     \n"// _sum = vld1q_f32(outptr+j);

  774. //                     "veor       q13, q13            \n"// _sum2 = 0;

  775. //                     "veor       q14, q14            \n"// _sum3 = 0;

  776. 

  777.                     "vext.32    q10, q8, q9, #1     \n"// _r01

  778.                     "vext.32    q11, q8, q9, #2     \n"// _r02

  779.                     "vext.32    q12, q8, q9, #3     \n"// _r03

  780. 

  781.                     "vmla.f32   q7, q8, %e14[0]     \n"

  782.                     "vmul.f32   q13, q10, %e14[1]   \n"

  783. 

  784.                     "pld        [%3, #256]          \n"

  785. 

  786.                     "vmul.f32   q14, q11, %f14[0]   \n"

  787.                     "vmul.f32   q15, q12, %f14[1]   \n"

  788.                     "vmla.f32   q7, q9, %e15[0]     \n"

  789. 

  790.                     "vld1.f32   {d16-d19}, [%3]     \n"

  791.                     "add        %3, #16             \n"

  792.                     "vext.32    q10, q8, q9, #1     \n"

  793.                     "vext.32    q11, q8, q9, #2     \n"

  794.                     "vext.32    q12, q8, q9, #3     \n"

  795. 

  796.                     "vmla.f32   q7, q8, %e15[1]     \n"

  797.                     "vmla.f32   q13, q10, %f15[0]   \n"

  798. 

  799.                     "pld        [%4, #256]          \n"

  800. 

  801.                     "vmla.f32   q14, q11, %f15[1]   \n"

  802.                     "vmla.f32   q15, q12, %e16[0]   \n"

  803.                     "vmla.f32   q7, q9, %e16[1]     \n"

  804. 

  805.                     "vld1.f32   {d16-d19}, [%4]     \n"

  806.                     "add        %4, #16             \n"

  807.                     "vext.32    q10, q8, q9, #1     \n"

  808.                     "vext.32    q11, q8, q9, #2     \n"

  809.                     "vext.32    q12, q8, q9, #3     \n"

  810. 

  811.                     "vmla.f32   q7, q8, %f16[0]     \n"

  812.                     "vmla.f32   q13, q10, %f16[1]   \n"

  813. 

  814.                     "pld        [%5, #256]          \n"

  815. 

  816.                     "vmla.f32   q14, q11, %e17[0]   \n"

  817.                     "vmla.f32   q15, q12, %e17[1]   \n"

  818.                     "vmla.f32   q7, q9, %f17[0]     \n"

  819. 

  820.                     "vld1.f32   {d16-d19}, [%5]     \n"

  821.                     "add        %5, #16             \n"

  822.                     "vext.32    q10, q8, q9, #1     \n"

  823.                     "vext.32    q11, q8, q9, #2     \n"

  824.                     "vext.32    q12, q8, q9, #3     \n"

  825. 

  826.                     "vmla.f32   q7, q8, %f17[1]     \n"

  827.                     "vmla.f32   q13, q10, %e18[0]   \n"

  828. 

  829.                     "pld        [%6, #256]          \n"

  830. 

  831.                     "vmla.f32   q14, q11, %e18[1]   \n"

  832.                     "vmla.f32   q15, q12, %f18[0]   \n"

  833.                     "vmla.f32   q7, q9, %f18[1]     \n"

  834. 

  835.                     "vld1.f32   {d16-d19}, [%6]     \n"

  836.                     "add        %6, #16             \n"

  837.                     "vext.32    q10, q8, q9, #1     \n"

  838.                     "vext.32    q11, q8, q9, #2     \n"

  839.                     "vext.32    q12, q8, q9, #3     \n"

  840. 

  841.                     "vmla.f32   q7, q8, %e19[0]     \n"

  842.                     "vmla.f32   q13, q10, %e19[1]   \n"

  843.                     "vmla.f32   q14, q11, %f19[0]   \n"

  844.                     "vmla.f32   q15, q12, %f19[1]   \n"

  845.                     "vmla.f32   q7, q9, %e20[0]     \n"

  846. 

  847.                     "vadd.f32   q14, q14, q15       \n"

  848.                     "vadd.f32   q7, q7, q13         \n"

  849. //                     "veor       q15, q15            \n"// _sum3 = 0;

  850. 

  851.                     "pld        [%2, #256]          \n"

  852. 

  853.                     "vadd.f32   q7, q7, q14         \n"

  854. 

  855.                     "vld1.f32   {d16-d19}, [%2]     \n"// _r00 = vld1q_f32(r0+j);

  856.                     "add        %2, #16             \n"

  857. 

  858.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  859. 

  860.                     "pld        [%1, #128]          \n"

  861. 

  862.                     "subs       %0, #1              \n"

  863.                     "bne        0b                  \n"

  864. 

  865.                     "sub        %2, #16             \n"

  866.                     : "=r"(nn),         // %0

  867.                       "=r"(outptr),     // %1

  868.                       "=r"(r0),         // %2

  869.                       "=r"(r1),         // %3

  870.                       "=r"(r2),         // %4

  871.                       "=r"(r3),         // %5

  872.                       "=r"(r4)          // %6

  873.                     : "0"(nn),

  874.                       "1"(outptr),

  875.                       "2"(r0),

  876.                       "3"(r1),

  877.                       "4"(r2),

  878.                       "5"(r3),

  879.                       "6"(r4),

  880.                       "w"(_k0123),      // %14

  881.                       "w"(_k4567),      // %15

  882.                       "w"(_k891011),    // %16

  883.                       "w"(_k12131415),  // %17

  884.                       "w"(_k16171819),  // %18

  885.                       "w"(_k20212223),  // %19

  886.                       "w"(_k24242424)   // %20

  887.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  888.                 );

  889.                 }

  890. #endif // __aarch64__

  891. #endif // __ARM_NEON

  892.                 for (; remain>0; remain--)

  893.                 {

  894.                     float sum = 0;

  895. #if __ARM_NEON

  896.                     float32x4_t _r0 = vld1q_f32(r0);

  897.                     float32x4_t _sum = vmulq_f32(_r0, _k0123);

  898. 

  899.                     float32x4_t _r1 = vld1q_f32(r1);

  900.                     _sum = vmlaq_f32(_sum, _r1, vld1q_f32(k1));

  901. 

  902.                     float32x4_t _r2 = vld1q_f32(r2);

  903.                     _sum = vmlaq_f32(_sum, _r2, vld1q_f32(k2));

  904. 

  905.                     float32x4_t _r3 = vld1q_f32(r3);

  906.                     _sum = vmlaq_f32(_sum, _r3, vld1q_f32(k3));

  907. 

  908.                     float32x4_t _r4 = vld1q_f32(r4);

  909.                     _sum = vmlaq_f32(_sum, _r4, _k20212223);

  910. 

  911.                     float32x4_t _k_t4;

  912.                     _k_t4 = vsetq_lane_f32(k0[4], _k_t4, 0);

  913.                     _k_t4 = vsetq_lane_f32(k1[4], _k_t4, 1);

  914.                     _k_t4 = vsetq_lane_f32(k2[4], _k_t4, 2);

  915.                     _k_t4 = vsetq_lane_f32(k3[4], _k_t4, 3);

  916. 

  917.                     float32x4_t _r_t4;

  918. 

  919.                     _r_t4 = vsetq_lane_f32(r0[4], _r_t4, 0);

  920.                     _r_t4 = vsetq_lane_f32(r1[4], _r_t4, 1);

  921.                     _r_t4 = vsetq_lane_f32(r2[4], _r_t4, 2);

  922.                     _r_t4 = vsetq_lane_f32(r3[4], _r_t4, 3);

  923.                     _sum = vmlaq_f32(_sum, _r_t4, _k_t4);

  924. 

  925.                     sum = r4[4] * k4[4];

  926. 

  927.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  928.                     _ss = vpadd_f32(_ss, _ss);

  929. 

  930.                     sum += vget_lane_f32(_ss, 0);

  931. #else

  932.                     sum += r0[0] * k0[0];

  933.                     sum += r0[1] * k0[1];

  934.                     sum += r0[2] * k0[2];

  935.                     sum += r0[3] * k0[3];

  936.                     sum += r0[4] * k0[4];

  937. 

  938.                     sum += r1[0] * k1[0];

  939.                     sum += r1[1] * k1[1];

  940.                     sum += r1[2] * k1[2];

  941.                     sum += r1[3] * k1[3];

  942.                     sum += r1[4] * k1[4];

  943. 

  944.                     sum += r2[0] * k2[0];

  945.                     sum += r2[1] * k2[1];

  946.                     sum += r2[2] * k2[2];

  947.                     sum += r2[3] * k2[3];

  948.                     sum += r2[4] * k2[4];

  949. 

  950.                     sum += r3[0] * k3[0];

  951.                     sum += r3[1] * k3[1];

  952.                     sum += r3[2] * k3[2];

  953.                     sum += r3[3] * k3[3];

  954.                     sum += r3[4] * k3[4];

  955. 

  956.                     sum += r4[0] * k4[0];

  957.                     sum += r4[1] * k4[1];

  958.                     sum += r4[2] * k4[2];

  959.                     sum += r4[3] * k4[3];

  960.                     sum += r4[4] * k4[4];

  961. #endif

  962.                     *outptr += sum;

  963. 

  964.                     r0++;

  965.                     r1++;

  966.                     r2++;

  967.                     r3++;

  968.                     r4++;

  969.                     outptr++;

  970.                 }

  971. 

  972.                 r0 += 4;

  973.                 r1 += 4;

  974.                 r2 += 4;

  975.                 r3 += 4;

  976.                 r4 += 4;

  977. 

  978.             }

  979. 

  980.         }

  981.     }

  982. 

  983. }

  984. 

  985. static void conv5x5s2_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  986. {

  987.     int w = bottom_blob.w;

  988.     int inch = bottom_blob.c;

  989. 

  990.     int outw = top_blob.w;

  991.     int outh = top_blob.h;

  992.     int outch = top_blob.c;

  993. 

  994.     const int tailstep = w - 2*outw + w;

  995. 

  996.     const float* kernel = _kernel;

  997.     const float* bias = _bias;

  998. 

  999.     #pragma omp parallel for num_threads(opt.num_threads)

  1000.     for (int p=0; p<outch; p++)

  1001.     {

  1002.         Mat out = top_blob.channel(p);

  1003. 

  1004.         const float bias0 = bias ? bias[p] : 0.f;

  1005. 

  1006.         out.fill(bias0);

  1007. 

  1008.         for (int q=0; q<inch; q++)

  1009.         {

  1010.             float* outptr = out;

  1011. 

  1012.             const float* img0 = bottom_blob.channel(q);

  1013. 

  1014.             const float* kernel0 = kernel + p*inch*25  + q*25;

  1015. 

  1016.             const float* r0 = img0;

  1017.             const float* r1 = img0 + w;

  1018.             const float* r2 = img0 + w*2;

  1019.             const float* r3 = img0 + w*3;

  1020.             const float* r4 = img0 + w*4;

  1021. 

  1022.             const float* k0 = kernel0;

  1023.             const float* k1 = kernel0 + 5;

  1024.             const float* k2 = kernel0 + 10;

  1025.             const float* k3 = kernel0 + 15;

  1026.             const float* k4 = kernel0 + 20;

  1027. 

  1028. #if __ARM_NEON

  1029.             float32x4_t _k0123 = vld1q_f32(kernel0);

  1030.             float32x4_t _k4567 = vld1q_f32(kernel0+4);

  1031.             float32x4_t _k891011 = vld1q_f32(kernel0+8);

  1032.             float32x4_t _k12131415 = vld1q_f32(kernel0+12);

  1033.             float32x4_t _k16171819 = vld1q_f32(kernel0+16);

  1034.             float32x4_t _k20212223 = vld1q_f32(kernel0+20);

  1035.             float32x4_t _k24242424 = vdupq_n_f32(kernel0[24]);

  1036. #endif // __ARM_NEON

  1037. 

  1038.             for (int i = 0; i < outh; i++)

  1039.             {

  1040. 

  1041. #if __ARM_NEON

  1042.                 int nn = outw >> 2;

  1043.                 int remain = outw - (nn << 2);

  1044. #else

  1045.                 int remain = outw;

  1046. #endif // __ARM_NEON

  1047. 

  1048. #if __ARM_NEON

  1049. #if __aarch64__

  1050.                 if (nn > 0)

  1051.                 {

  1052.                 asm volatile(

  1053.                     "prfm       pldl1keep, [%2, #256]          \n"

  1054.                     "ld2        {v8.4s, v9.4s}, [%2], #32      \n"// v8  = 0  2  4  6   q9  = 1  3  5  7

  1055. 

  1056.                     "prfm       pldl1keep, [%2, #256]          \n"

  1057.                     "ld2        {v10.4s, v11.4s}, [%2]         \n"// v10 = 8 10 12 14   v11 = 9 11 13 15

  1058. 

  1059.                     "prfm       pldl1keep, [%1, #128]          \n"

  1060.                     "0:                                        \n"

  1061. 

  1062.                     "ld1        {v7.4s}, [%1]                  \n" // v7 = outptr

  1063. 

  1064.                     "ext        v12.16b, v8.16b, v10.16b, #4   \n" // v12 = 2 4 6 8

  1065.                     "ext        v11.16b, v9.16b, v11.16b, #4   \n" // v11 = 3 5 7 9

  1066.                     "ext        v10.16b, v8.16b, v10.16b, #8   \n" // v10 = 4 6 8 10

  1067. 

  1068.                     "fmla       v7.4s,  v8.4s, %14.s[0]        \n"

  1069.                     "fmul       v13.4s, v9.4s, %14.s[1]        \n"

  1070. 

  1071.                     "prfm       pldl1keep, [%3, #256]          \n"

  1072. 

  1073.                     "fmul       v14.4s, v12.4s, %14.s[2]       \n"

  1074.                     "fmul       v15.4s, v11.4s, %14.s[3]       \n"

  1075.                     "fmla       v7.4s,  v10.4s, %15.s[0]       \n"

  1076. 

  1077.                     "ld2        {v8.4s, v9.4s}, [%3], #32      \n"

  1078. 

  1079.                     "prfm       pldl1keep, [%3, #256]          \n"

  1080. 

  1081.                     "ld2        {v10.4s, v11.4s}, [%3]         \n"

  1082.                     "ext        v12.16b, v8.16b, v10.16b, #4   \n"

  1083.                     "ext        v11.16b, v9.16b, v11.16b, #4   \n"

  1084.                     "ext        v10.16b, v8.16b, v10.16b, #8   \n"

  1085. 

  1086.                     "fmla       v7.4s,  v8.4s, %15.s[1]        \n"

  1087.                     "fmla       v13.4s, v9.4s, %15.s[2]        \n"

  1088. 

  1089.                     "prfm       pldl1keep, [%4, #256]          \n"

  1090. 

  1091.                     "fmla       v14.4s, v12.4s, %15.s[3]       \n"

  1092.                     "fmla       v15.4s, v11.4s, %16.s[0]       \n"

  1093.                     "fmla       v7.4s,  v10.4s, %16.s[1]       \n"

  1094. 

  1095.                     "ld2        {v8.4s, v9.4s}, [%4], #32      \n"

  1096. 

  1097.                     "prfm       pldl1keep, [%4, #256]          \n"

  1098. 

  1099.                     "ld2        {v10.4s, v11.4s}, [%4]         \n"

  1100.                     "ext        v12.16b, v8.16b, v10.16b, #4   \n"

  1101.                     "ext        v11.16b, v9.16b, v11.16b, #4   \n"

  1102.                     "ext        v10.16b, v8.16b, v10.16b, #8   \n"

  1103. 

  1104.                     "fmla       v7.4s,  v8.4s, %16.s[2]        \n"

  1105.                     "fmla       v13.4s, v9.4s, %16.s[3]        \n"

  1106. 

  1107.                     "prfm       pldl1keep, [%5, #256]          \n"

  1108. 

  1109.                     "fmla       v14.4s, v12.4s, %17.s[0]       \n"

  1110.                     "fmla       v15.4s, v11.4s, %17.s[1]       \n"

  1111.                     "fmla       v7.4s,  v10.4s, %17.s[2]       \n"

  1112. 

  1113.                     "ld2        {v8.4s, v9.4s}, [%5], #32      \n"

  1114. 

  1115.                     "prfm       pldl1keep, [%5, #256]          \n"

  1116. 

  1117.                     "ld2        {v10.4s, v11.4s}, [%5]         \n"

  1118.                     "ext        v12.16b, v8.16b, v10.16b, #4   \n"

  1119.                     "ext        v11.16b, v9.16b, v11.16b, #4   \n"

  1120.                     "ext        v10.16b, v8.16b, v10.16b, #8   \n"

  1121. 

  1122.                     "fmla       v7.4s,  v8.4s, %17.s[3]        \n"

  1123.                     "fmla       v13.4s, v9.4s, %18.s[0]        \n"

  1124. 

  1125.                     "prfm       pldl1keep, [%6, #256]          \n"

  1126. 

  1127.                     "fmla       v14.4s, v12.4s, %18.s[1]       \n"

  1128.                     "fmla       v15.4s, v11.4s, %18.s[2]       \n"

  1129.                     "fmla       v7.4s,  v10.4s, %18.s[3]       \n"

  1130. 

  1131.                     "ld2        {v8.4s, v9.4s}, [%6], #32      \n"

  1132. 

  1133.                     "prfm       pldl1keep, [%6, #256]          \n"

  1134. 

  1135.                     "ld2        {v10.4s, v11.4s}, [%6]         \n"

  1136.                     "ext        v12.16b, v8.16b, v10.16b, #4   \n"

  1137.                     "ext        v11.16b, v9.16b, v11.16b, #4   \n"

  1138.                     "ext        v10.16b, v8.16b, v10.16b, #8   \n"

  1139. 

  1140.                     "fmla       v7.4s,   v8.4s, %19.s[0]       \n"

  1141.                     "fmla       v13.4s,  v9.4s, %19.s[1]       \n"

  1142.                     "fmla       v14.4s, v12.4s, %19.s[2]       \n"

  1143.                     "fmla       v15.4s, v11.4s, %19.s[3]       \n"

  1144.                     "fmla       v7.4s,  v10.4s, %20.s[0]       \n"                    

  1145. 

  1146.                     "prfm       pldl1keep, [%2, #256]          \n"

  1147. 

  1148.                     "ld2        {v8.4s, v9.4s}, [%2], #32      \n"

  1149. 

  1150.                     "fadd       v14.4s, v14.4s, v15.4s         \n"

  1151.                     "fadd       v7.4s,   v7.4s, v13.4s         \n"

  1152.                     

  1153.                     "prfm       pldl1keep, [%2, #256]          \n"

  1154. 

  1155.                     "fadd       v7.4s, v7.4s, v14.4s           \n"   

  1156. 

  1157.                     "ld2        {v10.4s, v11.4s}, [%2]         \n"

  1158.                     "st1        {v7.4s}, [%1], #16             \n"

  1159. 

  1160.                     "prfm       pldl1keep, [%1, #128]          \n"

  1161. 

  1162.                     "subs       %w0, %w0, #1                   \n"

  1163.                     "bne        0b                             \n"

  1164. 

  1165.                     "sub        %2, %2, #32                    \n"

  1166.                     : "=r"(nn),         // %0

  1167.                       "=r"(outptr),     // %1

  1168.                       "=r"(r0),         // %2

  1169.                       "=r"(r1),         // %3

  1170.                       "=r"(r2),         // %4

  1171.                       "=r"(r3),         // %5

  1172.                       "=r"(r4)          // %6

  1173.                     : "0"(nn),

  1174.                       "1"(outptr),

  1175.                       "2"(r0),

  1176.                       "3"(r1),

  1177.                       "4"(r2),

  1178.                       "5"(r3),

  1179.                       "6"(r4),

  1180.                       "w"(_k0123),      // %14

  1181.                       "w"(_k4567),      // %15

  1182.                       "w"(_k891011),    // %16

  1183.                       "w"(_k12131415),  // %17

  1184.                       "w"(_k16171819),  // %18

  1185.                       "w"(_k20212223),  // %19

  1186.                       "w"(_k24242424)   // %20

  1187.                     : "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  1188.                 );

  1189.                 }

  1190. 

  1191. #else

  1192.                 if (nn > 0)

  1193.                 {

  1194.                 asm volatile(

  1195. //                     "veor       q15, q15            \n"// _sump3 = 0;

  1196. //                     "veor       q13, q13            \n"// _sump2 = 0;

  1197. //                     "veor       q14, q14            \n"// _sump3 = 0;

  1198. 

  1199.                     "pld        [%2, #256]          \n"

  1200.                     "vld2.f32   {d16-d19}, [%2]!    \n"// q8  = 0  2  4  6   q9  = 1  3  5  7

  1201. 

  1202.                     "pld        [%2, #256]          \n"

  1203.                     "vld2.f32   {d20-d23}, [%2]     \n"// q10 = 8 10 12 14   q11 = 9 11 13 15

  1204. 

  1205.                     "pld        [%1, #128]          \n"

  1206.                     "0:                             \n"

  1207. 

  1208.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = outptr

  1209. 

  1210.                     "vext.32    q12, q8, q10, #1    \n"// q12 = 2 4 6 8

  1211.                     "vext.32    q11, q9, q11, #1    \n"// q11 = 3 5 7 9

  1212.                     "vext.32    q10, q8, q10, #2    \n"// q10 = 4 6 8 10

  1213. 

  1214.                     "vmla.f32   q7, q8, %e14[0]     \n"

  1215.                     "vmul.f32   q13, q9, %e14[1]    \n"

  1216. 

  1217.                     "pld        [%3, #256]          \n"

  1218. 

  1219.                     "vmul.f32   q14, q12, %f14[0]   \n"

  1220.                     "vmul.f32   q15, q11, %f14[1]   \n"

  1221.                     "vmla.f32   q7, q10, %e15[0]    \n"

  1222. 

  1223.                     "vld2.f32   {d16-d19}, [%3]!    \n"

  1224. 

  1225.                     "pld        [%3, #256]          \n"

  1226. 

  1227.                     "vld2.f32   {d20-d23}, [%3]     \n"

  1228.                     "vext.32    q12, q8, q10, #1    \n"

  1229.                     "vext.32    q11, q9, q11, #1    \n"

  1230.                     "vext.32    q10, q8, q10, #2    \n"

  1231. 

  1232.                     "vmla.f32   q7, q8, %e15[1]     \n"

  1233.                     "vmla.f32   q13, q9, %f15[0]    \n"

  1234. 

  1235.                     "pld        [%4, #256]          \n"

  1236. 

  1237.                     "vmla.f32   q14, q12, %f15[1]   \n"

  1238.                     "vmla.f32   q15, q11, %e16[0]   \n"

  1239.                     "vmla.f32   q7, q10, %e16[1]    \n"

  1240. 

  1241.                     "vld2.f32   {d16-d19}, [%4]!    \n"

  1242. 

  1243.                     "pld        [%4, #256]          \n"

  1244. 

  1245.                     "vld2.f32   {d20-d23}, [%4]     \n"

  1246.                     "vext.32    q12, q8, q10, #1    \n"

  1247.                     "vext.32    q11, q9, q11, #1    \n"

  1248.                     "vext.32    q10, q8, q10, #2    \n"

  1249. 

  1250.                     "vmla.f32   q7, q8, %f16[0]     \n"

  1251.                     "vmla.f32   q13, q9, %f16[1]    \n"

  1252. 

  1253.                     "pld        [%5, #256]          \n"

  1254. 

  1255.                     "vmla.f32   q14, q12, %e17[0]   \n"

  1256.                     "vmla.f32   q15, q11, %e17[1]   \n"

  1257.                     "vmla.f32   q7, q10, %f17[0]    \n"

  1258. 

  1259.                     "vld2.f32   {d16-d19}, [%5]!    \n"

  1260. 

  1261.                     "pld        [%5, #256]          \n"

  1262. 

  1263.                     "vld2.f32   {d20-d23}, [%5]     \n"

  1264.                     "vext.32    q12, q8, q10, #1    \n"

  1265.                     "vext.32    q11, q9, q11, #1    \n"

  1266.                     "vext.32    q10, q8, q10, #2    \n"

  1267. 

  1268.                     "vmla.f32   q7, q8, %f17[1]     \n"

  1269.                     "vmla.f32   q13, q9, %e18[0]    \n"

  1270. 

  1271.                     "pld        [%6, #256]          \n"

  1272. 

  1273.                     "vmla.f32   q14, q12, %e18[1]   \n"

  1274.                     "vmla.f32   q15, q11, %f18[0]   \n"

  1275.                     "vmla.f32   q7, q10, %f18[1]    \n"

  1276. 

  1277.                     "vld2.f32   {d16-d19}, [%6]!    \n"

  1278. 

  1279.                     "pld        [%6, #256]          \n"

  1280. 

  1281.                     "vld2.f32   {d20-d23}, [%6]     \n"

  1282.                     "vext.32    q12, q8, q10, #1    \n"

  1283.                     "vext.32    q11, q9, q11, #1    \n"

  1284.                     "vext.32    q10, q8, q10, #2    \n"

  1285. 

  1286.                     "vmla.f32   q7, q8, %e19[0]     \n"

  1287.                     "vmla.f32   q13, q9, %e19[1]    \n"

  1288.                     "vmla.f32   q14, q12, %f19[0]   \n"

  1289.                     "vmla.f32   q15, q11, %f19[1]   \n"

  1290.                     "vmla.f32   q7, q10, %e20[0]    \n"

  1291. 

  1292.                     "pld        [%2, #256]          \n"

  1293. 

  1294.                     "vld2.f32   {d16-d19}, [%2]!    \n"// q8  = 0  2  4  6   q9  = 1  3  5  7

  1295. 

  1296.                     "vadd.f32   q14, q14, q15       \n"

  1297.                     "vadd.f32   q7, q7, q13         \n"

  1298. //                     "veor       q15, q15            \n"// _sump3 = 0;

  1299. //                     "veor       q13, q13            \n"// _sump2 = 0;

  1300. 

  1301.                     "pld        [%2, #256]          \n"

  1302. 

  1303.                     "vadd.f32   q7, q7, q14         \n"

  1304. 

  1305.                     "vld2.f32   {d20-d23}, [%2]     \n"// q10 = 8 10 12 14   q11 = 9 11 13 15

  1306. 

  1307. //                     "veor       q14, q14            \n"// _sump3 = 0;

  1308. 

  1309.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  1310. 

  1311.                     "pld        [%1, #128]          \n"

  1312. 

  1313.                     "subs       %0, #1              \n"

  1314.                     "bne        0b                  \n"

  1315. 

  1316.                     "sub        %2, #32             \n"

  1317.                     : "=r"(nn),         // %0

  1318.                       "=r"(outptr),     // %1

  1319.                       "=r"(r0),         // %2

  1320.                       "=r"(r1),         // %3

  1321.                       "=r"(r2),         // %4

  1322.                       "=r"(r3),         // %5

  1323.                       "=r"(r4)          // %6

  1324.                     : "0"(nn),

  1325.                       "1"(outptr),

  1326.                       "2"(r0),

  1327.                       "3"(r1),

  1328.                       "4"(r2),

  1329.                       "5"(r3),

  1330.                       "6"(r4),

  1331.                       "w"(_k0123),      // %14

  1332.                       "w"(_k4567),      // %15

  1333.                       "w"(_k891011),    // %16

  1334.                       "w"(_k12131415),  // %17

  1335.                       "w"(_k16171819),  // %18

  1336.                       "w"(_k20212223),  // %19

  1337.                       "w"(_k24242424)   // %20

  1338.                     : "cc", "memory", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  1339.                 );

  1340.                 }

  1341. #endif // __aarch64__

  1342. #endif // __ARM_NEON

  1343.                 for (; remain>0; remain--)

  1344.                 {

  1345.                     float sum = 0;

  1346. #if __ARM_NEON

  1347.                     float32x4_t _r0 = vld1q_f32(r0);

  1348.                     float32x4_t _sum = vmulq_f32(_r0, _k0123);

  1349. 

  1350.                     float32x4_t _r1 = vld1q_f32(r1);

  1351.                     _sum = vmlaq_f32(_sum, _r1, vld1q_f32(k1));

  1352. 

  1353.                     float32x4_t _r2 = vld1q_f32(r2);

  1354.                     _sum = vmlaq_f32(_sum, _r2, vld1q_f32(k2));

  1355. 

  1356.                     float32x4_t _r3 = vld1q_f32(r3);

  1357.                     _sum = vmlaq_f32(_sum, _r3, vld1q_f32(k3));

  1358. 

  1359.                     float32x4_t _r4 = vld1q_f32(r4);

  1360.                     _sum = vmlaq_f32(_sum, _r4, _k20212223);

  1361. 

  1362.                     sum += r0[4] * k0[4];

  1363.                     sum += r1[4] * k1[4];

  1364.                     sum += r2[4] * k2[4];

  1365.                     sum += r3[4] * k3[4];

  1366.                     sum += r4[4] * k4[4];

  1367. 

  1368.                     float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));

  1369.                     _ss = vpadd_f32(_ss, _ss);

  1370. 

  1371.                     sum += vget_lane_f32(_ss, 0);

  1372. #else

  1373.                     sum += r0[0] * k0[0];

  1374.                     sum += r0[1] * k0[1];

  1375.                     sum += r0[2] * k0[2];

  1376.                     sum += r0[3] * k0[3];

  1377.                     sum += r0[4] * k0[4];

  1378. 

  1379.                     sum += r1[0] * k1[0];

  1380.                     sum += r1[1] * k1[1];

  1381.                     sum += r1[2] * k1[2];

  1382.                     sum += r1[3] * k1[3];

  1383.                     sum += r1[4] * k1[4];

  1384. 

  1385.                     sum += r2[0] * k2[0];

  1386.                     sum += r2[1] * k2[1];

  1387.                     sum += r2[2] * k2[2];

  1388.                     sum += r2[3] * k2[3];

  1389.                     sum += r2[4] * k2[4];

  1390. 

  1391.                     sum += r3[0] * k3[0];

  1392.                     sum += r3[1] * k3[1];

  1393.                     sum += r3[2] * k3[2];

  1394.                     sum += r3[3] * k3[3];

  1395.                     sum += r3[4] * k3[4];

  1396. 

  1397.                     sum += r4[0] * k4[0];

  1398.                     sum += r4[1] * k4[1];

  1399.                     sum += r4[2] * k4[2];

  1400.                     sum += r4[3] * k4[3];

  1401.                     sum += r4[4] * k4[4];

  1402. #endif

  1403.                     *outptr += sum;

  1404. 

  1405.                     r0 += 2;

  1406.                     r1 += 2;

  1407.                     r2 += 2;

  1408.                     r3 += 2;

  1409.                     r4 += 2;

  1410.                     outptr++;

  1411.                 }

  1412. 

  1413.                 r0 += tailstep;

  1414.                 r1 += tailstep;

  1415.                 r2 += tailstep;

  1416.                 r3 += tailstep;

  1417.                 r4 += tailstep;

  1418.             }

  1419. 

  1420.         }

  1421.     }

  1422. 

  1423. }