nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 9706cd1447
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '9706cd1447'
${ noResults }
ncnn/src/layer/arm/convolution_4x4.h

411 lines
16 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #if __ARM_NEON

  16. #include <arm_neon.h>

  17. #endif // __ARM_NEON

  18. 

  19. static void conv4x4s4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias, const Option& opt)

  20. {

  21.     int w = bottom_blob.w;

  22.     int inch = bottom_blob.c;

  23. 

  24.     int outw = top_blob.w;

  25.     int outh = top_blob.h;

  26.     int outch = top_blob.c;

  27. 

  28.     const int tailstep = w - 4*outw + w*3;

  29. 

  30.     const float* kernel = _kernel;

  31.     const float* bias = _bias;

  32. 

  33.     #pragma omp parallel for num_threads(opt.num_threads)

  34.     for (int p=0; p<outch; p++)

  35.     {

  36.         Mat out = top_blob.channel(p);

  37. 

  38.         const float bias0 = bias ? bias[p] : 0.f;

  39. 

  40.         out.fill(bias0);

  41. 

  42.         for (int q=0; q<inch; q++)

  43.         {

  44.             float* outptr = out;

  45. 

  46.             const float* img0 = bottom_blob.channel(q);

  47. 

  48.             const float* kernel0 = kernel + p*inch*16 + q*16;

  49. 

  50.             const float* r0 = img0;

  51.             const float* r1 = img0 + w;

  52.             const float* r2 = img0 + w*2;

  53.             const float* r3 = img0 + w*3;

  54. 

  55. #if __ARM_NEON

  56.             float32x4_t _k0123 = vld1q_f32(kernel0);

  57.             float32x4_t _k4567 = vld1q_f32(kernel0+4);

  58.             float32x4_t _k891011 = vld1q_f32(kernel0+8);

  59.             float32x4_t _k12131415 = vld1q_f32(kernel0+12);

  60. #else

  61.             const float* k0 = kernel0;

  62.             const float* k1 = kernel0 + 4;

  63.             const float* k2 = kernel0 + 8;

  64.             const float* k3 = kernel0 + 12;

  65. #endif // __ARM_NEON

  66. 

  67.             for (int i = 0; i < outh; i++)

  68.             {

  69. #if __ARM_NEON

  70.                 int nn = outw >> 2;

  71.                 int remain = outw - (nn << 2);

  72. #else

  73.                 int remain = outw;

  74. #endif // __ARM_NEON

  75. 

  76. #if __ARM_NEON

  77. #if __aarch64__

  78.                 if (nn > 0)

  79.                 {

  80.                 asm volatile(

  81.                     "prfm       pldl1keep, [%1, #128]          \n"

  82.                     "0:                                        \n"

  83. 

  84.                     "prfm       pldl1keep, [%2, #512]          \n"

  85.                     "prfm       pldl1keep, [%3, #512]          \n"

  86. 

  87.                     "ld1        {v7.4s}, [%1]                  \n" // v7 = outptr

  88. 

  89.                     "ld1        {v8.4s}, [%2], #16             \n"// v8  = r0

  90.                     "ld1        {v9.4s}, [%3], #16             \n"// v9  = r1

  91. 

  92.                     "prfm       pldl1keep, [%4, #512]          \n"

  93.                     "prfm       pldl1keep, [%5, #512]          \n"

  94. 

  95.                     "fmul       v12.4s, v8.4s, %12.4s          \n"

  96.                     "fmul       v13.4s, v9.4s, %13.4s          \n"

  97. 

  98.                     "ld1        {v10.4s}, [%4], #16            \n"// v10 = r2

  99.                     "ld1        {v11.4s}, [%5], #16            \n"// v11 = r3

  100. 

  101.                     "fmla       v12.4s, v10.4s, %14.4s         \n"

  102.                     "fmla       v13.4s, v11.4s, %15.4s         \n"

  103. 

  104.                     "fadd       v5.4s, v12.4s, v13.4s          \n"

  105. 

  106.                     "ld1        {v8.4s}, [%2], #16             \n"// v8  = r0

  107.                     "ld1        {v9.4s}, [%3], #16             \n"// v9  = r1

  108. 

  109.                     "fmul       v12.4s, v8.4s, %12.4s          \n"

  110.                     "fmul       v13.4s, v9.4s, %13.4s          \n"

  111. 

  112.                     "ld1        {v10.4s}, [%4], #16            \n"// v10 = r2

  113.                     "ld1        {v11.4s}, [%5], #16            \n"// v11 = r3

  114.                     

  115.                     "fmla       v12.4s, v10.4s, %14.4s         \n"

  116.                     "fmla       v13.4s, v11.4s, %15.4s         \n"

  117. 

  118.                     "fadd       v6.4s, v12.4s, v13.4s          \n"

  119. 

  120.                     "ld1        {v8.4s}, [%2], #16             \n"// v8  = r0

  121.                     "ld1        {v9.4s}, [%3], #16             \n"// v9  = r1

  122. 

  123.                     "fmul       v12.4s, v8.4s, %12.4s          \n"

  124.                     "fmul       v13.4s, v9.4s, %13.4s          \n"

  125. 

  126.                     "ld1        {v10.4s}, [%4], #16            \n"// v10 = r2

  127.                     "ld1        {v11.4s}, [%5], #16            \n"// v11 = r3

  128. 

  129.                     "fmla       v12.4s, v10.4s, %14.4s         \n"

  130.                     "fmla       v13.4s, v11.4s, %15.4s         \n"

  131. 

  132.                     "fadd       v14.4s, v12.4s, v13.4s         \n"

  133.                     "faddp      v5.4s, v5.4s, v6.4s            \n"  // Move to here to enhance ILP

  134. 

  135.                     "ld1        {v8.4s}, [%2], #16             \n"// v8  = r0

  136.                     "ld1        {v9.4s}, [%3], #16             \n"// v9  = r1

  137. 

  138.                     "fmul       v12.4s, v8.4s, %12.4s          \n"

  139.                     "fmul       v13.4s, v9.4s, %13.4s          \n"

  140. 

  141.                     "ld1        {v10.4s}, [%4], #16            \n"// v10 = r2

  142.                     "ld1        {v11.4s}, [%5], #16            \n"// v11 = r3

  143. 

  144.                     "fmla       v12.4s, v10.4s, %14.4s         \n"

  145.                     "fmla       v13.4s, v11.4s, %15.4s         \n"

  146. 

  147.                     "fadd       v15.4s, v12.4s, v13.4s         \n"

  148. 

  149. //                  "faddp      v5.4s ,  v5.4s,  v6.4s         \n"  // Move this line upward.

  150.                     "faddp      v14.4s, v14.4s, v15.4s         \n"

  151.                     "faddp      v5.4s ,  v5.4s, v14.4s         \n"

  152.             

  153.                     "fadd       v7.4s, v7.4s, v5.4s            \n"

  154. 

  155.                     "st1        {v7.4s}, [%1], #16             \n"

  156. 

  157.                     "prfm       pldl1keep, [%1, #128]          \n"

  158. 

  159.                     "subs       %w0, %w0, #1                   \n"

  160.                     "bne        0b                             \n"

  161.                     : "=r"(nn),         // %0

  162.                       "=r"(outptr),     // %1

  163.                       "=r"(r0),         // %2

  164.                       "=r"(r1),         // %3

  165.                       "=r"(r2),         // %4

  166.                       "=r"(r3)          // %5

  167.                     : "0"(nn),

  168.                       "1"(outptr),

  169.                       "2"(r0),

  170.                       "3"(r1),

  171.                       "4"(r2),

  172.                       "5"(r3),

  173.                       "w"(_k0123),      // %12

  174.                       "w"(_k4567),      // %13

  175.                       "w"(_k891011),    // %14

  176.                       "w"(_k12131415)   // %15

  177.                     : "cc", "memory", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"

  178.                 );

  179.                 }

  180. #else

  181.                 if (nn > 0)

  182.                 {

  183.                 asm volatile(

  184. 

  185.                     "pld        [%1, #128]          \n"

  186. 

  187.                     "0:                             \n"

  188. 

  189.                     "pld        [%2, #512]          \n"

  190.                     "pld        [%3, #512]          \n"

  191. 

  192.                     "vld1.f32   {d14-d15}, [%1]     \n"// q7 = outptr

  193. 

  194.                     "vld1.f32   {d16-d17}, [%2]!    \n"// q8  = r0

  195.                     "vld1.f32   {d18-d19}, [%3]!    \n"// q9  = r1

  196. 

  197.                     "pld        [%4, #512]          \n"

  198.                     "pld        [%5, #512]          \n"

  199. 

  200.                     "vmul.f32   q12, q8, %q12       \n"

  201.                     "vmul.f32   q13, q9, %q13       \n"

  202. 

  203.                     "vld1.f32   {d20-d21}, [%4]!    \n"// q10 = r2

  204.                     "vld1.f32   {d22-d23}, [%5]!    \n"// q11 = r3

  205. 

  206.                     "vmla.f32   q12, q10, %q14      \n"

  207.                     "vmla.f32   q13, q11, %q15      \n"

  208. 

  209.                     "vadd.f32   q5, q12, q13        \n"

  210. 

  211.                     "vld1.f32   {d16-d17}, [%2]!    \n"// q8  = r0

  212.                     "vld1.f32   {d18-d19}, [%3]!    \n"// q9  = r1

  213. 

  214.                     "vmul.f32   q12, q8, %q12       \n"

  215.                     "vmul.f32   q13, q9, %q13       \n"

  216. 

  217.                     "vld1.f32   {d20-d21}, [%4]!    \n"// q10 = r2

  218.                     "vld1.f32   {d22-d23}, [%5]!    \n"// q11 = r3

  219. 

  220.                     "vmla.f32   q12, q10, %q14      \n"

  221.                     "vmla.f32   q13, q11, %q15      \n"

  222. 

  223.                     "vadd.f32   q6, q12, q13        \n"

  224. 

  225.                     "vld1.f32   {d16-d17}, [%2]!    \n"// q8  = r0

  226.                     "vld1.f32   {d18-d19}, [%3]!    \n"// q9  = r1

  227. 

  228.                     "vmul.f32   q12, q8, %q12       \n"

  229.                     "vmul.f32   q13, q9, %q13       \n"

  230. 

  231.                     "vld1.f32   {d20-d21}, [%4]!    \n"// q10 = r2

  232.                     "vld1.f32   {d22-d23}, [%5]!    \n"// q11 = r3

  233. 

  234.                     "vmla.f32   q12, q10, %q14      \n"

  235.                     "vmla.f32   q13, q11, %q15      \n"

  236. 

  237.                     "vadd.f32   q14, q12, q13       \n"

  238. 

  239.                     "vld1.f32   {d16-d17}, [%2]!    \n"// q8  = r0

  240.                     "vld1.f32   {d18-d19}, [%3]!    \n"// q9  = r1

  241. 

  242.                     "vmul.f32   q12, q8, %q12       \n"

  243.                     "vmul.f32   q13, q9, %q13       \n"

  244. 

  245.                     "vld1.f32   {d20-d21}, [%4]!    \n"// q10 = r2

  246.                     "vld1.f32   {d22-d23}, [%5]!    \n"// q11 = r3

  247. 

  248.                     "vmla.f32   q12, q10, %q14      \n"

  249.                     "vmla.f32   q13, q11, %q15      \n"

  250. 

  251.                     "vadd.f32   q15, q12, q13       \n"

  252. 

  253.                     "vadd.f32   d10, d10, d11       \n"

  254.                     "vadd.f32   d28, d28, d29       \n"

  255.                     "vadd.f32   d11, d12, d13       \n"

  256.                     "vadd.f32   d29, d30, d31       \n"

  257. 

  258.                     "vpadd.f32  d10, d10, d11       \n"

  259.                     "vpadd.f32  d11, d28, d29       \n"

  260. 

  261.                     "vadd.f32   q7, q7, q5          \n"

  262. 

  263.                     "vst1.f32   {d14-d15}, [%1]!    \n"

  264. 

  265.                     "pld        [%1, #128]          \n"

  266. 

  267.                     "subs       %0, #1              \n"

  268.                     "bne        0b                  \n"

  269.                     : "=r"(nn),         // %0

  270.                       "=r"(outptr),     // %1

  271.                       "=r"(r0),         // %2

  272.                       "=r"(r1),         // %3

  273.                       "=r"(r2),         // %4

  274.                       "=r"(r3)          // %5

  275.                     : "0"(nn),

  276.                       "1"(outptr),

  277.                       "2"(r0),

  278.                       "3"(r1),

  279.                       "4"(r2),

  280.                       "5"(r3),

  281.                       "w"(_k0123),      // %12

  282.                       "w"(_k4567),      // %13

  283.                       "w"(_k891011),    // %14

  284.                       "w"(_k12131415)   // %15

  285.                     : "cc", "memory", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"

  286.                 );

  287.                 }

  288. #endif // __aarch64__

  289. #endif // __ARM_NEON

  290.                 for (; remain>0; remain--)

  291.                 {

  292. #if __ARM_NEON

  293. #if __aarch64__

  294.                     float sum = 0.f;

  295. 

  296.                     asm volatile(

  297.                         "ld1        {v8.4s}, [%0], #16             \n"// v8  = r0

  298.                         "ld1        {v9.4s}, [%1], #16             \n"// v9  = r1

  299. 

  300.                         "fmul       v12.4s, v8.4s, %9.4s           \n"

  301.                         "fmul       v13.4s, v9.4s, %10.4s          \n"

  302. 

  303.                         "ld1        {v10.4s}, [%2], #16            \n"// v10 = r2

  304.                         "ld1        {v11.4s}, [%3], #16            \n"// v11 = r3

  305. 

  306.                         "fmla       v12.4s, v10.4s, %11.4s         \n"

  307.                         "fmla       v13.4s, v11.4s, %12.4s         \n"

  308. 

  309.                         "fadd       v5.4s, v12.4s, v13.4s          \n"            

  310.                         "faddp      v5.4s, v5.4s, v5.4s            \n"  

  311.                         "faddp      s5, v5.2s                      \n" 

  312.                         "fmov       %w4, s5                        \n"

  313.                         : "=r"(r0),         // %0

  314.                           "=r"(r1),         // %1

  315.                           "=r"(r2),         // %2

  316.                           "=r"(r3),         // %3

  317.                           "=r"(sum)         // %4

  318.                         : "0"(r0),

  319.                           "1"(r1),

  320.                           "2"(r2),

  321.                           "3"(r3),

  322.                           "w"(_k0123),      // %9

  323.                           "w"(_k4567),      // %10

  324.                           "w"(_k891011),    // %11

  325.                           "w"(_k12131415)   // %12

  326.                         : "cc", "memory", "v5", "v6", "v8", "v9", "v10", "v11", "v12", "v13"

  327.                     );

  328.                     

  329.                     *outptr += sum;

  330. #else

  331.                     float sum = 0.f;

  332. 

  333.                     asm volatile(

  334.                         "vld1.f32   {d16-d17}, [%0]!    \n"// q8  = r0

  335.                         "vld1.f32   {d18-d19}, [%1]!    \n"// q9  = r1

  336. 

  337.                         "vmul.f32   q12, q8, %q9        \n"

  338.                         "vmul.f32   q13, q9, %q10       \n"

  339. 

  340.                         "vld1.f32   {d20-d21}, [%2]!    \n"// q10 = r2

  341.                         "vld1.f32   {d22-d23}, [%3]!    \n"// q11 = r3

  342. 

  343.                         "vmla.f32   q12, q10, %q11      \n"

  344.                         "vmla.f32   q13, q11, %q12      \n"

  345. 

  346.                         "vadd.f32   q5, q12, q13        \n"

  347.                         "vadd.f32   d10, d10, d11       \n"

  348.                         "vpadd.f32  d10, d10, d10       \n"

  349.                         "vmov.f32   %4, d10[0]          \n"

  350.                         : "=r"(r0),         // %0

  351.                           "=r"(r1),         // %1

  352.                           "=r"(r2),         // %2

  353.                           "=r"(r3),         // %3

  354.                           "=r"(sum)         // %4

  355.                         : "0"(r0),

  356.                           "1"(r1),

  357.                           "2"(r2),

  358.                           "3"(r3),

  359.                           "w"(_k0123),      // %9

  360.                           "w"(_k4567),      // %10

  361.                           "w"(_k891011),    // %11

  362.                           "w"(_k12131415)   // %12

  363.                         : "cc", "memory", "q5", "q6", "q8", "q9", "q10", "q11", "q12", "q13"

  364.                     );

  365. 

  366.                     *outptr += sum;

  367. #endif // __aarch64__

  368. #else

  369.                     float sum = 0;

  370. 

  371.                     sum += r0[0] * k0[0];

  372.                     sum += r0[1] * k0[1];

  373.                     sum += r0[2] * k0[2];

  374.                     sum += r0[3] * k0[3];

  375. 

  376.                     sum += r1[0] * k1[0];

  377.                     sum += r1[1] * k1[1];

  378.                     sum += r1[2] * k1[2];

  379.                     sum += r1[3] * k1[3];

  380. 

  381.                     sum += r2[0] * k2[0];

  382.                     sum += r2[1] * k2[1];

  383.                     sum += r2[2] * k2[2];

  384.                     sum += r2[3] * k2[3];

  385. 

  386.                     sum += r3[0] * k3[0];

  387.                     sum += r3[1] * k3[1];

  388.                     sum += r3[2] * k3[2];

  389.                     sum += r3[3] * k3[3];

  390. 

  391.                     *outptr += sum;

  392. 

  393.                     r0 += 4;

  394.                     r1 += 4;

  395.                     r2 += 4;

  396.                     r3 += 4;

  397. #endif // __ARM_NEON

  398.                     outptr++;

  399.                 }

  400. 

  401.                 r0 += tailstep;

  402.                 r1 += tailstep;

  403.                 r2 += tailstep;

  404.                 r3 += tailstep;

  405.             }

  406. 

  407.         }

  408.     }

  409. 

  410. }