nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: dba87f8cad
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'dba87f8cad'
${ noResults }
ncnn/src/layer/arm/convolution1d_packed_fp16s.h

1884 lines
73 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. static void convolution1d_transform_kernel_packed_fp16s(const Mat& kernel, Mat& kernel_tm, int inh, int outh, int kernel_w)

  16. {

  17.     // src = kw-inh-outh

  18.     // dst = pb-pa-kw-inh/pa-outh/pb

  19. 

  20.     // clang-format off

  21.     // *INDENT-OFF*

  22.     if (outh >= 8)

  23.     {

  24.         if (inh >= 8)

  25.             kernel_tm.create(8 * 8 * kernel_w, inh / 8 + (inh % 8) / 4 + (inh % 4) / 2 + inh % 2, outh / 8 + (outh % 8) / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  26.         else if (inh >= 4)

  27.             kernel_tm.create(8 * 4 * kernel_w, inh / 4 + (inh % 4) / 2 + inh % 2, outh / 8 + (outh % 8) / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  28.         else if (inh >= 2)

  29.             kernel_tm.create(8 * 2 * kernel_w, inh / 2 + inh % 2, outh / 8 + (outh % 8) / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  30.         else

  31.             kernel_tm.create(8 * kernel_w, inh, outh / 8 + (outh % 8) / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  32.     }

  33.     else if (outh >= 4)

  34.     {

  35.         if (inh >= 8)

  36.             kernel_tm.create(4 * 8 * kernel_w, inh / 8 + (inh % 8) / 4 + (inh % 4) / 2 + inh % 2, outh / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  37.         else if (inh >= 4)

  38.             kernel_tm.create(4 * 4 * kernel_w, inh / 4 + (inh % 4) / 2 + inh % 2, outh / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  39.         else if (inh >= 2)

  40.             kernel_tm.create(4 * 2 * kernel_w, inh / 2 + inh % 2, outh / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  41.         else

  42.             kernel_tm.create(4 * kernel_w, inh, outh / 4 + (outh % 4) / 2 + outh % 2, (size_t)2u);

  43.     }

  44.     else if (outh >= 2)

  45.     {

  46.         if (inh >= 8)

  47.             kernel_tm.create(2 * 8 * kernel_w, inh / 8 + (inh % 8) / 4 + (inh % 4) / 2 + inh % 2, outh / 2 + outh % 2, (size_t)2u);

  48.         else if (inh >= 4)

  49.             kernel_tm.create(2 * 4 * kernel_w, inh / 4 + (inh % 4) / 2 + inh % 2, outh / 2 + outh % 2, (size_t)2u);

  50.         else if (inh >= 2)

  51.             kernel_tm.create(2 * 2 * kernel_w, inh / 2 + inh % 2, outh / 2 + outh % 2, (size_t)2u);

  52.         else

  53.             kernel_tm.create(2 * kernel_w, inh, outh / 2 + outh % 2, (size_t)2u);

  54.     }

  55.     else

  56.     {

  57.         if (inh >= 8)

  58.             kernel_tm.create(8 * kernel_w, inh / 8 + (inh % 8) / 4 + (inh % 4) / 2 + inh % 2, outh, (size_t)2u);

  59.         else if (inh >= 4)

  60.             kernel_tm.create(4 * kernel_w, inh / 4 + (inh % 4) / 2 + inh % 2, outh, (size_t)2u);

  61.         else if (inh >= 2)

  62.             kernel_tm.create(2 * kernel_w, inh / 2 + inh % 2, outh, (size_t)2u);

  63.         else

  64.             kernel_tm.create(kernel_w, inh, outh, (size_t)2u);

  65.     }

  66.     // *INDENT-ON*

  67.     // clang-format on

  68. 

  69.     int q = 0;

  70.     for (; q + 7 < outh; q += 8)

  71.     {

  72.         const float* kptr0 = (const float*)kernel + q * inh * kernel_w;

  73.         const float* kptr1 = (const float*)kernel + (q + 1) * inh * kernel_w;

  74.         const float* kptr2 = (const float*)kernel + (q + 2) * inh * kernel_w;

  75.         const float* kptr3 = (const float*)kernel + (q + 3) * inh * kernel_w;

  76.         const float* kptr4 = (const float*)kernel + (q + 4) * inh * kernel_w;

  77.         const float* kptr5 = (const float*)kernel + (q + 5) * inh * kernel_w;

  78.         const float* kptr6 = (const float*)kernel + (q + 6) * inh * kernel_w;

  79.         const float* kptr7 = (const float*)kernel + (q + 7) * inh * kernel_w;

  80. 

  81.         __fp16* g00 = kernel_tm.channel(q / 8);

  82. 

  83.         int p = 0;

  84.         for (; p + 7 < inh; p += 8)

  85.         {

  86.             for (int k = 0; k < kernel_w; k++)

  87.             {

  88.                 const float* k0 = kptr0 + p * kernel_w;

  89.                 const float* k1 = kptr1 + p * kernel_w;

  90.                 const float* k2 = kptr2 + p * kernel_w;

  91.                 const float* k3 = kptr3 + p * kernel_w;

  92.                 const float* k4 = kptr4 + p * kernel_w;

  93.                 const float* k5 = kptr5 + p * kernel_w;

  94.                 const float* k6 = kptr6 + p * kernel_w;

  95.                 const float* k7 = kptr7 + p * kernel_w;

  96. 

  97.                 for (int i = 0; i < 8; i++)

  98.                 {

  99.                     g00[0] = (__fp16)k0[k];

  100.                     g00[1] = (__fp16)k1[k];

  101.                     g00[2] = (__fp16)k2[k];

  102.                     g00[3] = (__fp16)k3[k];

  103.                     g00[4] = (__fp16)k4[k];

  104.                     g00[5] = (__fp16)k5[k];

  105.                     g00[6] = (__fp16)k6[k];

  106.                     g00[7] = (__fp16)k7[k];

  107.                     k0 += kernel_w;

  108.                     k1 += kernel_w;

  109.                     k2 += kernel_w;

  110.                     k3 += kernel_w;

  111.                     k4 += kernel_w;

  112.                     k5 += kernel_w;

  113.                     k6 += kernel_w;

  114.                     k7 += kernel_w;

  115.                     g00 += 8;

  116.                 }

  117.             }

  118.         }

  119.         for (; p + 3 < inh; p += 4)

  120.         {

  121.             for (int k = 0; k < kernel_w; k++)

  122.             {

  123.                 const float* k0 = kptr0 + p * kernel_w;

  124.                 const float* k1 = kptr1 + p * kernel_w;

  125.                 const float* k2 = kptr2 + p * kernel_w;

  126.                 const float* k3 = kptr3 + p * kernel_w;

  127.                 const float* k4 = kptr4 + p * kernel_w;

  128.                 const float* k5 = kptr5 + p * kernel_w;

  129.                 const float* k6 = kptr6 + p * kernel_w;

  130.                 const float* k7 = kptr7 + p * kernel_w;

  131. 

  132.                 for (int i = 0; i < 4; i++)

  133.                 {

  134.                     g00[0] = (__fp16)k0[k];

  135.                     g00[1] = (__fp16)k1[k];

  136.                     g00[2] = (__fp16)k2[k];

  137.                     g00[3] = (__fp16)k3[k];

  138.                     g00[4] = (__fp16)k4[k];

  139.                     g00[5] = (__fp16)k5[k];

  140.                     g00[6] = (__fp16)k6[k];

  141.                     g00[7] = (__fp16)k7[k];

  142.                     k0 += kernel_w;

  143.                     k1 += kernel_w;

  144.                     k2 += kernel_w;

  145.                     k3 += kernel_w;

  146.                     k4 += kernel_w;

  147.                     k5 += kernel_w;

  148.                     k6 += kernel_w;

  149.                     k7 += kernel_w;

  150.                     g00 += 8;

  151.                 }

  152.             }

  153.         }

  154.         for (; p + 1 < inh; p += 2)

  155.         {

  156.             for (int k = 0; k < kernel_w; k++)

  157.             {

  158.                 const float* k0 = kptr0 + p * kernel_w;

  159.                 const float* k1 = kptr1 + p * kernel_w;

  160.                 const float* k2 = kptr2 + p * kernel_w;

  161.                 const float* k3 = kptr3 + p * kernel_w;

  162.                 const float* k4 = kptr4 + p * kernel_w;

  163.                 const float* k5 = kptr5 + p * kernel_w;

  164.                 const float* k6 = kptr6 + p * kernel_w;

  165.                 const float* k7 = kptr7 + p * kernel_w;

  166. 

  167.                 for (int i = 0; i < 2; i++)

  168.                 {

  169.                     g00[0] = (__fp16)k0[k];

  170.                     g00[1] = (__fp16)k1[k];

  171.                     g00[2] = (__fp16)k2[k];

  172.                     g00[3] = (__fp16)k3[k];

  173.                     g00[4] = (__fp16)k4[k];

  174.                     g00[5] = (__fp16)k5[k];

  175.                     g00[6] = (__fp16)k6[k];

  176.                     g00[7] = (__fp16)k7[k];

  177.                     k0 += kernel_w;

  178.                     k1 += kernel_w;

  179.                     k2 += kernel_w;

  180.                     k3 += kernel_w;

  181.                     k4 += kernel_w;

  182.                     k5 += kernel_w;

  183.                     k6 += kernel_w;

  184.                     k7 += kernel_w;

  185.                     g00 += 8;

  186.                 }

  187.             }

  188.         }

  189.         for (; p < inh; p++)

  190.         {

  191.             const float* k0 = kptr0 + p * kernel_w;

  192.             const float* k1 = kptr1 + p * kernel_w;

  193.             const float* k2 = kptr2 + p * kernel_w;

  194.             const float* k3 = kptr3 + p * kernel_w;

  195.             const float* k4 = kptr4 + p * kernel_w;

  196.             const float* k5 = kptr5 + p * kernel_w;

  197.             const float* k6 = kptr6 + p * kernel_w;

  198.             const float* k7 = kptr7 + p * kernel_w;

  199. 

  200.             for (int k = 0; k < kernel_w; k++)

  201.             {

  202.                 g00[0] = (__fp16)k0[k];

  203.                 g00[1] = (__fp16)k1[k];

  204.                 g00[2] = (__fp16)k2[k];

  205.                 g00[3] = (__fp16)k3[k];

  206.                 g00[4] = (__fp16)k4[k];

  207.                 g00[5] = (__fp16)k5[k];

  208.                 g00[6] = (__fp16)k6[k];

  209.                 g00[7] = (__fp16)k7[k];

  210.                 g00 += 8;

  211.             }

  212.         }

  213.     }

  214.     for (; q + 3 < outh; q += 4)

  215.     {

  216.         const float* kptr0 = (const float*)kernel + q * inh * kernel_w;

  217.         const float* kptr1 = (const float*)kernel + (q + 1) * inh * kernel_w;

  218.         const float* kptr2 = (const float*)kernel + (q + 2) * inh * kernel_w;

  219.         const float* kptr3 = (const float*)kernel + (q + 3) * inh * kernel_w;

  220. 

  221.         __fp16* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4);

  222. 

  223.         int p = 0;

  224.         for (; p + 7 < inh; p += 8)

  225.         {

  226.             for (int k = 0; k < kernel_w; k++)

  227.             {

  228.                 const float* k0 = kptr0 + p * kernel_w;

  229.                 const float* k1 = kptr1 + p * kernel_w;

  230.                 const float* k2 = kptr2 + p * kernel_w;

  231.                 const float* k3 = kptr3 + p * kernel_w;

  232. 

  233.                 for (int i = 0; i < 8; i++)

  234.                 {

  235.                     g00[0] = (__fp16)k0[k];

  236.                     g00[1] = (__fp16)k1[k];

  237.                     g00[2] = (__fp16)k2[k];

  238.                     g00[3] = (__fp16)k3[k];

  239.                     k0 += kernel_w;

  240.                     k1 += kernel_w;

  241.                     k2 += kernel_w;

  242.                     k3 += kernel_w;

  243.                     g00 += 4;

  244.                 }

  245.             }

  246.         }

  247.         for (; p + 3 < inh; p += 4)

  248.         {

  249.             for (int k = 0; k < kernel_w; k++)

  250.             {

  251.                 const float* k0 = kptr0 + p * kernel_w;

  252.                 const float* k1 = kptr1 + p * kernel_w;

  253.                 const float* k2 = kptr2 + p * kernel_w;

  254.                 const float* k3 = kptr3 + p * kernel_w;

  255. 

  256.                 for (int i = 0; i < 4; i++)

  257.                 {

  258.                     g00[0] = (__fp16)k0[k];

  259.                     g00[1] = (__fp16)k1[k];

  260.                     g00[2] = (__fp16)k2[k];

  261.                     g00[3] = (__fp16)k3[k];

  262.                     k0 += kernel_w;

  263.                     k1 += kernel_w;

  264.                     k2 += kernel_w;

  265.                     k3 += kernel_w;

  266.                     g00 += 4;

  267.                 }

  268.             }

  269.         }

  270.         for (; p + 1 < inh; p += 2)

  271.         {

  272.             for (int k = 0; k < kernel_w; k++)

  273.             {

  274.                 const float* k0 = kptr0 + p * kernel_w;

  275.                 const float* k1 = kptr1 + p * kernel_w;

  276.                 const float* k2 = kptr2 + p * kernel_w;

  277.                 const float* k3 = kptr3 + p * kernel_w;

  278. 

  279.                 for (int i = 0; i < 2; i++)

  280.                 {

  281.                     g00[0] = (__fp16)k0[k];

  282.                     g00[1] = (__fp16)k1[k];

  283.                     g00[2] = (__fp16)k2[k];

  284.                     g00[3] = (__fp16)k3[k];

  285.                     k0 += kernel_w;

  286.                     k1 += kernel_w;

  287.                     k2 += kernel_w;

  288.                     k3 += kernel_w;

  289.                     g00 += 4;

  290.                 }

  291.             }

  292.         }

  293.         for (; p < inh; p++)

  294.         {

  295.             const float* k0 = kptr0 + p * kernel_w;

  296.             const float* k1 = kptr1 + p * kernel_w;

  297.             const float* k2 = kptr2 + p * kernel_w;

  298.             const float* k3 = kptr3 + p * kernel_w;

  299. 

  300.             for (int k = 0; k < kernel_w; k++)

  301.             {

  302.                 g00[0] = (__fp16)k0[k];

  303.                 g00[1] = (__fp16)k1[k];

  304.                 g00[2] = (__fp16)k2[k];

  305.                 g00[3] = (__fp16)k3[k];

  306.                 g00 += 4;

  307.             }

  308.         }

  309.     }

  310.     for (; q + 1 < outh; q += 2)

  311.     {

  312.         const float* kptr0 = (const float*)kernel + q * inh * kernel_w;

  313.         const float* kptr1 = (const float*)kernel + (q + 1) * inh * kernel_w;

  314. 

  315.         __fp16* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4 + (q % 4) / 2);

  316. 

  317.         int p = 0;

  318.         for (; p + 7 < inh; p += 8)

  319.         {

  320.             for (int k = 0; k < kernel_w; k++)

  321.             {

  322.                 const float* k0 = kptr0 + p * kernel_w + k;

  323.                 const float* k1 = kptr1 + p * kernel_w + k;

  324. 

  325.                 g00[0] = (__fp16)k0[0];

  326.                 g00[1] = (__fp16)k0[kernel_w];

  327.                 g00[2] = (__fp16)k0[kernel_w * 2];

  328.                 g00[3] = (__fp16)k0[kernel_w * 3];

  329.                 g00[4] = (__fp16)k0[kernel_w * 4];

  330.                 g00[5] = (__fp16)k0[kernel_w * 5];

  331.                 g00[6] = (__fp16)k0[kernel_w * 6];

  332.                 g00[7] = (__fp16)k0[kernel_w * 7];

  333.                 g00[8] = (__fp16)k1[0];

  334.                 g00[9] = (__fp16)k1[kernel_w];

  335.                 g00[10] = (__fp16)k1[kernel_w * 2];

  336.                 g00[11] = (__fp16)k1[kernel_w * 3];

  337.                 g00[12] = (__fp16)k1[kernel_w * 4];

  338.                 g00[13] = (__fp16)k1[kernel_w * 5];

  339.                 g00[14] = (__fp16)k1[kernel_w * 6];

  340.                 g00[15] = (__fp16)k1[kernel_w * 7];

  341.                 g00 += 16;

  342.             }

  343.         }

  344.         for (; p + 3 < inh; p += 4)

  345.         {

  346.             for (int k = 0; k < kernel_w; k++)

  347.             {

  348.                 const float* k0 = kptr0 + p * kernel_w + k;

  349.                 const float* k1 = kptr1 + p * kernel_w + k;

  350. 

  351.                 g00[0] = (__fp16)k0[0];

  352.                 g00[1] = (__fp16)k0[kernel_w];

  353.                 g00[2] = (__fp16)k0[kernel_w * 2];

  354.                 g00[3] = (__fp16)k0[kernel_w * 3];

  355.                 g00[4] = (__fp16)k1[0];

  356.                 g00[5] = (__fp16)k1[kernel_w];

  357.                 g00[6] = (__fp16)k1[kernel_w * 2];

  358.                 g00[7] = (__fp16)k1[kernel_w * 3];

  359.                 g00 += 8;

  360.             }

  361.         }

  362.         for (; p + 1 < inh; p += 2)

  363.         {

  364.             for (int k = 0; k < kernel_w; k++)

  365.             {

  366.                 const float* k0 = kptr0 + p * kernel_w;

  367.                 const float* k1 = kptr1 + p * kernel_w;

  368. 

  369.                 for (int i = 0; i < 2; i++)

  370.                 {

  371.                     g00[0] = (__fp16)k0[k];

  372.                     g00[1] = (__fp16)k1[k];

  373.                     k0 += kernel_w;

  374.                     k1 += kernel_w;

  375.                     g00 += 2;

  376.                 }

  377.             }

  378.         }

  379.         for (; p < inh; p++)

  380.         {

  381.             const float* k0 = kptr0 + p * kernel_w;

  382.             const float* k1 = kptr1 + p * kernel_w;

  383. 

  384.             for (int k = 0; k < kernel_w; k++)

  385.             {

  386.                 g00[0] = (__fp16)k0[k];

  387.                 g00[1] = (__fp16)k1[k];

  388.                 g00 += 2;

  389.             }

  390.         }

  391.     }

  392.     for (; q < outh; q++)

  393.     {

  394.         const float* kptr = (const float*)kernel + q * inh * kernel_w;

  395. 

  396.         __fp16* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4 + (q % 4) / 2 + q % 2);

  397. 

  398.         int p = 0;

  399.         for (; p + 7 < inh; p += 8)

  400.         {

  401.             for (int k = 0; k < kernel_w; k++)

  402.             {

  403.                 const float* k0 = kptr + p * kernel_w;

  404. 

  405.                 for (int i = 0; i < 8; i++)

  406.                 {

  407.                     g00[0] = (__fp16)k0[k];

  408.                     k0 += kernel_w;

  409.                     g00 += 1;

  410.                 }

  411.             }

  412.         }

  413.         for (; p + 3 < inh; p += 4)

  414.         {

  415.             for (int k = 0; k < kernel_w; k++)

  416.             {

  417.                 const float* k0 = kptr + p * kernel_w;

  418. 

  419.                 for (int i = 0; i < 4; i++)

  420.                 {

  421.                     g00[0] = (__fp16)k0[k];

  422.                     k0 += kernel_w;

  423.                     g00 += 1;

  424.                 }

  425.             }

  426.         }

  427.         for (; p + 1 < inh; p += 2)

  428.         {

  429.             for (int k = 0; k < kernel_w; k++)

  430.             {

  431.                 const float* k0 = kptr + p * kernel_w;

  432. 

  433.                 for (int i = 0; i < 2; i++)

  434.                 {

  435.                     g00[0] = (__fp16)k0[k];

  436.                     k0 += kernel_w;

  437.                     g00 += 1;

  438.                 }

  439.             }

  440.         }

  441.         for (; p < inh; p++)

  442.         {

  443.             const float* k0 = kptr + p * kernel_w;

  444. 

  445.             for (int k = 0; k < kernel_w; k++)

  446.             {

  447.                 g00[0] = (__fp16)k0[k];

  448.                 g00++;

  449.             }

  450.         }

  451.     }

  452. }

  453. 

  454. static void convolution1d_packed_fp16s(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_tm, const Mat& bias_data, int kernel_w, int dilation_w, int stride_w, int activation_type, const Mat& activation_params, const Option& opt)

  455. {

  456.     const int elempack = bottom_blob.elempack;

  457.     const int inh = bottom_blob.h * elempack;

  458. 

  459.     const int N = bottom_blob.w * elempack;

  460. 

  461.     const int outw = top_blob.w;

  462.     const int out_elempack = top_blob.elempack;

  463.     const int outh = top_blob.h * out_elempack;

  464. 

  465.     const int M = top_blob.w * out_elempack;

  466. 

  467.     const float* bias_data_ptr = bias_data;

  468. 

  469.     int nn_outh = 0;

  470.     int remain_outh_start = 0;

  471.     nn_outh = (outh - remain_outh_start) / 8;

  472.     #pragma omp parallel for num_threads(opt.num_threads)

  473.     for (int pp = 0; pp < nn_outh; pp++)

  474.     {

  475.         const int p = remain_outh_start + pp * 8;

  476. 

  477.         // shadowed variable for less openmp task args

  478.         const int elempack = bottom_blob.elempack;

  479.         const int inh = bottom_blob.h * elempack;

  480.         const int outw = top_blob.w;

  481.         const int out_elempack = top_blob.elempack;

  482. 

  483.         __fp16* outptr = top_blob.row<__fp16>(p / out_elempack);

  484. 

  485.         for (int j = 0; j < outw; j++)

  486.         {

  487.             float32x4_t _sum0 = vdupq_n_f32(0.f);

  488.             float32x4_t _sum1 = vdupq_n_f32(0.f);

  489.             float32x4_t _sum2 = vdupq_n_f32(0.f);

  490.             float32x4_t _sum3 = vdupq_n_f32(0.f);

  491.             float32x4_t _sum4 = vdupq_n_f32(0.f);

  492.             float32x4_t _sum5 = vdupq_n_f32(0.f);

  493.             float32x4_t _sum6 = vdupq_n_f32(0.f);

  494.             float32x4_t _sum7 = vdupq_n_f32(0.f);

  495. 

  496.             if (bias_data_ptr)

  497.             {

  498.                 _sum0 = vld1q_f32(bias_data_ptr + p);

  499.                 _sum1 = vld1q_f32(bias_data_ptr + p + 4);

  500.             }

  501. 

  502.             const __fp16* kptr = weight_data_tm.channel(p / 8);

  503. 

  504.             int q = 0;

  505.             for (; q + 7 < inh; q += 8)

  506.             {

  507.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  508. 

  509.                 for (int k = 0; k < kernel_w; k++)

  510.                 {

  511.                     float32x4_t _r0;

  512.                     float32x4_t _r1;

  513.                     if (elempack == 4)

  514.                     {

  515.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  516.                         _r1 = vcvt_f32_f16(vld1_f16(r0 + N));

  517.                         r0 += dilation_w * 4;

  518.                     }

  519.                     else // if (elempack == 1)

  520.                     {

  521.                         float16x8_t _r_f16 = float16x8_t();

  522.                         _r_f16 = vsetq_lane_f16(r0[0], _r_f16, 0);

  523.                         _r_f16 = vsetq_lane_f16(r0[N], _r_f16, 1);

  524.                         _r_f16 = vsetq_lane_f16(r0[N * 2], _r_f16, 2);

  525.                         _r_f16 = vsetq_lane_f16(r0[N * 3], _r_f16, 3);

  526.                         _r_f16 = vsetq_lane_f16(r0[N * 4], _r_f16, 4);

  527.                         _r_f16 = vsetq_lane_f16(r0[N * 5], _r_f16, 5);

  528.                         _r_f16 = vsetq_lane_f16(r0[N * 6], _r_f16, 6);

  529.                         _r_f16 = vsetq_lane_f16(r0[N * 7], _r_f16, 7);

  530.                         _r0 = vcvt_f32_f16(vget_low_f16(_r_f16));

  531.                         _r1 = vcvt_f32_f16(vget_high_f16(_r_f16));

  532.                         r0 += dilation_w;

  533.                     }

  534. 

  535.                     float16x8_t _w01 = vld1q_f16(kptr);

  536.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  537.                     float16x8_t _w45 = vld1q_f16(kptr + 16);

  538.                     float16x8_t _w67 = vld1q_f16(kptr + 24);

  539.                     float16x8_t _w89 = vld1q_f16(kptr + 32);

  540.                     float16x8_t _wab = vld1q_f16(kptr + 40);

  541.                     float16x8_t _wcd = vld1q_f16(kptr + 48);

  542.                     float16x8_t _wef = vld1q_f16(kptr + 56);

  543.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  544.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  545.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  546.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  547.                     float32x4_t _w4 = vcvt_f32_f16(vget_low_f16(_w45));

  548.                     float32x4_t _w5 = vcvt_f32_f16(vget_high_f16(_w45));

  549.                     float32x4_t _w6 = vcvt_f32_f16(vget_low_f16(_w67));

  550.                     float32x4_t _w7 = vcvt_f32_f16(vget_high_f16(_w67));

  551.                     float32x4_t _w8 = vcvt_f32_f16(vget_low_f16(_w89));

  552.                     float32x4_t _w9 = vcvt_f32_f16(vget_high_f16(_w89));

  553.                     float32x4_t _wa = vcvt_f32_f16(vget_low_f16(_wab));

  554.                     float32x4_t _wb = vcvt_f32_f16(vget_high_f16(_wab));

  555.                     float32x4_t _wc = vcvt_f32_f16(vget_low_f16(_wcd));

  556.                     float32x4_t _wd = vcvt_f32_f16(vget_high_f16(_wcd));

  557.                     float32x4_t _we = vcvt_f32_f16(vget_low_f16(_wef));

  558.                     float32x4_t _wf = vcvt_f32_f16(vget_high_f16(_wef));

  559.                     _sum0 = vfmaq_laneq_f32(_sum0, _w0, _r0, 0);

  560.                     _sum1 = vfmaq_laneq_f32(_sum1, _w1, _r0, 0);

  561.                     _sum2 = vfmaq_laneq_f32(_sum2, _w2, _r0, 1);

  562.                     _sum3 = vfmaq_laneq_f32(_sum3, _w3, _r0, 1);

  563.                     _sum4 = vfmaq_laneq_f32(_sum4, _w4, _r0, 2);

  564.                     _sum5 = vfmaq_laneq_f32(_sum5, _w5, _r0, 2);

  565.                     _sum6 = vfmaq_laneq_f32(_sum6, _w6, _r0, 3);

  566.                     _sum7 = vfmaq_laneq_f32(_sum7, _w7, _r0, 3);

  567.                     _sum0 = vfmaq_laneq_f32(_sum0, _w8, _r1, 0);

  568.                     _sum1 = vfmaq_laneq_f32(_sum1, _w9, _r1, 0);

  569.                     _sum2 = vfmaq_laneq_f32(_sum2, _wa, _r1, 1);

  570.                     _sum3 = vfmaq_laneq_f32(_sum3, _wb, _r1, 1);

  571.                     _sum4 = vfmaq_laneq_f32(_sum4, _wc, _r1, 2);

  572.                     _sum5 = vfmaq_laneq_f32(_sum5, _wd, _r1, 2);

  573.                     _sum6 = vfmaq_laneq_f32(_sum6, _we, _r1, 3);

  574.                     _sum7 = vfmaq_laneq_f32(_sum7, _wf, _r1, 3);

  575. 

  576.                     kptr += 64;

  577.                 }

  578.             }

  579.             for (; q + 3 < inh; q += 4)

  580.             {

  581.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  582. 

  583.                 for (int k = 0; k < kernel_w; k++)

  584.                 {

  585.                     float32x4_t _r0;

  586.                     if (elempack == 4)

  587.                     {

  588.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  589.                         r0 += dilation_w * 4;

  590.                     }

  591.                     else // if (elempack == 1)

  592.                     {

  593.                         float16x4_t _r_f16 = float16x4_t();

  594.                         _r_f16 = vset_lane_f16(r0[0], _r_f16, 0);

  595.                         _r_f16 = vset_lane_f16(r0[N], _r_f16, 1);

  596.                         _r_f16 = vset_lane_f16(r0[N * 2], _r_f16, 2);

  597.                         _r_f16 = vset_lane_f16(r0[N * 3], _r_f16, 3);

  598.                         _r0 = vcvt_f32_f16(_r_f16);

  599.                         r0 += dilation_w;

  600.                     }

  601. 

  602.                     float16x8_t _w01 = vld1q_f16(kptr);

  603.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  604.                     float16x8_t _w45 = vld1q_f16(kptr + 16);

  605.                     float16x8_t _w67 = vld1q_f16(kptr + 24);

  606.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  607.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  608.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  609.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  610.                     float32x4_t _w4 = vcvt_f32_f16(vget_low_f16(_w45));

  611.                     float32x4_t _w5 = vcvt_f32_f16(vget_high_f16(_w45));

  612.                     float32x4_t _w6 = vcvt_f32_f16(vget_low_f16(_w67));

  613.                     float32x4_t _w7 = vcvt_f32_f16(vget_high_f16(_w67));

  614.                     _sum0 = vfmaq_laneq_f32(_sum0, _w0, _r0, 0);

  615.                     _sum1 = vfmaq_laneq_f32(_sum1, _w1, _r0, 0);

  616.                     _sum2 = vfmaq_laneq_f32(_sum2, _w2, _r0, 1);

  617.                     _sum3 = vfmaq_laneq_f32(_sum3, _w3, _r0, 1);

  618.                     _sum4 = vfmaq_laneq_f32(_sum4, _w4, _r0, 2);

  619.                     _sum5 = vfmaq_laneq_f32(_sum5, _w5, _r0, 2);

  620.                     _sum6 = vfmaq_laneq_f32(_sum6, _w6, _r0, 3);

  621.                     _sum7 = vfmaq_laneq_f32(_sum7, _w7, _r0, 3);

  622. 

  623.                     kptr += 32;

  624.                 }

  625.             }

  626.             for (; q + 1 < inh; q += 2)

  627.             {

  628.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  629. 

  630.                 for (int k = 0; k < kernel_w; k++)

  631.                 {

  632.                     float val0;

  633.                     float val1;

  634.                     // if (elempack == 1)

  635.                     {

  636.                         val0 = (float)(r0[0]);

  637.                         val1 = (float)(r0[N]);

  638.                         r0 += dilation_w;

  639.                     }

  640. 

  641.                     float16x8_t _w01 = vld1q_f16(kptr);

  642.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  643.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  644.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  645.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  646.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  647.                     _sum0 = vfmaq_n_f32(_sum0, _w0, val0);

  648.                     _sum1 = vfmaq_n_f32(_sum1, _w1, val0);

  649.                     _sum2 = vfmaq_n_f32(_sum2, _w2, val1);

  650.                     _sum3 = vfmaq_n_f32(_sum3, _w3, val1);

  651. 

  652.                     kptr += 16;

  653.                 }

  654.             }

  655.             for (; q < inh; q++)

  656.             {

  657.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  658. 

  659.                 for (int k = 0; k < kernel_w; k++)

  660.                 {

  661.                     float32x4_t _val;

  662.                     // if (elempack == 1)

  663.                     {

  664.                         _val = vcvt_f32_f16(vdup_n_f16(r0[0]));

  665.                         r0 += dilation_w;

  666.                     }

  667. 

  668.                     float16x8_t _w = vld1q_f16(kptr);

  669.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w));

  670.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w));

  671.                     _sum0 = vfmaq_f32(_sum0, _w0, _val);

  672.                     _sum1 = vfmaq_f32(_sum1, _w1, _val);

  673. 

  674.                     kptr += 8;

  675.                 }

  676.             }

  677. 

  678.             _sum0 = vaddq_f32(_sum0, _sum2);

  679.             _sum1 = vaddq_f32(_sum1, _sum3);

  680.             _sum4 = vaddq_f32(_sum4, _sum6);

  681.             _sum5 = vaddq_f32(_sum5, _sum7);

  682.             _sum0 = vaddq_f32(_sum0, _sum4);

  683.             _sum1 = vaddq_f32(_sum1, _sum5);

  684. 

  685.             _sum0 = activation_ps(_sum0, activation_type, activation_params);

  686.             _sum1 = activation_ps(_sum1, activation_type, activation_params);

  687. 

  688.             if (out_elempack == 4)

  689.             {

  690.                 vst1_f16(outptr, vcvt_f16_f32(_sum0));

  691.                 vst1_f16(outptr + M, vcvt_f16_f32(_sum1));

  692.                 outptr += 4;

  693.             }

  694.             else // if (out_elempack == 1)

  695.             {

  696.                 float16x4_t _sum0_f16 = vcvt_f16_f32(_sum0);

  697.                 float16x4_t _sum1_f16 = vcvt_f16_f32(_sum1);

  698.                 outptr[0] = vget_lane_f16(_sum0_f16, 0);

  699.                 outptr[M] = vget_lane_f16(_sum0_f16, 1);

  700.                 outptr[M * 2] = vget_lane_f16(_sum0_f16, 2);

  701.                 outptr[M * 3] = vget_lane_f16(_sum0_f16, 3);

  702.                 outptr[M * 4] = vget_lane_f16(_sum1_f16, 0);

  703.                 outptr[M * 5] = vget_lane_f16(_sum1_f16, 1);

  704.                 outptr[M * 6] = vget_lane_f16(_sum1_f16, 2);

  705.                 outptr[M * 7] = vget_lane_f16(_sum1_f16, 3);

  706.                 outptr += 1;

  707.             }

  708.         }

  709.     }

  710.     remain_outh_start += nn_outh * 8;

  711.     nn_outh = (outh - remain_outh_start) / 4;

  712.     for (int pp = 0; pp < nn_outh; pp++)

  713.     {

  714.         const int p = remain_outh_start + pp * 4;

  715. 

  716.         // shadowed variable for less openmp task args

  717.         const int elempack = bottom_blob.elempack;

  718.         const int inh = bottom_blob.h * elempack;

  719.         const int outw = top_blob.w;

  720.         const int out_elempack = top_blob.elempack;

  721. 

  722.         __fp16* outptr = top_blob.row<__fp16>(p / out_elempack);

  723. 

  724.         for (int j = 0; j < outw; j++)

  725.         {

  726.             float32x4_t _sum0 = vdupq_n_f32(0.f);

  727.             float32x4_t _sum1 = vdupq_n_f32(0.f);

  728.             float32x4_t _sum2 = vdupq_n_f32(0.f);

  729.             float32x4_t _sum3 = vdupq_n_f32(0.f);

  730. 

  731.             if (bias_data_ptr)

  732.             {

  733.                 _sum0 = vld1q_f32(bias_data_ptr + p);

  734.             }

  735. 

  736.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4);

  737. 

  738.             int q = 0;

  739.             for (; q + 7 < inh; q += 8)

  740.             {

  741.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  742. 

  743.                 for (int k = 0; k < kernel_w; k++)

  744.                 {

  745.                     float32x4_t _r0;

  746.                     float32x4_t _r1;

  747.                     if (elempack == 4)

  748.                     {

  749.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  750.                         _r1 = vcvt_f32_f16(vld1_f16(r0 + N));

  751.                         r0 += dilation_w * 4;

  752.                     }

  753.                     else // if (elempack == 1)

  754.                     {

  755.                         float16x8_t _r_f16 = float16x8_t();

  756.                         _r_f16 = vsetq_lane_f16(r0[0], _r_f16, 0);

  757.                         _r_f16 = vsetq_lane_f16(r0[N], _r_f16, 1);

  758.                         _r_f16 = vsetq_lane_f16(r0[N * 2], _r_f16, 2);

  759.                         _r_f16 = vsetq_lane_f16(r0[N * 3], _r_f16, 3);

  760.                         _r_f16 = vsetq_lane_f16(r0[N * 4], _r_f16, 4);

  761.                         _r_f16 = vsetq_lane_f16(r0[N * 5], _r_f16, 5);

  762.                         _r_f16 = vsetq_lane_f16(r0[N * 6], _r_f16, 6);

  763.                         _r_f16 = vsetq_lane_f16(r0[N * 7], _r_f16, 7);

  764.                         _r0 = vcvt_f32_f16(vget_low_f16(_r_f16));

  765.                         _r1 = vcvt_f32_f16(vget_high_f16(_r_f16));

  766.                         r0 += dilation_w;

  767.                     }

  768. 

  769.                     float16x8_t _w01 = vld1q_f16(kptr);

  770.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  771.                     float16x8_t _w45 = vld1q_f16(kptr + 16);

  772.                     float16x8_t _w67 = vld1q_f16(kptr + 24);

  773.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  774.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  775.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  776.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  777.                     float32x4_t _w4 = vcvt_f32_f16(vget_low_f16(_w45));

  778.                     float32x4_t _w5 = vcvt_f32_f16(vget_high_f16(_w45));

  779.                     float32x4_t _w6 = vcvt_f32_f16(vget_low_f16(_w67));

  780.                     float32x4_t _w7 = vcvt_f32_f16(vget_high_f16(_w67));

  781.                     _sum0 = vfmaq_laneq_f32(_sum0, _w0, _r0, 0);

  782.                     _sum1 = vfmaq_laneq_f32(_sum1, _w1, _r0, 1);

  783.                     _sum2 = vfmaq_laneq_f32(_sum2, _w2, _r0, 2);

  784.                     _sum3 = vfmaq_laneq_f32(_sum3, _w3, _r0, 3);

  785.                     _sum0 = vfmaq_laneq_f32(_sum0, _w4, _r1, 0);

  786.                     _sum1 = vfmaq_laneq_f32(_sum1, _w5, _r1, 1);

  787.                     _sum2 = vfmaq_laneq_f32(_sum2, _w6, _r1, 2);

  788.                     _sum3 = vfmaq_laneq_f32(_sum3, _w7, _r1, 3);

  789. 

  790.                     kptr += 32;

  791.                 }

  792.             }

  793.             for (; q + 3 < inh; q += 4)

  794.             {

  795.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  796. 

  797.                 for (int k = 0; k < kernel_w; k++)

  798.                 {

  799.                     float32x4_t _r0;

  800.                     if (elempack == 4)

  801.                     {

  802.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  803.                         r0 += dilation_w * 4;

  804.                     }

  805.                     else // if (elempack == 1)

  806.                     {

  807.                         float16x4_t _r_f16 = float16x4_t();

  808.                         _r_f16 = vset_lane_f16(r0[0], _r_f16, 0);

  809.                         _r_f16 = vset_lane_f16(r0[N], _r_f16, 1);

  810.                         _r_f16 = vset_lane_f16(r0[N * 2], _r_f16, 2);

  811.                         _r_f16 = vset_lane_f16(r0[N * 3], _r_f16, 3);

  812.                         _r0 = vcvt_f32_f16(_r_f16);

  813.                         r0 += dilation_w;

  814.                     }

  815. 

  816.                     float16x8_t _w01 = vld1q_f16(kptr);

  817.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  818.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  819.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  820.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  821.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  822.                     _sum0 = vfmaq_laneq_f32(_sum0, _w0, _r0, 0);

  823.                     _sum1 = vfmaq_laneq_f32(_sum1, _w1, _r0, 1);

  824.                     _sum2 = vfmaq_laneq_f32(_sum2, _w2, _r0, 2);

  825.                     _sum3 = vfmaq_laneq_f32(_sum3, _w3, _r0, 3);

  826. 

  827.                     kptr += 16;

  828.                 }

  829.             }

  830.             for (; q + 1 < inh; q += 2)

  831.             {

  832.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  833. 

  834.                 for (int k = 0; k < kernel_w; k++)

  835.                 {

  836.                     float val0;

  837.                     float val1;

  838.                     // if (elempack == 1)

  839.                     {

  840.                         val0 = (float)(r0[0]);

  841.                         val1 = (float)(r0[N]);

  842.                         r0 += dilation_w;

  843.                     }

  844. 

  845.                     float16x8_t _w = vld1q_f16(kptr);

  846.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w));

  847.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w));

  848.                     _sum0 = vfmaq_n_f32(_sum0, _w0, val0);

  849.                     _sum1 = vfmaq_n_f32(_sum1, _w1, val1);

  850. 

  851.                     kptr += 8;

  852.                 }

  853.             }

  854.             for (; q < inh; q++)

  855.             {

  856.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  857. 

  858.                 for (int k = 0; k < kernel_w; k++)

  859.                 {

  860.                     float32x4_t _val;

  861.                     // if (elempack == 1)

  862.                     {

  863.                         _val = vcvt_f32_f16(vdup_n_f16(r0[0]));

  864.                         r0 += dilation_w;

  865.                     }

  866. 

  867.                     float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));

  868.                     _sum0 = vfmaq_f32(_sum0, _val, _w);

  869. 

  870.                     kptr += 4;

  871.                 }

  872.             }

  873. 

  874.             _sum0 = vaddq_f32(_sum0, _sum1);

  875.             _sum2 = vaddq_f32(_sum2, _sum3);

  876.             _sum0 = vaddq_f32(_sum0, _sum2);

  877. 

  878.             _sum0 = activation_ps(_sum0, activation_type, activation_params);

  879. 

  880.             if (out_elempack == 4)

  881.             {

  882.                 vst1_f16(outptr, vcvt_f16_f32(_sum0));

  883.                 outptr += 4;

  884.             }

  885.             else // if (out_elempack == 1)

  886.             {

  887.                 float16x4_t _sum0_f16 = vcvt_f16_f32(_sum0);

  888.                 outptr[0] = vget_lane_f16(_sum0_f16, 0);

  889.                 outptr[M] = vget_lane_f16(_sum0_f16, 1);

  890.                 outptr[M * 2] = vget_lane_f16(_sum0_f16, 2);

  891.                 outptr[M * 3] = vget_lane_f16(_sum0_f16, 3);

  892.                 outptr += 1;

  893.             }

  894.         }

  895.     }

  896.     remain_outh_start += nn_outh * 4;

  897.     nn_outh = (outh - remain_outh_start) / 2;

  898.     for (int pp = 0; pp < nn_outh; pp++)

  899.     {

  900.         const int p = remain_outh_start + pp * 2;

  901. 

  902.         // shadowed variable for less openmp task args

  903.         const int elempack = bottom_blob.elempack;

  904.         const int inh = bottom_blob.h * elempack;

  905.         const int outw = top_blob.w;

  906. 

  907.         __fp16* outptr0 = top_blob.row<__fp16>(p);

  908.         __fp16* outptr1 = top_blob.row<__fp16>(p + 1);

  909. 

  910.         for (int j = 0; j < outw; j++)

  911.         {

  912.             float sum0 = 0.f;

  913.             float sum1 = 0.f;

  914. 

  915.             if (bias_data_ptr)

  916.             {

  917.                 sum0 = bias_data_ptr[p];

  918.                 sum1 = bias_data_ptr[p + 1];

  919.             }

  920. 

  921.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4 + (p % 4) / 2);

  922. 

  923.             int q = 0;

  924.             float32x4_t _sum0 = vdupq_n_f32(0.f);

  925.             float32x4_t _sum1 = vdupq_n_f32(0.f);

  926.             float32x4_t _sum2 = vdupq_n_f32(0.f);

  927.             float32x4_t _sum3 = vdupq_n_f32(0.f);

  928.             for (; q + 7 < inh; q += 8)

  929.             {

  930.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  931. 

  932.                 for (int k = 0; k < kernel_w; k++)

  933.                 {

  934.                     float32x4_t _r0;

  935.                     float32x4_t _r1;

  936.                     if (elempack == 4)

  937.                     {

  938.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  939.                         _r1 = vcvt_f32_f16(vld1_f16(r0 + N));

  940.                         r0 += dilation_w * 4;

  941.                     }

  942.                     else // if (elempack == 1)

  943.                     {

  944.                         float16x8_t _r01_f16 = float16x8_t();

  945.                         _r01_f16 = vsetq_lane_f16(r0[0], _r01_f16, 0);

  946.                         _r01_f16 = vsetq_lane_f16(r0[N], _r01_f16, 1);

  947.                         _r01_f16 = vsetq_lane_f16(r0[N * 2], _r01_f16, 2);

  948.                         _r01_f16 = vsetq_lane_f16(r0[N * 3], _r01_f16, 3);

  949.                         _r01_f16 = vsetq_lane_f16(r0[N * 4], _r01_f16, 4);

  950.                         _r01_f16 = vsetq_lane_f16(r0[N * 5], _r01_f16, 5);

  951.                         _r01_f16 = vsetq_lane_f16(r0[N * 6], _r01_f16, 6);

  952.                         _r01_f16 = vsetq_lane_f16(r0[N * 7], _r01_f16, 7);

  953.                         _r0 = vcvt_f32_f16(vget_low_f16(_r01_f16));

  954.                         _r1 = vcvt_f32_f16(vget_high_f16(_r01_f16));

  955.                         r0 += dilation_w;

  956.                     }

  957. 

  958.                     float16x8_t _w01 = vld1q_f16(kptr);

  959.                     float16x8_t _w23 = vld1q_f16(kptr + 8);

  960.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));

  961.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));

  962.                     float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));

  963.                     float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));

  964.                     _sum0 = vfmaq_f32(_sum0, _r0, _w0);

  965.                     _sum1 = vfmaq_f32(_sum1, _r1, _w1);

  966.                     _sum2 = vfmaq_f32(_sum2, _r0, _w2);

  967.                     _sum3 = vfmaq_f32(_sum3, _r1, _w3);

  968. 

  969.                     kptr += 16;

  970.                 }

  971.             }

  972.             _sum0 = vaddq_f32(_sum0, _sum1);

  973.             _sum2 = vaddq_f32(_sum2, _sum3);

  974.             sum0 += vaddvq_f32(_sum0);

  975.             sum1 += vaddvq_f32(_sum2);

  976.             _sum0 = vdupq_n_f32(0.f);

  977.             _sum1 = vdupq_n_f32(0.f);

  978.             for (; q + 3 < inh; q += 4)

  979.             {

  980.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  981. 

  982.                 for (int k = 0; k < kernel_w; k++)

  983.                 {

  984.                     float32x4_t _r0;

  985.                     if (elempack == 4)

  986.                     {

  987.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  988.                         r0 += dilation_w * 4;

  989.                     }

  990.                     else // if (elempack == 1)

  991.                     {

  992.                         float16x4_t _r0_f16 = float16x4_t();

  993.                         _r0_f16 = vset_lane_f16(r0[0], _r0_f16, 0);

  994.                         _r0_f16 = vset_lane_f16(r0[N], _r0_f16, 1);

  995.                         _r0_f16 = vset_lane_f16(r0[N * 2], _r0_f16, 2);

  996.                         _r0_f16 = vset_lane_f16(r0[N * 3], _r0_f16, 3);

  997.                         _r0 = vcvt_f32_f16(_r0_f16);

  998.                         r0 += dilation_w;

  999.                     }

  1000. 

  1001.                     float16x8_t _w = vld1q_f16(kptr);

  1002.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w));

  1003.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w));

  1004.                     _sum0 = vfmaq_f32(_sum0, _r0, _w0);

  1005.                     _sum1 = vfmaq_f32(_sum1, _r0, _w1);

  1006. 

  1007.                     kptr += 8;

  1008.                 }

  1009.             }

  1010.             sum0 += vaddvq_f32(_sum0);

  1011.             sum1 += vaddvq_f32(_sum1);

  1012.             for (; q + 1 < inh; q += 2)

  1013.             {

  1014.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1015. 

  1016.                 for (int k = 0; k < kernel_w; k++)

  1017.                 {

  1018.                     float val0;

  1019.                     float val1;

  1020.                     // if (elempack == 1)

  1021.                     {

  1022.                         val0 = (float)(r0[0]);

  1023.                         val1 = (float)(r0[N]);

  1024.                         r0 += dilation_w;

  1025.                     }

  1026. 

  1027.                     sum0 += val0 * (float)(kptr[0]);

  1028.                     sum1 += val0 * (float)(kptr[1]);

  1029.                     sum0 += val1 * (float)(kptr[2]);

  1030.                     sum1 += val1 * (float)(kptr[3]);

  1031. 

  1032.                     kptr += 4;

  1033.                 }

  1034.             }

  1035.             for (; q < inh; q++)

  1036.             {

  1037.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1038. 

  1039.                 for (int k = 0; k < kernel_w; k++)

  1040.                 {

  1041.                     float val;

  1042.                     // if (elempack == 1)

  1043.                     {

  1044.                         val = (float)(r0[0]);

  1045.                         r0 += dilation_w;

  1046.                     }

  1047. 

  1048.                     sum0 += val * (float)(kptr[0]);

  1049.                     sum1 += val * (float)(kptr[1]);

  1050. 

  1051.                     kptr += 2;

  1052.                 }

  1053.             }

  1054. 

  1055.             sum0 = activation_ss(sum0, activation_type, activation_params);

  1056.             sum1 = activation_ss(sum1, activation_type, activation_params);

  1057. 

  1058.             outptr0[0] = (__fp16)(sum0);

  1059.             outptr1[0] = (__fp16)(sum1);

  1060.             outptr0 += 1;

  1061.             outptr1 += 1;

  1062.         }

  1063.     }

  1064.     remain_outh_start += nn_outh * 2;

  1065.     for (int p = remain_outh_start; p < outh; p++)

  1066.     {

  1067.         __fp16* outptr = top_blob.row<__fp16>(p);

  1068. 

  1069.         for (int j = 0; j < outw; j++)

  1070.         {

  1071.             float sum = 0.f;

  1072. 

  1073.             if (bias_data_ptr)

  1074.             {

  1075.                 sum = bias_data_ptr[p];

  1076.             }

  1077. 

  1078.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4 + (p % 4) / 2 + p % 2);

  1079. 

  1080.             int q = 0;

  1081.             float32x4_t _sum0 = vdupq_n_f32(0.f);

  1082.             float32x4_t _sum1 = vdupq_n_f32(0.f);

  1083.             for (; q + 7 < inh; q += 8)

  1084.             {

  1085.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1086. 

  1087.                 for (int k = 0; k < kernel_w; k++)

  1088.                 {

  1089.                     float32x4_t _r0;

  1090.                     float32x4_t _r1;

  1091.                     if (elempack == 4)

  1092.                     {

  1093.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  1094.                         _r1 = vcvt_f32_f16(vld1_f16(r0 + N));

  1095.                         r0 += dilation_w * 4;

  1096.                     }

  1097.                     else // if (elempack == 1)

  1098.                     {

  1099.                         float16x8_t _r01_f16 = float16x8_t();

  1100.                         _r01_f16 = vsetq_lane_f16(r0[0], _r01_f16, 0);

  1101.                         _r01_f16 = vsetq_lane_f16(r0[N], _r01_f16, 1);

  1102.                         _r01_f16 = vsetq_lane_f16(r0[N * 2], _r01_f16, 2);

  1103.                         _r01_f16 = vsetq_lane_f16(r0[N * 3], _r01_f16, 3);

  1104.                         _r01_f16 = vsetq_lane_f16(r0[N * 4], _r01_f16, 4);

  1105.                         _r01_f16 = vsetq_lane_f16(r0[N * 5], _r01_f16, 5);

  1106.                         _r01_f16 = vsetq_lane_f16(r0[N * 6], _r01_f16, 6);

  1107.                         _r01_f16 = vsetq_lane_f16(r0[N * 7], _r01_f16, 7);

  1108.                         _r0 = vcvt_f32_f16(vget_low_f16(_r01_f16));

  1109.                         _r1 = vcvt_f32_f16(vget_high_f16(_r01_f16));

  1110.                         r0 += dilation_w;

  1111.                     }

  1112. 

  1113.                     float16x8_t _w = vld1q_f16(kptr);

  1114.                     float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w));

  1115.                     float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w));

  1116.                     _sum0 = vfmaq_f32(_sum0, _r0, _w0);

  1117.                     _sum1 = vfmaq_f32(_sum1, _r1, _w1);

  1118. 

  1119.                     kptr += 8;

  1120.                 }

  1121.             }

  1122.             _sum0 = vaddq_f32(_sum0, _sum1);

  1123.             sum += vaddvq_f32(_sum0);

  1124.             float32x4_t _sum = vdupq_n_f32(0.f);

  1125.             for (; q + 3 < inh; q += 4)

  1126.             {

  1127.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1128. 

  1129.                 for (int k = 0; k < kernel_w; k++)

  1130.                 {

  1131.                     float32x4_t _r0;

  1132.                     if (elempack == 4)

  1133.                     {

  1134.                         _r0 = vcvt_f32_f16(vld1_f16(r0));

  1135.                         r0 += dilation_w * 4;

  1136.                     }

  1137.                     else // if (elempack == 1)

  1138.                     {

  1139.                         float16x4_t _r0_f16 = float16x4_t();

  1140.                         _r0_f16 = vset_lane_f16(r0[0], _r0_f16, 0);

  1141.                         _r0_f16 = vset_lane_f16(r0[N], _r0_f16, 1);

  1142.                         _r0_f16 = vset_lane_f16(r0[N * 2], _r0_f16, 2);

  1143.                         _r0_f16 = vset_lane_f16(r0[N * 3], _r0_f16, 3);

  1144.                         _r0 = vcvt_f32_f16(_r0_f16);

  1145.                         r0 += dilation_w;

  1146.                     }

  1147. 

  1148.                     float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));

  1149.                     _sum = vfmaq_f32(_sum, _r0, _w);

  1150. 

  1151.                     kptr += 4;

  1152.                 }

  1153.             }

  1154.             sum += vaddvq_f32(_sum);

  1155.             for (; q + 1 < inh; q += 2)

  1156.             {

  1157.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1158. 

  1159.                 for (int k = 0; k < kernel_w; k++)

  1160.                 {

  1161.                     float val0;

  1162.                     float val1;

  1163.                     // if (elempack == 1)

  1164.                     {

  1165.                         val0 = (float)(r0[0]);

  1166.                         val1 = (float)(r0[N]);

  1167.                         r0 += dilation_w;

  1168.                     }

  1169. 

  1170.                     sum += val0 * (float)(kptr[0]);

  1171.                     sum += val1 * (float)(kptr[1]);

  1172. 

  1173.                     kptr += 2;

  1174.                 }

  1175.             }

  1176.             for (; q < inh; q++)

  1177.             {

  1178.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1179. 

  1180.                 for (int k = 0; k < kernel_w; k++)

  1181.                 {

  1182.                     float val;

  1183.                     // if (elempack == 1)

  1184.                     {

  1185.                         val = (float)(r0[0]);

  1186.                         r0 += dilation_w;

  1187.                     }

  1188. 

  1189.                     sum += val * (float)(kptr[0]);

  1190. 

  1191.                     kptr += 1;

  1192.                 }

  1193.             }

  1194. 

  1195.             sum = activation_ss(sum, activation_type, activation_params);

  1196. 

  1197.             outptr[0] = (__fp16)(sum);

  1198.             outptr += 1;

  1199.         }

  1200.     }

  1201. }

  1202. 

  1203. static void convolution1d_packed_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_tm, const Mat& bias_data, int kernel_w, int dilation_w, int stride_w, int activation_type, const Mat& activation_params, const Option& opt)

  1204. {

  1205.     const int elempack = bottom_blob.elempack;

  1206.     const int inh = bottom_blob.h * elempack;

  1207. 

  1208.     const int N = bottom_blob.w * elempack;

  1209. 

  1210.     const int outw = top_blob.w;

  1211.     const int out_elempack = top_blob.elempack;

  1212.     const int outh = top_blob.h * out_elempack;

  1213. 

  1214.     const int M = top_blob.w * out_elempack;

  1215. 

  1216.     const __fp16* bias_data_ptr = bias_data;

  1217. 

  1218.     int nn_outh = 0;

  1219.     int remain_outh_start = 0;

  1220.     nn_outh = (outh - remain_outh_start) / 8;

  1221.     #pragma omp parallel for num_threads(opt.num_threads)

  1222.     for (int pp = 0; pp < nn_outh; pp++)

  1223.     {

  1224.         const int p = remain_outh_start + pp * 8;

  1225. 

  1226.         // shadowed variable for less openmp task args

  1227.         const int elempack = bottom_blob.elempack;

  1228.         const int inh = bottom_blob.h * elempack;

  1229.         const int outw = top_blob.w;

  1230.         const int out_elempack = top_blob.elempack;

  1231. 

  1232.         __fp16* outptr = top_blob.row<__fp16>(p / out_elempack);

  1233. 

  1234.         for (int j = 0; j < outw; j++)

  1235.         {

  1236.             float16x8_t _sum0 = vdupq_n_f16(0.f);

  1237.             float16x8_t _sum1 = vdupq_n_f16(0.f);

  1238.             float16x8_t _sum2 = vdupq_n_f16(0.f);

  1239.             float16x8_t _sum3 = vdupq_n_f16(0.f);

  1240. 

  1241.             if (bias_data_ptr)

  1242.             {

  1243.                 _sum0 = vld1q_f16(bias_data_ptr + p);

  1244.             }

  1245. 

  1246.             const __fp16* kptr = weight_data_tm.channel(p / 8);

  1247. 

  1248.             int q = 0;

  1249.             for (; q + 7 < inh; q += 8)

  1250.             {

  1251.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1252. 

  1253.                 for (int k = 0; k < kernel_w; k++)

  1254.                 {

  1255.                     float16x8_t _r0;

  1256.                     if (elempack == 8)

  1257.                     {

  1258.                         _r0 = vld1q_f16(r0);

  1259.                         r0 += dilation_w * 8;

  1260.                     }

  1261.                     else if (elempack == 4)

  1262.                     {

  1263.                         _r0 = vcombine_f16(vld1_f16(r0), vld1_f16(r0 + N));

  1264.                         r0 += dilation_w * 4;

  1265.                     }

  1266.                     else // if (elempack == 1)

  1267.                     {

  1268.                         _r0 = float16x8_t();

  1269.                         _r0 = vsetq_lane_f16(r0[0], _r0, 0);

  1270.                         _r0 = vsetq_lane_f16(r0[N], _r0, 1);

  1271.                         _r0 = vsetq_lane_f16(r0[N * 2], _r0, 2);

  1272.                         _r0 = vsetq_lane_f16(r0[N * 3], _r0, 3);

  1273.                         _r0 = vsetq_lane_f16(r0[N * 4], _r0, 4);

  1274.                         _r0 = vsetq_lane_f16(r0[N * 5], _r0, 5);

  1275.                         _r0 = vsetq_lane_f16(r0[N * 6], _r0, 6);

  1276.                         _r0 = vsetq_lane_f16(r0[N * 7], _r0, 7);

  1277.                         r0 += dilation_w;

  1278.                     }

  1279. 

  1280.                     float16x8_t _w0 = vld1q_f16(kptr);

  1281.                     float16x8_t _w1 = vld1q_f16(kptr + 8);

  1282.                     float16x8_t _w2 = vld1q_f16(kptr + 8 * 2);

  1283.                     float16x8_t _w3 = vld1q_f16(kptr + 8 * 3);

  1284.                     float16x8_t _w4 = vld1q_f16(kptr + 8 * 4);

  1285.                     float16x8_t _w5 = vld1q_f16(kptr + 8 * 5);

  1286.                     float16x8_t _w6 = vld1q_f16(kptr + 8 * 6);

  1287.                     float16x8_t _w7 = vld1q_f16(kptr + 8 * 7);

  1288.                     _sum0 = vfmaq_laneq_f16(_sum0, _w0, _r0, 0);

  1289.                     _sum1 = vfmaq_laneq_f16(_sum1, _w1, _r0, 1);

  1290.                     _sum2 = vfmaq_laneq_f16(_sum2, _w2, _r0, 2);

  1291.                     _sum3 = vfmaq_laneq_f16(_sum3, _w3, _r0, 3);

  1292.                     _sum0 = vfmaq_laneq_f16(_sum0, _w4, _r0, 4);

  1293.                     _sum1 = vfmaq_laneq_f16(_sum1, _w5, _r0, 5);

  1294.                     _sum2 = vfmaq_laneq_f16(_sum2, _w6, _r0, 6);

  1295.                     _sum3 = vfmaq_laneq_f16(_sum3, _w7, _r0, 7);

  1296. 

  1297.                     kptr += 64;

  1298.                 }

  1299.             }

  1300.             for (; q + 3 < inh; q += 4)

  1301.             {

  1302.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1303. 

  1304.                 for (int k = 0; k < kernel_w; k++)

  1305.                 {

  1306.                     float16x4_t _r0;

  1307.                     if (elempack == 4)

  1308.                     {

  1309.                         _r0 = vld1_f16(r0);

  1310.                         r0 += dilation_w * 4;

  1311.                     }

  1312.                     else // if (elempack == 1)

  1313.                     {

  1314.                         _r0 = float16x4_t();

  1315.                         _r0 = vset_lane_f16(r0[0], _r0, 0);

  1316.                         _r0 = vset_lane_f16(r0[N], _r0, 1);

  1317.                         _r0 = vset_lane_f16(r0[N * 2], _r0, 2);

  1318.                         _r0 = vset_lane_f16(r0[N * 3], _r0, 3);

  1319.                         r0 += dilation_w;

  1320.                     }

  1321. 

  1322.                     float16x8_t _w0 = vld1q_f16(kptr);

  1323.                     float16x8_t _w1 = vld1q_f16(kptr + 8);

  1324.                     float16x8_t _w2 = vld1q_f16(kptr + 8 * 2);

  1325.                     float16x8_t _w3 = vld1q_f16(kptr + 8 * 3);

  1326.                     _sum0 = vfmaq_lane_f16(_sum0, _w0, _r0, 0);

  1327.                     _sum1 = vfmaq_lane_f16(_sum1, _w1, _r0, 1);

  1328.                     _sum2 = vfmaq_lane_f16(_sum2, _w2, _r0, 2);

  1329.                     _sum3 = vfmaq_lane_f16(_sum3, _w3, _r0, 3);

  1330. 

  1331.                     kptr += 32;

  1332.                 }

  1333.             }

  1334.             for (; q + 1 < inh; q += 2)

  1335.             {

  1336.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1337. 

  1338.                 for (int k = 0; k < kernel_w; k++)

  1339.                 {

  1340.                     __fp16 val0;

  1341.                     __fp16 val1;

  1342.                     // if (elempack == 1)

  1343.                     {

  1344.                         val0 = r0[0];

  1345.                         val1 = r0[N];

  1346.                         r0 += dilation_w;

  1347.                     }

  1348. 

  1349.                     float16x8_t _w0 = vld1q_f16(kptr);

  1350.                     float16x8_t _w1 = vld1q_f16(kptr + 8);

  1351.                     _sum0 = vfmaq_n_f16(_sum0, _w0, val0);

  1352.                     _sum1 = vfmaq_n_f16(_sum1, _w1, val1);

  1353. 

  1354.                     kptr += 16;

  1355.                 }

  1356.             }

  1357.             for (; q < inh; q++)

  1358.             {

  1359.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1360. 

  1361.                 for (int k = 0; k < kernel_w; k++)

  1362.                 {

  1363.                     float16x8_t _val;

  1364.                     // if (elempack == 1)

  1365.                     {

  1366.                         _val = vdupq_n_f16(r0[0]);

  1367.                         r0 += dilation_w;

  1368.                     }

  1369. 

  1370.                     float16x8_t _w0 = vld1q_f16(kptr);

  1371.                     _sum0 = vfmaq_f16(_sum0, _w0, _val);

  1372. 

  1373.                     kptr += 8;

  1374.                 }

  1375.             }

  1376. 

  1377.             _sum0 = vaddq_f16(_sum0, _sum1);

  1378.             _sum2 = vaddq_f16(_sum2, _sum3);

  1379.             _sum0 = vaddq_f16(_sum0, _sum2);

  1380. 

  1381.             _sum0 = activation_ps_f16(_sum0, activation_type, activation_params);

  1382. 

  1383.             if (out_elempack == 8)

  1384.             {

  1385.                 vst1q_f16(outptr, _sum0);

  1386.                 outptr += 8;

  1387.             }

  1388.             else if (out_elempack == 4)

  1389.             {

  1390.                 vst1_f16(outptr, vget_low_f16(_sum0));

  1391.                 vst1_f16(outptr + M, vget_high_f16(_sum0));

  1392.                 outptr += 4;

  1393.             }

  1394.             else // if (out_elempack == 1)

  1395.             {

  1396.                 outptr[0] = vgetq_lane_f16(_sum0, 0);

  1397.                 outptr[M] = vgetq_lane_f16(_sum0, 1);

  1398.                 outptr[M * 2] = vgetq_lane_f16(_sum0, 2);

  1399.                 outptr[M * 3] = vgetq_lane_f16(_sum0, 3);

  1400.                 outptr[M * 4] = vgetq_lane_f16(_sum0, 4);

  1401.                 outptr[M * 5] = vgetq_lane_f16(_sum0, 5);

  1402.                 outptr[M * 6] = vgetq_lane_f16(_sum0, 6);

  1403.                 outptr[M * 7] = vgetq_lane_f16(_sum0, 7);

  1404.                 outptr += 1;

  1405.             }

  1406.         }

  1407.     }

  1408.     remain_outh_start += nn_outh * 8;

  1409.     nn_outh = (outh - remain_outh_start) / 4;

  1410.     for (int pp = 0; pp < nn_outh; pp++)

  1411.     {

  1412.         const int p = remain_outh_start + pp * 4;

  1413. 

  1414.         // shadowed variable for less openmp task args

  1415.         const int elempack = bottom_blob.elempack;

  1416.         const int inh = bottom_blob.h * elempack;

  1417.         const int outw = top_blob.w;

  1418.         const int out_elempack = top_blob.elempack;

  1419. 

  1420.         __fp16* outptr = top_blob.row<__fp16>(p / out_elempack);

  1421. 

  1422.         for (int j = 0; j < outw; j++)

  1423.         {

  1424.             float16x4_t _sum0 = vdup_n_f16(0.f);

  1425.             float16x4_t _sum1 = vdup_n_f16(0.f);

  1426.             float16x4_t _sum2 = vdup_n_f16(0.f);

  1427.             float16x4_t _sum3 = vdup_n_f16(0.f);

  1428. 

  1429.             if (bias_data_ptr)

  1430.             {

  1431.                 _sum0 = vld1_f16(bias_data_ptr + p);

  1432.             }

  1433. 

  1434.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4);

  1435. 

  1436.             int q = 0;

  1437.             for (; q + 7 < inh; q += 8)

  1438.             {

  1439.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1440. 

  1441.                 for (int k = 0; k < kernel_w; k++)

  1442.                 {

  1443.                     float16x4_t _r0;

  1444.                     float16x4_t _r1;

  1445.                     if (elempack == 8)

  1446.                     {

  1447.                         float16x8_t _r01 = vld1q_f16(r0);

  1448.                         _r0 = vget_low_f16(_r01);

  1449.                         _r1 = vget_high_f16(_r01);

  1450.                         r0 += dilation_w * 8;

  1451.                     }

  1452.                     else if (elempack == 4)

  1453.                     {

  1454.                         _r0 = vld1_f16(r0);

  1455.                         _r1 = vld1_f16(r0 + N);

  1456.                         r0 += dilation_w * 4;

  1457.                     }

  1458.                     else // if (elempack == 1)

  1459.                     {

  1460.                         _r0 = float16x4_t();

  1461.                         _r1 = float16x4_t();

  1462.                         _r0 = vset_lane_f16(r0[0], _r0, 0);

  1463.                         _r0 = vset_lane_f16(r0[N], _r0, 1);

  1464.                         _r0 = vset_lane_f16(r0[N * 2], _r0, 2);

  1465.                         _r0 = vset_lane_f16(r0[N * 3], _r0, 3);

  1466.                         _r1 = vset_lane_f16(r0[N * 4], _r1, 0);

  1467.                         _r1 = vset_lane_f16(r0[N * 5], _r1, 1);

  1468.                         _r1 = vset_lane_f16(r0[N * 6], _r1, 2);

  1469.                         _r1 = vset_lane_f16(r0[N * 7], _r1, 3);

  1470.                         r0 += dilation_w;

  1471.                     }

  1472. 

  1473.                     float16x4_t _w0 = vld1_f16(kptr);

  1474.                     float16x4_t _w1 = vld1_f16(kptr + 4);

  1475.                     float16x4_t _w2 = vld1_f16(kptr + 8);

  1476.                     float16x4_t _w3 = vld1_f16(kptr + 12);

  1477.                     float16x4_t _w4 = vld1_f16(kptr + 16);

  1478.                     float16x4_t _w5 = vld1_f16(kptr + 20);

  1479.                     float16x4_t _w6 = vld1_f16(kptr + 24);

  1480.                     float16x4_t _w7 = vld1_f16(kptr + 28);

  1481.                     _sum0 = vfma_lane_f16(_sum0, _w0, _r0, 0);

  1482.                     _sum1 = vfma_lane_f16(_sum1, _w1, _r0, 1);

  1483.                     _sum2 = vfma_lane_f16(_sum2, _w2, _r0, 2);

  1484.                     _sum3 = vfma_lane_f16(_sum3, _w3, _r0, 3);

  1485.                     _sum0 = vfma_lane_f16(_sum0, _w4, _r1, 0);

  1486.                     _sum1 = vfma_lane_f16(_sum1, _w5, _r1, 1);

  1487.                     _sum2 = vfma_lane_f16(_sum2, _w6, _r1, 2);

  1488.                     _sum3 = vfma_lane_f16(_sum3, _w7, _r1, 3);

  1489. 

  1490.                     kptr += 32;

  1491.                 }

  1492.             }

  1493.             for (; q + 3 < inh; q += 4)

  1494.             {

  1495.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1496. 

  1497.                 for (int k = 0; k < kernel_w; k++)

  1498.                 {

  1499.                     float16x4_t _r0;

  1500.                     if (elempack == 4)

  1501.                     {

  1502.                         _r0 = vld1_f16(r0);

  1503.                         r0 += dilation_w * 4;

  1504.                     }

  1505.                     else // if (elempack == 1)

  1506.                     {

  1507.                         _r0 = float16x4_t();

  1508.                         _r0 = vset_lane_f16(r0[0], _r0, 0);

  1509.                         _r0 = vset_lane_f16(r0[N], _r0, 1);

  1510.                         _r0 = vset_lane_f16(r0[N * 2], _r0, 2);

  1511.                         _r0 = vset_lane_f16(r0[N * 3], _r0, 3);

  1512.                         r0 += dilation_w;

  1513.                     }

  1514. 

  1515.                     float16x4_t _w0 = vld1_f16(kptr);

  1516.                     float16x4_t _w1 = vld1_f16(kptr + 4);

  1517.                     float16x4_t _w2 = vld1_f16(kptr + 8);

  1518.                     float16x4_t _w3 = vld1_f16(kptr + 12);

  1519.                     _sum0 = vfma_lane_f16(_sum0, _w0, _r0, 0);

  1520.                     _sum1 = vfma_lane_f16(_sum1, _w1, _r0, 1);

  1521.                     _sum2 = vfma_lane_f16(_sum2, _w2, _r0, 2);

  1522.                     _sum3 = vfma_lane_f16(_sum3, _w3, _r0, 3);

  1523. 

  1524.                     kptr += 16;

  1525.                 }

  1526.             }

  1527.             for (; q + 1 < inh; q += 2)

  1528.             {

  1529.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1530. 

  1531.                 for (int k = 0; k < kernel_w; k++)

  1532.                 {

  1533.                     __fp16 val0;

  1534.                     __fp16 val1;

  1535.                     // if (elempack == 1)

  1536.                     {

  1537.                         val0 = r0[0];

  1538.                         val1 = r0[N];

  1539.                         r0 += dilation_w;

  1540.                     }

  1541. 

  1542.                     float16x4_t _w0 = vld1_f16(kptr);

  1543.                     float16x4_t _w1 = vld1_f16(kptr + 4);

  1544.                     _sum0 = vfma_n_f16(_sum0, _w0, val0);

  1545.                     _sum1 = vfma_n_f16(_sum1, _w1, val1);

  1546. 

  1547.                     kptr += 8;

  1548.                 }

  1549.             }

  1550.             for (; q < inh; q++)

  1551.             {

  1552.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1553. 

  1554.                 for (int k = 0; k < kernel_w; k++)

  1555.                 {

  1556.                     float16x4_t _val;

  1557.                     // if (elempack == 1)

  1558.                     {

  1559.                         _val = vdup_n_f16(r0[0]);

  1560.                         r0 += dilation_w;

  1561.                     }

  1562. 

  1563.                     float16x4_t _w = vld1_f16(kptr);

  1564.                     _sum0 = vfma_f16(_sum0, _val, _w);

  1565. 

  1566.                     kptr += 4;

  1567.                 }

  1568.             }

  1569. 

  1570.             _sum0 = vadd_f16(_sum0, _sum1);

  1571.             _sum2 = vadd_f16(_sum2, _sum3);

  1572.             _sum0 = vadd_f16(_sum0, _sum2);

  1573. 

  1574.             _sum0 = activation_ps_f16(_sum0, activation_type, activation_params);

  1575. 

  1576.             if (out_elempack == 4)

  1577.             {

  1578.                 vst1_f16(outptr, _sum0);

  1579.                 outptr += 4;

  1580.             }

  1581.             else // if (out_elempack == 1)

  1582.             {

  1583.                 outptr[0] = vget_lane_f16(_sum0, 0);

  1584.                 outptr[M] = vget_lane_f16(_sum0, 1);

  1585.                 outptr[M * 2] = vget_lane_f16(_sum0, 2);

  1586.                 outptr[M * 3] = vget_lane_f16(_sum0, 3);

  1587.                 outptr += 1;

  1588.             }

  1589.         }

  1590.     }

  1591.     remain_outh_start += nn_outh * 4;

  1592.     nn_outh = (outh - remain_outh_start) / 2;

  1593.     for (int pp = 0; pp < nn_outh; pp++)

  1594.     {

  1595.         const int p = remain_outh_start + pp * 2;

  1596. 

  1597.         // shadowed variable for less openmp task args

  1598.         const int elempack = bottom_blob.elempack;

  1599.         const int inh = bottom_blob.h * elempack;

  1600.         const int outw = top_blob.w;

  1601. 

  1602.         __fp16* outptr0 = top_blob.row<__fp16>(p);

  1603.         __fp16* outptr1 = top_blob.row<__fp16>(p + 1);

  1604. 

  1605.         for (int j = 0; j < outw; j++)

  1606.         {

  1607.             __fp16 sum0 = 0.f;

  1608.             __fp16 sum1 = 0.f;

  1609. 

  1610.             if (bias_data_ptr)

  1611.             {

  1612.                 sum0 = bias_data_ptr[p];

  1613.                 sum1 = bias_data_ptr[p + 1];

  1614.             }

  1615. 

  1616.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4 + (p % 4) / 2);

  1617. 

  1618.             int q = 0;

  1619.             float16x8_t _sum0 = vdupq_n_f16(0.f);

  1620.             float16x8_t _sum1 = vdupq_n_f16(0.f);

  1621.             for (; q + 7 < inh; q += 8)

  1622.             {

  1623.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1624. 

  1625.                 for (int k = 0; k < kernel_w; k++)

  1626.                 {

  1627.                     float16x8_t _r0;

  1628.                     if (elempack == 8)

  1629.                     {

  1630.                         _r0 = vld1q_f16(r0);

  1631.                         r0 += dilation_w * 8;

  1632.                     }

  1633.                     else if (elempack == 4)

  1634.                     {

  1635.                         _r0 = vcombine_f16(vld1_f16(r0), vld1_f16(r0 + N));

  1636.                         r0 += dilation_w * 4;

  1637.                     }

  1638.                     else // if (elempack == 1)

  1639.                     {

  1640.                         _r0 = float16x8_t();

  1641.                         _r0 = vsetq_lane_f16(r0[0], _r0, 0);

  1642.                         _r0 = vsetq_lane_f16(r0[N], _r0, 1);

  1643.                         _r0 = vsetq_lane_f16(r0[N * 2], _r0, 2);

  1644.                         _r0 = vsetq_lane_f16(r0[N * 3], _r0, 3);

  1645.                         _r0 = vsetq_lane_f16(r0[N * 4], _r0, 4);

  1646.                         _r0 = vsetq_lane_f16(r0[N * 5], _r0, 5);

  1647.                         _r0 = vsetq_lane_f16(r0[N * 6], _r0, 6);

  1648.                         _r0 = vsetq_lane_f16(r0[N * 7], _r0, 7);

  1649.                         r0 += dilation_w;

  1650.                     }

  1651. 

  1652.                     float16x8_t _w0 = vld1q_f16(kptr);

  1653.                     float16x8_t _w1 = vld1q_f16(kptr + 8);

  1654.                     _sum0 = vfmaq_f16(_sum0, _r0, _w0);

  1655.                     _sum1 = vfmaq_f16(_sum1, _r0, _w1);

  1656. 

  1657.                     kptr += 16;

  1658.                 }

  1659.             }

  1660.             for (; q + 3 < inh; q += 4)

  1661.             {

  1662.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1663. 

  1664.                 for (int k = 0; k < kernel_w; k++)

  1665.                 {

  1666.                     float16x4_t _r0;

  1667.                     if (elempack == 4)

  1668.                     {

  1669.                         _r0 = vld1_f16(r0);

  1670.                         r0 += dilation_w * 4;

  1671.                     }

  1672.                     else // if (elempack == 1)

  1673.                     {

  1674.                         _r0 = float16x4_t();

  1675.                         _r0 = vset_lane_f16(r0[0], _r0, 0);

  1676.                         _r0 = vset_lane_f16(r0[N], _r0, 1);

  1677.                         _r0 = vset_lane_f16(r0[N * 2], _r0, 2);

  1678.                         _r0 = vset_lane_f16(r0[N * 3], _r0, 3);

  1679.                         r0 += dilation_w;

  1680.                     }

  1681. 

  1682.                     float16x4_t _w0 = vld1_f16(kptr);

  1683.                     float16x4_t _w1 = vld1_f16(kptr + 4);

  1684.                     _sum0 = vcombine_f16(vfma_f16(vget_low_f16(_sum0), _r0, _w0), vget_high_f16(_sum0));

  1685.                     _sum1 = vcombine_f16(vfma_f16(vget_low_f16(_sum1), _r0, _w1), vget_high_f16(_sum1));

  1686. 

  1687.                     kptr += 8;

  1688.                 }

  1689.             }

  1690.             for (; q + 1 < inh; q += 2)

  1691.             {

  1692.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1693. 

  1694.                 for (int k = 0; k < kernel_w; k++)

  1695.                 {

  1696.                     __fp16 val0;

  1697.                     __fp16 val1;

  1698.                     // if (elempack == 1)

  1699.                     {

  1700.                         val0 = r0[0];

  1701.                         val1 = r0[N];

  1702.                         r0 += dilation_w;

  1703.                     }

  1704. 

  1705.                     sum0 += val0 * kptr[0];

  1706.                     sum1 += val0 * kptr[1];

  1707.                     sum0 += val1 * kptr[2];

  1708.                     sum1 += val1 * kptr[3];

  1709. 

  1710.                     kptr += 4;

  1711.                 }

  1712.             }

  1713.             for (; q < inh; q++)

  1714.             {

  1715.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1716. 

  1717.                 for (int k = 0; k < kernel_w; k++)

  1718.                 {

  1719.                     __fp16 val;

  1720.                     // if (elempack == 1)

  1721.                     {

  1722.                         val = r0[0];

  1723.                         r0 += dilation_w;

  1724.                     }

  1725. 

  1726.                     sum0 += val * kptr[0];

  1727.                     sum1 += val * kptr[1];

  1728. 

  1729.                     kptr += 2;

  1730.                 }

  1731.             }

  1732. 

  1733.             float16x4_t _ss0 = vadd_f16(vget_low_f16(_sum0), vget_high_f16(_sum0));

  1734.             float16x4_t _ss1 = vadd_f16(vget_low_f16(_sum1), vget_high_f16(_sum1));

  1735.             float16x4_t _ss = vpadd_f16(_ss0, _ss1);

  1736.             _ss = vpadd_f16(_ss, _ss);

  1737.             sum0 += vget_lane_f16(_ss, 0);

  1738.             sum1 += vget_lane_f16(_ss, 1);

  1739. 

  1740.             sum0 = activation_ss_f16(sum0, activation_type, activation_params);

  1741.             sum1 = activation_ss_f16(sum1, activation_type, activation_params);

  1742. 

  1743.             outptr0[0] = sum0;

  1744.             outptr1[0] = sum1;

  1745.             outptr0 += 1;

  1746.             outptr1 += 1;

  1747.         }

  1748.     }

  1749.     remain_outh_start += nn_outh * 2;

  1750.     for (int p = remain_outh_start; p < outh; p++)

  1751.     {

  1752.         __fp16* outptr = top_blob.row<__fp16>(p);

  1753. 

  1754.         for (int j = 0; j < outw; j++)

  1755.         {

  1756.             __fp16 sum = 0.f;

  1757. 

  1758.             if (bias_data_ptr)

  1759.             {

  1760.                 sum = bias_data_ptr[p];

  1761.             }

  1762. 

  1763.             const __fp16* kptr = weight_data_tm.channel(p / 8 + (p % 8) / 4 + (p % 4) / 2 + p % 2);

  1764. 

  1765.             int q = 0;

  1766.             float16x8_t _sum = vdupq_n_f16(0.f);

  1767.             for (; q + 7 < inh; q += 8)

  1768.             {

  1769.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1770. 

  1771.                 for (int k = 0; k < kernel_w; k++)

  1772.                 {

  1773.                     float16x8_t _r0;

  1774.                     if (elempack == 8)

  1775.                     {

  1776.                         _r0 = vld1q_f16(r0);

  1777.                         r0 += dilation_w * 8;

  1778.                     }

  1779.                     else if (elempack == 4)

  1780.                     {

  1781.                         _r0 = vcombine_f16(vld1_f16(r0), vld1_f16(r0 + N));

  1782.                         r0 += dilation_w * 4;

  1783.                     }

  1784.                     else // if (elempack == 1)

  1785.                     {

  1786.                         _r0 = float16x8_t();

  1787.                         _r0 = vsetq_lane_f16(r0[0], _r0, 0);

  1788.                         _r0 = vsetq_lane_f16(r0[N], _r0, 1);

  1789.                         _r0 = vsetq_lane_f16(r0[N * 2], _r0, 2);

  1790.                         _r0 = vsetq_lane_f16(r0[N * 3], _r0, 3);

  1791.                         _r0 = vsetq_lane_f16(r0[N * 4], _r0, 4);

  1792.                         _r0 = vsetq_lane_f16(r0[N * 5], _r0, 5);

  1793.                         _r0 = vsetq_lane_f16(r0[N * 6], _r0, 6);

  1794.                         _r0 = vsetq_lane_f16(r0[N * 7], _r0, 7);

  1795.                         r0 += dilation_w;

  1796.                     }

  1797. 

  1798.                     float16x8_t _w0 = vld1q_f16(kptr);

  1799.                     _sum = vfmaq_f16(_sum, _r0, _w0);

  1800. 

  1801.                     kptr += 8;

  1802.                 }

  1803.             }

  1804.             for (; q + 3 < inh; q += 4)

  1805.             {

  1806.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q / elempack) + j * stride_w * elempack;

  1807. 

  1808.                 for (int k = 0; k < kernel_w; k++)

  1809.                 {

  1810.                     float16x4_t _r0;

  1811.                     if (elempack == 4)

  1812.                     {

  1813.                         _r0 = vld1_f16(r0);

  1814.                         r0 += dilation_w * 4;

  1815.                     }

  1816.                     else // if (elempack == 1)

  1817.                     {

  1818.                         _r0 = float16x4_t();

  1819.                         _r0 = vset_lane_f16(r0[0], _r0, 0);

  1820.                         _r0 = vset_lane_f16(r0[N], _r0, 1);

  1821.                         _r0 = vset_lane_f16(r0[N * 2], _r0, 2);

  1822.                         _r0 = vset_lane_f16(r0[N * 3], _r0, 3);

  1823.                         r0 += dilation_w;

  1824.                     }

  1825. 

  1826.                     float16x4_t _w = vld1_f16(kptr);

  1827.                     _sum = vcombine_f16(vfma_f16(vget_low_f16(_sum), _r0, _w), vget_high_f16(_sum));

  1828. 

  1829.                     kptr += 4;

  1830.                 }

  1831.             }

  1832.             for (; q + 1 < inh; q += 2)

  1833.             {

  1834.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1835. 

  1836.                 for (int k = 0; k < kernel_w; k++)

  1837.                 {

  1838.                     __fp16 val0;

  1839.                     __fp16 val1;

  1840.                     // if (elempack == 1)

  1841.                     {

  1842.                         val0 = r0[0];

  1843.                         val1 = r0[N];

  1844.                         r0 += dilation_w;

  1845.                     }

  1846. 

  1847.                     sum += val0 * kptr[0];

  1848.                     sum += val1 * kptr[1];

  1849. 

  1850.                     kptr += 2;

  1851.                 }

  1852.             }

  1853.             for (; q < inh; q++)

  1854.             {

  1855.                 const __fp16* r0 = bottom_blob.row<const __fp16>(q) + j * stride_w;

  1856. 

  1857.                 for (int k = 0; k < kernel_w; k++)

  1858.                 {

  1859.                     __fp16 val;

  1860.                     // if (elempack == 1)

  1861.                     {

  1862.                         val = r0[0];

  1863.                         r0 += dilation_w;

  1864.                     }

  1865. 

  1866.                     sum += val * kptr[0];

  1867. 

  1868.                     kptr += 1;

  1869.                 }

  1870.             }

  1871. 

  1872.             float16x4_t _ss = vadd_f16(vget_low_f16(_sum), vget_high_f16(_sum));

  1873.             _ss = vpadd_f16(_ss, _ss);

  1874.             _ss = vpadd_f16(_ss, _ss);

  1875.             sum += vget_lane_f16(_ss, 0);

  1876. 

  1877.             sum = activation_ss_f16(sum, activation_type, activation_params);

  1878. 

  1879.             outptr[0] = sum;

  1880.             outptr += 1;

  1881.         }

  1882.     }

  1883. }