nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f893d2440d
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f893d2440d'
${ noResults }
ncnn/src/layer/arm/innerproduct_arm_asimdhp.cpp

798 lines
32 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #include "innerproduct_arm.h"

  16. 

  17. #include "cpu.h"

  18. 

  19. #if __ARM_NEON

  20. #include <arm_neon.h>

  21. #endif // __ARM_NEON

  22. 

  23. #include "arm_activation.h"

  24. #include "arm_usability.h"

  25. 

  26. namespace ncnn {

  27. 

  28. #include "innerproduct_fp16s.h"

  29. #include "innerproduct_gemm_fp16s.h"

  30. 

  31. void innerproduct_pack4_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)

  32. {

  33.     innerproduct_pack4_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);

  34. }

  35. 

  36. void innerproduct_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)

  37. {

  38.     innerproduct_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);

  39. }

  40. 

  41. void innerproduct_gemm_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)

  42. {

  43.     innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);

  44. }

  45. 

  46. void innerproduct_transform_kernel_fp16s_neon_asimdhp(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt)

  47. {

  48.     innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt);

  49. }

  50. 

  51. #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  52. int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  53. {

  54.     const int num_input = weight_data_size / num_output;

  55. 

  56.     if (bottom_blob.dims == 2 && bottom_blob.w == num_input)

  57.     {

  58.         // gemm

  59.         int h = bottom_blob.h;

  60.         size_t elemsize = bottom_blob.elemsize;

  61.         int elempack = bottom_blob.elempack;

  62. 

  63.         top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator);

  64.         if (top_blob.empty())

  65.             return -100;

  66. 

  67.         int num_output_elempack = 1;

  68.         if (opt.use_packing_layout)

  69.         {

  70.             num_output_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  71.         }

  72. 

  73.         #pragma omp parallel for num_threads(opt.num_threads)

  74.         for (int j = 0; j < h; j++)

  75.         {

  76.             if (elempack == 8 && num_output_elempack == 8)

  77.             {

  78.                 __fp16* outptr = top_blob.row<__fp16>(j);

  79. 

  80.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  81.                 {

  82.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8;

  83.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  84. 

  85.                     float16x8_t _sum0 = vdupq_n_f16((__fp16)0.f);

  86.                     float16x8_t _sum1 = vdupq_n_f16((__fp16)0.f);

  87.                     float16x8_t _sum2 = vdupq_n_f16((__fp16)0.f);

  88.                     float16x8_t _sum3 = vdupq_n_f16((__fp16)0.f);

  89.                     float16x8_t _sum4 = vdupq_n_f16((__fp16)0.f);

  90.                     float16x8_t _sum5 = vdupq_n_f16((__fp16)0.f);

  91.                     float16x8_t _sum6 = vdupq_n_f16((__fp16)0.f);

  92.                     float16x8_t _sum7 = vdupq_n_f16((__fp16)0.f);

  93. 

  94.                     if (bias_term)

  95.                     {

  96.                         _sum0 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 0]);

  97.                         _sum1 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 1]);

  98.                         _sum2 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 2]);

  99.                         _sum3 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 3]);

  100.                         _sum4 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 4]);

  101.                         _sum5 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 5]);

  102.                         _sum6 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 6]);

  103.                         _sum7 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 7]);

  104.                     }

  105. 

  106.                     for (int i = 0; i < num_input; i++)

  107.                     {

  108.                         float16x8_t _val = vld1q_f16(m);

  109.                         float16x8_t _k = vld1q_f16(kptr);

  110.                         _sum0 = vfmaq_laneq_f16(_sum0, _val, _k, 0);

  111.                         _sum1 = vfmaq_laneq_f16(_sum1, _val, _k, 1);

  112.                         _sum2 = vfmaq_laneq_f16(_sum2, _val, _k, 2);

  113.                         _sum3 = vfmaq_laneq_f16(_sum3, _val, _k, 3);

  114.                         _sum4 = vfmaq_laneq_f16(_sum4, _val, _k, 4);

  115.                         _sum5 = vfmaq_laneq_f16(_sum5, _val, _k, 5);

  116.                         _sum6 = vfmaq_laneq_f16(_sum6, _val, _k, 6);

  117.                         _sum7 = vfmaq_laneq_f16(_sum7, _val, _k, 7);

  118. 

  119.                         m += 8;

  120.                         kptr += 8;

  121.                     }

  122. 

  123.                     _sum0 = activation_ps(_sum0, activation_type, activation_params);

  124.                     _sum1 = activation_ps(_sum1, activation_type, activation_params);

  125.                     _sum2 = activation_ps(_sum2, activation_type, activation_params);

  126.                     _sum3 = activation_ps(_sum3, activation_type, activation_params);

  127.                     _sum4 = activation_ps(_sum4, activation_type, activation_params);

  128.                     _sum5 = activation_ps(_sum5, activation_type, activation_params);

  129.                     _sum6 = activation_ps(_sum6, activation_type, activation_params);

  130.                     _sum7 = activation_ps(_sum7, activation_type, activation_params);

  131. 

  132.                     vst1q_f16(outptr, _sum0);

  133.                     vst1q_f16(outptr + 8, _sum1);

  134.                     vst1q_f16(outptr + 16, _sum2);

  135.                     vst1q_f16(outptr + 24, _sum3);

  136.                     vst1q_f16(outptr + 32, _sum4);

  137.                     vst1q_f16(outptr + 40, _sum5);

  138.                     vst1q_f16(outptr + 48, _sum6);

  139.                     vst1q_f16(outptr + 56, _sum7);

  140.                     outptr += 64;

  141.                 }

  142.             }

  143. 

  144.             if (elempack == 1 && num_output_elempack == 8)

  145.             {

  146.                 __fp16* outptr = top_blob.row<__fp16>(j);

  147. 

  148.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  149.                 {

  150.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8;

  151.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  152. 

  153.                     float16x8_t _sum = vdupq_n_f16(0.f);

  154. 

  155.                     if (bias_term)

  156.                     {

  157.                         _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);

  158.                     }

  159. 

  160.                     for (int i = 0; i < num_input; i++)

  161.                     {

  162.                         float16x8_t _val = vdupq_n_f16(m[0]);

  163.                         float16x8_t _k = vld1q_f16(kptr);

  164.                         _sum = vfmaq_f16(_sum, _val, _k);

  165. 

  166.                         m += 1;

  167.                         kptr += 8;

  168.                     }

  169. 

  170.                     _sum = activation_ps(_sum, activation_type, activation_params);

  171. 

  172.                     vst1q_f16(outptr, _sum);

  173.                     outptr += 8;

  174.                 }

  175.             }

  176. 

  177.             if (elempack == 4 && num_output_elempack == 8)

  178.             {

  179.                 __fp16* outptr = top_blob.row<__fp16>(j);

  180. 

  181.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  182.                 {

  183.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8;

  184.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  185. 

  186.                     float16x4_t _sum0 = vdup_n_f16(0.f);

  187.                     float16x4_t _sum1 = vdup_n_f16(0.f);

  188.                     float16x4_t _sum2 = vdup_n_f16(0.f);

  189.                     float16x4_t _sum3 = vdup_n_f16(0.f);

  190.                     float16x4_t _sum4 = vdup_n_f16(0.f);

  191.                     float16x4_t _sum5 = vdup_n_f16(0.f);

  192.                     float16x4_t _sum6 = vdup_n_f16(0.f);

  193.                     float16x4_t _sum7 = vdup_n_f16(0.f);

  194. 

  195.                     if (bias_term)

  196.                     {

  197.                         _sum0 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 0]);

  198.                         _sum1 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 1]);

  199.                         _sum2 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 2]);

  200.                         _sum3 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 3]);

  201.                         _sum4 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 4]);

  202.                         _sum5 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 5]);

  203.                         _sum6 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 6]);

  204.                         _sum7 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 8 + 7]);

  205.                     }

  206. 

  207.                     for (int i = 0; i < num_input; i++)

  208.                     {

  209.                         float16x4_t _val = vld1_f16(m);

  210.                         float16x8_t _k = vld1q_f16(kptr);

  211.                         _sum0 = vfma_laneq_f16(_sum0, _val, _k, 0);

  212.                         _sum1 = vfma_laneq_f16(_sum1, _val, _k, 1);

  213.                         _sum2 = vfma_laneq_f16(_sum2, _val, _k, 2);

  214.                         _sum3 = vfma_laneq_f16(_sum3, _val, _k, 3);

  215.                         _sum4 = vfma_laneq_f16(_sum4, _val, _k, 4);

  216.                         _sum5 = vfma_laneq_f16(_sum5, _val, _k, 5);

  217.                         _sum6 = vfma_laneq_f16(_sum6, _val, _k, 6);

  218.                         _sum7 = vfma_laneq_f16(_sum7, _val, _k, 7);

  219. 

  220.                         m += 4;

  221.                         kptr += 8;

  222.                     }

  223. 

  224.                     _sum0 = activation_ps(_sum0, activation_type, activation_params);

  225.                     _sum1 = activation_ps(_sum1, activation_type, activation_params);

  226.                     _sum2 = activation_ps(_sum2, activation_type, activation_params);

  227.                     _sum3 = activation_ps(_sum3, activation_type, activation_params);

  228.                     _sum4 = activation_ps(_sum4, activation_type, activation_params);

  229.                     _sum5 = activation_ps(_sum5, activation_type, activation_params);

  230.                     _sum6 = activation_ps(_sum6, activation_type, activation_params);

  231.                     _sum7 = activation_ps(_sum7, activation_type, activation_params);

  232. 

  233.                     vst1_f16(outptr, _sum0);

  234.                     vst1_f16(outptr + 4, _sum1);

  235.                     vst1_f16(outptr + 8, _sum2);

  236.                     vst1_f16(outptr + 12, _sum3);

  237.                     vst1_f16(outptr + 16, _sum4);

  238.                     vst1_f16(outptr + 20, _sum5);

  239.                     vst1_f16(outptr + 24, _sum6);

  240.                     vst1_f16(outptr + 28, _sum7);

  241.                     outptr += 32;

  242.                 }

  243.             }

  244. 

  245.             if (elempack == 8 && num_output_elempack == 1)

  246.             {

  247.                 __fp16* outptr = top_blob.row<__fp16>(j);

  248. 

  249.                 for (int p = 0; p < num_output; p++)

  250.                 {

  251.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p;

  252.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  253. 

  254.                     float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

  255. 

  256.                     if (bias_term)

  257.                     {

  258.                         _sum = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p]);

  259.                     }

  260. 

  261.                     for (int i = 0; i < num_input; i++)

  262.                     {

  263.                         float16x8_t _val = vld1q_f16(m);

  264.                         float16x8_t _k = vdupq_n_f16(kptr[0]);

  265.                         _sum = vfmaq_f16(_sum, _val, _k);

  266. 

  267.                         m += 8;

  268.                         kptr += 1;

  269.                     }

  270. 

  271.                     _sum = activation_ps(_sum, activation_type, activation_params);

  272. 

  273.                     vst1q_f16(outptr, _sum);

  274.                     outptr += 8;

  275.                 }

  276.             }

  277. 

  278.             if (elempack == 8 && num_output_elempack == 4)

  279.             {

  280.                 __fp16* outptr = top_blob.row<__fp16>(j);

  281. 

  282.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  283.                 {

  284.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4;

  285.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  286. 

  287.                     float16x8_t _sum0 = vdupq_n_f16((__fp16)0.f);

  288.                     float16x8_t _sum1 = vdupq_n_f16((__fp16)0.f);

  289.                     float16x8_t _sum2 = vdupq_n_f16((__fp16)0.f);

  290.                     float16x8_t _sum3 = vdupq_n_f16((__fp16)0.f);

  291. 

  292.                     if (bias_term)

  293.                     {

  294.                         _sum0 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 0]);

  295.                         _sum1 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 1]);

  296.                         _sum2 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 2]);

  297.                         _sum3 = vdupq_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 3]);

  298.                     }

  299. 

  300.                     for (int i = 0; i < num_input; i++)

  301.                     {

  302.                         float16x8_t _val = vld1q_f16(m);

  303.                         float16x4_t _k = vld1_f16(kptr);

  304.                         _sum0 = vfmaq_lane_f16(_sum0, _val, _k, 0);

  305.                         _sum1 = vfmaq_lane_f16(_sum1, _val, _k, 1);

  306.                         _sum2 = vfmaq_lane_f16(_sum2, _val, _k, 2);

  307.                         _sum3 = vfmaq_lane_f16(_sum3, _val, _k, 3);

  308. 

  309.                         m += 8;

  310.                         kptr += 4;

  311.                     }

  312. 

  313.                     _sum0 = activation_ps(_sum0, activation_type, activation_params);

  314.                     _sum1 = activation_ps(_sum1, activation_type, activation_params);

  315.                     _sum2 = activation_ps(_sum2, activation_type, activation_params);

  316.                     _sum3 = activation_ps(_sum3, activation_type, activation_params);

  317. 

  318.                     vst1q_f16(outptr, _sum0);

  319.                     vst1q_f16(outptr + 8, _sum1);

  320.                     vst1q_f16(outptr + 16, _sum2);

  321.                     vst1q_f16(outptr + 24, _sum3);

  322.                     outptr += 32;

  323.                 }

  324.             }

  325. 

  326.             if (elempack == 4 && num_output_elempack == 4)

  327.             {

  328.                 __fp16* outptr = top_blob.row<__fp16>(j);

  329. 

  330.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  331.                 {

  332.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4;

  333.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  334. 

  335.                     float16x4_t _sum0 = vdup_n_f16(0.f);

  336.                     float16x4_t _sum1 = vdup_n_f16(0.f);

  337.                     float16x4_t _sum2 = vdup_n_f16(0.f);

  338.                     float16x4_t _sum3 = vdup_n_f16(0.f);

  339. 

  340.                     if (bias_term)

  341.                     {

  342.                         _sum0 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 0]);

  343.                         _sum1 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 1]);

  344.                         _sum2 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 2]);

  345.                         _sum3 = vdup_n_f16(((const __fp16*)bias_data_fp16)[p * 4 + 3]);

  346.                     }

  347. 

  348.                     for (int i = 0; i < num_input; i++)

  349.                     {

  350.                         float16x4_t _val = vld1_f16(m);

  351.                         float16x4_t _k = vld1_f16(kptr);

  352.                         _sum0 = vfma_lane_f16(_sum0, _val, _k, 0);

  353.                         _sum1 = vfma_lane_f16(_sum1, _val, _k, 1);

  354.                         _sum2 = vfma_lane_f16(_sum2, _val, _k, 2);

  355.                         _sum3 = vfma_lane_f16(_sum3, _val, _k, 3);

  356. 

  357.                         m += 4;

  358.                         kptr += 4;

  359.                     }

  360. 

  361.                     _sum0 = activation_ps(_sum0, activation_type, activation_params);

  362.                     _sum1 = activation_ps(_sum1, activation_type, activation_params);

  363.                     _sum2 = activation_ps(_sum2, activation_type, activation_params);

  364.                     _sum3 = activation_ps(_sum3, activation_type, activation_params);

  365. 

  366.                     vst1_f16(outptr, _sum0);

  367.                     vst1_f16(outptr + 4, _sum1);

  368.                     vst1_f16(outptr + 8, _sum2);

  369.                     vst1_f16(outptr + 12, _sum3);

  370.                     outptr += 16;

  371.                 }

  372.             }

  373. 

  374.             if (elempack == 1 && num_output_elempack == 4)

  375.             {

  376.                 __fp16* outptr = top_blob.row<__fp16>(j);

  377. 

  378.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  379.                 {

  380.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4;

  381.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  382. 

  383.                     float16x4_t _sum = vdup_n_f16(0.f);

  384. 

  385.                     if (bias_term)

  386.                     {

  387.                         _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);

  388.                     }

  389. 

  390.                     for (int i = 0; i < num_input; i++)

  391.                     {

  392.                         float16x4_t _val = vdup_n_f16(m[0]);

  393.                         float16x4_t _k = vld1_f16(kptr);

  394.                         _sum = vfma_f16(_sum, _val, _k);

  395. 

  396.                         m += 1;

  397.                         kptr += 4;

  398.                     }

  399. 

  400.                     _sum = activation_ps(_sum, activation_type, activation_params);

  401. 

  402.                     vst1_f16(outptr, _sum);

  403.                     outptr += 4;

  404.                 }

  405.             }

  406. 

  407.             if (elempack == 4 && num_output_elempack == 1)

  408.             {

  409.                 __fp16* outptr = top_blob.row<__fp16>(j);

  410. 

  411.                 for (int p = 0; p < num_output; p++)

  412.                 {

  413.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p;

  414.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  415. 

  416.                     float16x4_t _sum = vdup_n_f16(0.f);

  417. 

  418.                     if (bias_term)

  419.                     {

  420.                         _sum = vdup_n_f16(((const __fp16*)bias_data_fp16)[p]);

  421.                     }

  422. 

  423.                     for (int i = 0; i < num_input; i++)

  424.                     {

  425.                         float16x4_t _val = vld1_f16(m);

  426.                         float16x4_t _k = vdup_n_f16(kptr[0]);

  427.                         _sum = vfma_f16(_sum, _val, _k);

  428. 

  429.                         m += 4;

  430.                         kptr += 1;

  431.                     }

  432. 

  433.                     _sum = activation_ps(_sum, activation_type, activation_params);

  434. 

  435.                     vst1_f16(outptr, _sum);

  436.                     outptr += 4;

  437.                 }

  438.             }

  439. 

  440.             if (elempack == 1 && num_output_elempack == 1)

  441.             {

  442.                 __fp16* outptr = top_blob.row<__fp16>(j);

  443. 

  444.                 for (int p = 0; p < num_output; p++)

  445.                 {

  446.                     const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p;

  447.                     const __fp16* m = bottom_blob.row<const __fp16>(j);

  448. 

  449.                     float sum = 0.f;

  450. 

  451.                     if (bias_term)

  452.                     {

  453.                         sum = bias_data[p];

  454.                     }

  455. 

  456.                     for (int i = 0; i < num_input; i++)

  457.                     {

  458.                         sum += (float)(*m * *kptr);

  459. 

  460.                         m += 1;

  461.                         kptr += 1;

  462.                     }

  463. 

  464.                     sum = activation_ss(sum, activation_type, activation_params);

  465. 

  466.                     outptr[0] = (__fp16)sum;

  467.                     outptr += 1;

  468.                 }

  469.             }

  470.         }

  471. 

  472.         return 0;

  473.     }

  474. 

  475.     // flatten

  476.     Mat bottom_blob_flattened = bottom_blob;

  477.     if (bottom_blob.dims != 1)

  478.     {

  479.         Option opt_flatten = opt;

  480.         opt_flatten.blob_allocator = opt.workspace_allocator;

  481. 

  482.         flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);

  483.     }

  484. 

  485.     size_t elemsize = bottom_blob_flattened.elemsize;

  486.     int elempack = bottom_blob_flattened.elempack;

  487. 

  488.     int out_elempack = 1;

  489.     if (opt.use_packing_layout)

  490.     {

  491.         out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  492.     }

  493.     size_t out_elemsize = elemsize / elempack * out_elempack;

  494. 

  495.     top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);

  496.     if (top_blob.empty())

  497.         return -100;

  498. 

  499.     if (out_elempack == 8)

  500.     {

  501.         // num_output

  502.         #pragma omp parallel for num_threads(opt.num_threads)

  503.         for (int p = 0; p < num_output / out_elempack; p++)

  504.         {

  505.             float16x8_t _sum0 = vdupq_n_f16(0.f);

  506.             float16x8_t _sum1 = vdupq_n_f16(0.f);

  507.             float16x8_t _sum2 = vdupq_n_f16(0.f);

  508.             float16x8_t _sum3 = vdupq_n_f16(0.f);

  509.             float16x8_t _sum4 = vdupq_n_f16(0.f);

  510.             float16x8_t _sum5 = vdupq_n_f16(0.f);

  511.             float16x8_t _sum6 = vdupq_n_f16(0.f);

  512.             float16x8_t _sum7 = vdupq_n_f16(0.f);

  513. 

  514.             if (bias_term)

  515.             {

  516.                 _sum0 = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);

  517.             }

  518. 

  519.             const __fp16* kptr = weight_data_tm.row<const __fp16>(p);

  520. 

  521.             const __fp16* sptr = bottom_blob_flattened;

  522. 

  523.             int i = 0;

  524.             for (; i + 7 < num_input; i += 8)

  525.             {

  526.                 asm volatile(

  527.                     "prfm   pldl1keep, [%8, #128]       \n"

  528.                     "ld1    {v0.8h}, [%8], #16          \n" // _val

  529. 

  530.                     "prfm   pldl1keep, [%9, #512]       \n"

  531.                     "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%9], #64 \n" // w0123

  532. 

  533.                     "prfm   pldl1keep, [%9, #512]       \n"

  534.                     "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%9], #64 \n" // w4567

  535. 

  536.                     "fmla   %0.8h, v8.8h, v0.h[0]       \n"

  537.                     "fmla   %1.8h, v9.8h, v0.h[1]       \n"

  538.                     "fmla   %2.8h, v10.8h, v0.h[2]      \n"

  539.                     "fmla   %3.8h, v11.8h, v0.h[3]      \n"

  540.                     "fmla   %4.8h, v12.8h, v0.h[4]      \n"

  541.                     "fmla   %5.8h, v13.8h, v0.h[5]      \n"

  542.                     "fmla   %6.8h, v14.8h, v0.h[6]      \n"

  543.                     "fmla   %7.8h, v15.8h, v0.h[7]      \n"

  544. 

  545.                     : "=w"(_sum0), // %0

  546.                     "=w"(_sum1), // %1

  547.                     "=w"(_sum2), // %2

  548.                     "=w"(_sum3), // %3

  549.                     "=w"(_sum4), // %4

  550.                     "=w"(_sum5), // %5

  551.                     "=w"(_sum6), // %6

  552.                     "=w"(_sum7), // %7

  553.                     "=r"(sptr),  // %8

  554.                     "=r"(kptr)   // %9

  555.                     : "0"(_sum0),

  556.                     "1"(_sum1),

  557.                     "2"(_sum2),

  558.                     "3"(_sum3),

  559.                     "4"(_sum4),

  560.                     "5"(_sum5),

  561.                     "6"(_sum6),

  562.                     "7"(_sum7),

  563.                     "8"(sptr),

  564.                     "9"(kptr)

  565.                     : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15");

  566.             }

  567.             for (; i + 3 < num_input; i += 4)

  568.             {

  569.                 asm volatile(

  570.                     "prfm   pldl1keep, [%4, #128]       \n"

  571.                     "ld1    {v0.4h}, [%4], #8           \n" // _val

  572. 

  573.                     "prfm   pldl1keep, [%5, #512]       \n"

  574.                     "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%5], #64 \n" // w0123

  575. 

  576.                     "fmla   %0.8h, v8.8h, v0.h[0]       \n"

  577.                     "fmla   %1.8h, v9.8h, v0.h[1]       \n"

  578.                     "fmla   %2.8h, v10.8h, v0.h[2]      \n"

  579.                     "fmla   %3.8h, v11.8h, v0.h[3]      \n"

  580. 

  581.                     : "=w"(_sum0), // %0

  582.                     "=w"(_sum1), // %1

  583.                     "=w"(_sum2), // %2

  584.                     "=w"(_sum3), // %3

  585.                     "=r"(sptr),  // %4

  586.                     "=r"(kptr)   // %5

  587.                     : "0"(_sum0),

  588.                     "1"(_sum1),

  589.                     "2"(_sum2),

  590.                     "3"(_sum3),

  591.                     "4"(sptr),

  592.                     "5"(kptr)

  593.                     : "cc", "memory", "v0", "v8", "v9", "v10", "v11");

  594.             }

  595.             for (; i < num_input; i++)

  596.             {

  597.                 float16x8_t _val = vdupq_n_f16(sptr[0]);

  598. 

  599.                 float16x8_t _w = vld1q_f16(kptr);

  600. 

  601.                 _sum0 = vfmaq_f16(_sum0, _val, _w);

  602. 

  603.                 sptr += 1;

  604.                 kptr += 8;

  605.             }

  606. 

  607.             _sum0 = vaddq_f16(_sum0, _sum1);

  608.             _sum2 = vaddq_f16(_sum2, _sum3);

  609.             _sum4 = vaddq_f16(_sum4, _sum5);

  610.             _sum6 = vaddq_f16(_sum6, _sum7);

  611.             _sum0 = vaddq_f16(_sum0, _sum2);

  612.             _sum4 = vaddq_f16(_sum4, _sum6);

  613.             _sum0 = vaddq_f16(_sum0, _sum4);

  614. 

  615.             _sum0 = activation_ps(_sum0, activation_type, activation_params);

  616. 

  617.             __fp16* outptr = (__fp16*)top_blob;

  618.             vst1q_f16(outptr + p * 8, _sum0);

  619.         }

  620.     }

  621. 

  622.     if (out_elempack == 4)

  623.     {

  624.         // num_output

  625.         #pragma omp parallel for num_threads(opt.num_threads)

  626.         for (int p = 0; p < num_output / out_elempack; p++)

  627.         {

  628.             float16x4_t _sum0 = vdup_n_f16(0.f);

  629.             float16x4_t _sum1 = vdup_n_f16(0.f);

  630.             float16x4_t _sum2 = vdup_n_f16(0.f);

  631.             float16x4_t _sum3 = vdup_n_f16(0.f);

  632.             float16x4_t _sum4 = vdup_n_f16(0.f);

  633.             float16x4_t _sum5 = vdup_n_f16(0.f);

  634.             float16x4_t _sum6 = vdup_n_f16(0.f);

  635.             float16x4_t _sum7 = vdup_n_f16(0.f);

  636. 

  637.             if (bias_term)

  638.             {

  639.                 _sum0 = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);

  640.             }

  641. 

  642.             const __fp16* kptr = weight_data_tm.row<const __fp16>(p);

  643. 

  644.             const __fp16* sptr = bottom_blob_flattened;

  645. 

  646.             int i = 0;

  647.             for (; i + 7 < num_input; i += 8)

  648.             {

  649.                 asm volatile(

  650.                     "prfm   pldl1keep, [%8, #128]       \n"

  651.                     "ld1    {v0.8h}, [%8], #16          \n" // _val

  652. 

  653.                     "prfm   pldl1keep, [%9, #256]       \n"

  654.                     "ld1    {v8.4h, v9.4h, v10.4h, v11.4h}, [%9], #32 \n" // w0123

  655. 

  656.                     "prfm   pldl1keep, [%9, #256]       \n"

  657.                     "ld1    {v12.4h, v13.4h, v14.4h, v15.4h}, [%9], #32 \n" // w4567

  658. 

  659.                     "fmla   %0.4h, v8.4h, v0.h[0]       \n"

  660.                     "fmla   %1.4h, v9.4h, v0.h[1]       \n"

  661.                     "fmla   %2.4h, v10.4h, v0.h[2]      \n"

  662.                     "fmla   %3.4h, v11.4h, v0.h[3]      \n"

  663.                     "fmla   %4.4h, v12.4h, v0.h[4]      \n"

  664.                     "fmla   %5.4h, v13.4h, v0.h[5]      \n"

  665.                     "fmla   %6.4h, v14.4h, v0.h[6]      \n"

  666.                     "fmla   %7.4h, v15.4h, v0.h[7]      \n"

  667. 

  668.                     : "=w"(_sum0), // %0

  669.                     "=w"(_sum1), // %1

  670.                     "=w"(_sum2), // %2

  671.                     "=w"(_sum3), // %3

  672.                     "=w"(_sum4), // %4

  673.                     "=w"(_sum5), // %5

  674.                     "=w"(_sum6), // %6

  675.                     "=w"(_sum7), // %7

  676.                     "=r"(sptr),  // %8

  677.                     "=r"(kptr)   // %9

  678.                     : "0"(_sum0),

  679.                     "1"(_sum1),

  680.                     "2"(_sum2),

  681.                     "3"(_sum3),

  682.                     "4"(_sum4),

  683.                     "5"(_sum5),

  684.                     "6"(_sum6),

  685.                     "7"(_sum7),

  686.                     "8"(sptr),

  687.                     "9"(kptr)

  688.                     : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15");

  689.             }

  690.             for (; i + 3 < num_input; i += 4)

  691.             {

  692.                 asm volatile(

  693.                     "prfm   pldl1keep, [%4, #128]       \n"

  694.                     "ld1    {v0.4h}, [%4], #8           \n" // _val

  695. 

  696.                     "prfm   pldl1keep, [%5, #256]       \n"

  697.                     "ld1    {v8.4h, v9.4h, v10.4h, v11.4h}, [%5], #32 \n" // w0123

  698. 

  699.                     "fmla   %0.4h, v8.4h, v0.h[0]       \n"

  700.                     "fmla   %1.4h, v9.4h, v0.h[1]       \n"

  701.                     "fmla   %2.4h, v10.4h, v0.h[2]      \n"

  702.                     "fmla   %3.4h, v11.4h, v0.h[3]      \n"

  703. 

  704.                     : "=w"(_sum0), // %0

  705.                     "=w"(_sum1), // %1

  706.                     "=w"(_sum2), // %2

  707.                     "=w"(_sum3), // %3

  708.                     "=r"(sptr),  // %4

  709.                     "=r"(kptr)   // %5

  710.                     : "0"(_sum0),

  711.                     "1"(_sum1),

  712.                     "2"(_sum2),

  713.                     "3"(_sum3),

  714.                     "4"(sptr),

  715.                     "5"(kptr)

  716.                     : "cc", "memory", "v0", "v8", "v9", "v10", "v11");

  717.             }

  718.             for (; i < num_input; i++)

  719.             {

  720.                 float16x4_t _val = vdup_n_f16(sptr[0]);

  721. 

  722.                 float16x4_t _w = vld1_f16(kptr);

  723. 

  724.                 _sum0 = vfma_f16(_sum0, _val, _w);

  725. 

  726.                 sptr += 1;

  727.                 kptr += 4;

  728.             }

  729. 

  730.             _sum0 = vadd_f16(_sum0, _sum1);

  731.             _sum2 = vadd_f16(_sum2, _sum3);

  732.             _sum4 = vadd_f16(_sum4, _sum5);

  733.             _sum6 = vadd_f16(_sum6, _sum7);

  734.             _sum0 = vadd_f16(_sum0, _sum2);

  735.             _sum4 = vadd_f16(_sum4, _sum6);

  736.             _sum0 = vadd_f16(_sum0, _sum4);

  737. 

  738.             _sum0 = activation_ps(_sum0, activation_type, activation_params);

  739. 

  740.             __fp16* outptr = (__fp16*)top_blob;

  741.             vst1_f16(outptr + p * 4, _sum0);

  742.         }

  743.     }

  744. 

  745.     if (out_elempack == 1)

  746.     {

  747.         // num_output

  748.         #pragma omp parallel for num_threads(opt.num_threads)

  749.         for (int p = 0; p < num_output; p++)

  750.         {

  751.             float sum = 0.f;

  752. 

  753.             if (bias_term)

  754.                 sum = bias_data[p];

  755. 

  756.             const __fp16* kptr = weight_data_tm.row<__fp16>(p);

  757. 

  758.             const __fp16* sptr = bottom_blob_flattened;

  759. 

  760.             float16x8_t _sum = vdupq_n_f16(0.f);

  761.             int i = 0;

  762.             for (; i + 7 < num_input; i += 8)

  763.             {

  764.                 float16x8_t _m = vld1q_f16(sptr);

  765.                 float16x8_t _w = vld1q_f16(kptr);

  766. 

  767.                 _sum = vfmaq_f16(_sum, _m, _w);

  768. 

  769.                 sptr += 8;

  770.                 kptr += 8;

  771.             }

  772.             for (; i < num_input; i++)

  773.             {

  774.                 __fp16 v = *sptr;

  775.                 __fp16 k = *kptr;

  776. 

  777.                 sum += (float)(v * k);

  778. 

  779.                 sptr++;

  780.                 kptr++;

  781.             }

  782. 

  783.             float16x4_t _s4 = vadd_f16(vget_low_f16(_sum), vget_high_f16(_sum));

  784.             sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot

  785. 

  786.             sum = activation_ss(sum, activation_type, activation_params);

  787. 

  788.             __fp16* outptr = (__fp16*)top_blob;

  789.             outptr[p] = (__fp16)sum;

  790.         }

  791.     }

  792. 

  793.     return 0;

  794. }

  795. #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  796. 

  797. } // namespace ncnn