nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 40a69a2dd3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '40a69a2dd3'
${ noResults }
ncnn/src/layer/x86/innerproduct_x86.cpp

2502 lines
108 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #include "innerproduct_x86.h"

  16. 

  17. #if __SSE2__

  18. #include <emmintrin.h>

  19. #if __AVX__

  20. #include <immintrin.h>

  21. #endif

  22. #endif // __SSE2__

  23. 

  24. #include "x86_activation.h"

  25. #include "x86_usability.h"

  26. 

  27. #include "layer_type.h"

  28. 

  29. namespace ncnn {

  30. 

  31. InnerProduct_x86::InnerProduct_x86()

  32. {

  33. #if __SSE2__

  34.     support_packing = true;

  35. #endif // __SSE2__

  36. 

  37.     flatten = 0;

  38. }

  39. 

  40. int InnerProduct_x86::create_pipeline(const Option& opt)

  41. {

  42.     //     if (opt.use_packing_layout)

  43.     {

  44.         flatten = ncnn::create_layer(ncnn::LayerType::Flatten);

  45. 

  46.         ncnn::ParamDict pd;

  47. 

  48.         flatten->load_param(pd);

  49. 

  50.         flatten->create_pipeline(opt);

  51.     }

  52. 

  53. #if NCNN_INT8

  54.     if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)

  55.     {

  56.         return create_pipeline_int8_x86(opt);

  57.     }

  58. #endif

  59. 

  60.     const int num_input = weight_data_size / num_output;

  61. 

  62.     int out_elempack = 1;

  63. 

  64. #if __SSE2__

  65.     if (opt.use_packing_layout)

  66.     {

  67. #if __AVX512F__

  68.         out_elempack = num_output % 16 == 0 ? 16 : num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  69. #elif __AVX__

  70.         out_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  71. #else

  72.         out_elempack = num_output % 4 == 0 ? 4 : 1;

  73. #endif

  74.     }

  75. #endif // __SSE2__

  76. 

  77.     if (out_elempack != 1)

  78.     {

  79.         // src = inch-outch

  80.         // dst = pb-inch-outch/pb

  81.         {

  82.             Mat weight_data_r2 = weight_data.reshape(num_input, num_output);

  83. 

  84.             weight_data_tm.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack);

  85. 

  86.             for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)

  87.             {

  88.                 float* g0 = weight_data_tm.row(q / out_elempack);

  89. 

  90.                 for (int p = 0; p < num_input; p++)

  91.                 {

  92.                     for (int j = 0; j < out_elempack; j++)

  93.                     {

  94.                         *g0++ = weight_data_r2.row(q + j)[p];

  95.                     }

  96.                 }

  97.             }

  98.         }

  99.     }

  100.     else

  101.     {

  102.         weight_data_tm = weight_data;

  103.     }

  104. 

  105.     if (opt.lightmode)

  106.     {

  107.         weight_data.release();

  108.     }

  109. 

  110.     return 0;

  111. }

  112. 

  113. int InnerProduct_x86::destroy_pipeline(const Option& opt)

  114. {

  115.     if (flatten)

  116.     {

  117.         flatten->destroy_pipeline(opt);

  118.         delete flatten;

  119.         flatten = 0;

  120.     }

  121. 

  122.     return 0;

  123. }

  124. 

  125. int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  126. {

  127. #if NCNN_INT8

  128.     if (opt.use_int8_inference && int8_scale_term)

  129.     {

  130.         return forward_int8_x86(bottom_blob, top_blob, opt);

  131.     }

  132. #endif

  133. 

  134.     const int num_input = weight_data_size / num_output;

  135. 

  136.     if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)

  137.     {

  138.         // gemm

  139.         int h = bottom_blob.h;

  140.         size_t elemsize = bottom_blob.elemsize;

  141.         int elempack = bottom_blob.elempack;

  142. 

  143.         top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator);

  144.         if (top_blob.empty())

  145.             return -100;

  146. 

  147.         int num_output_elempack = 1;

  148. #if __SSE2__

  149.         if (opt.use_packing_layout)

  150.         {

  151. #if __AVX512F__

  152.             num_output_elempack = num_output % 16 == 0 ? 16 : num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  153. #elif __AVX__

  154.             num_output_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  155. #else

  156.             num_output_elempack = num_output % 4 == 0 ? 4 : 1;

  157. #endif

  158.         }

  159. #endif // __SSE2__

  160. 

  161.         #pragma omp parallel for num_threads(opt.num_threads)

  162.         for (int j = 0; j < h; j++)

  163.         {

  164. #if __SSE2__

  165. #if __AVX__

  166. #if __AVX512F__

  167.             if (elempack == 16 && num_output_elempack == 16)

  168.             {

  169.                 float* outptr = top_blob.row(j);

  170. 

  171.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  172.                 {

  173.                     const float* kptr = weight_data_tm.row(p);

  174.                     const float* m = bottom_blob.row(j);

  175. 

  176.                     __m512 _sum0 = _mm512_set1_ps(0.f);

  177.                     __m512 _sum1 = _mm512_set1_ps(0.f);

  178.                     __m512 _sum2 = _mm512_set1_ps(0.f);

  179.                     __m512 _sum3 = _mm512_set1_ps(0.f);

  180.                     __m512 _sum4 = _mm512_set1_ps(0.f);

  181.                     __m512 _sum5 = _mm512_set1_ps(0.f);

  182.                     __m512 _sum6 = _mm512_set1_ps(0.f);

  183.                     __m512 _sum7 = _mm512_set1_ps(0.f);

  184.                     __m512 _sum8 = _mm512_set1_ps(0.f);

  185.                     __m512 _sum9 = _mm512_set1_ps(0.f);

  186.                     __m512 _suma = _mm512_set1_ps(0.f);

  187.                     __m512 _sumb = _mm512_set1_ps(0.f);

  188.                     __m512 _sumc = _mm512_set1_ps(0.f);

  189.                     __m512 _sumd = _mm512_set1_ps(0.f);

  190.                     __m512 _sume = _mm512_set1_ps(0.f);

  191.                     __m512 _sumf = _mm512_set1_ps(0.f);

  192. 

  193.                     if (bias_term)

  194.                     {

  195.                         _sum0 = _mm512_set1_ps(bias_data[p * 16 + 0]);

  196.                         _sum1 = _mm512_set1_ps(bias_data[p * 16 + 1]);

  197.                         _sum2 = _mm512_set1_ps(bias_data[p * 16 + 2]);

  198.                         _sum3 = _mm512_set1_ps(bias_data[p * 16 + 3]);

  199.                         _sum4 = _mm512_set1_ps(bias_data[p * 16 + 4]);

  200.                         _sum5 = _mm512_set1_ps(bias_data[p * 16 + 5]);

  201.                         _sum6 = _mm512_set1_ps(bias_data[p * 16 + 6]);

  202.                         _sum7 = _mm512_set1_ps(bias_data[p * 16 + 7]);

  203.                         _sum8 = _mm512_set1_ps(bias_data[p * 16 + 8]);

  204.                         _sum9 = _mm512_set1_ps(bias_data[p * 16 + 9]);

  205.                         _suma = _mm512_set1_ps(bias_data[p * 16 + 10]);

  206.                         _sumb = _mm512_set1_ps(bias_data[p * 16 + 11]);

  207.                         _sumc = _mm512_set1_ps(bias_data[p * 16 + 12]);

  208.                         _sumd = _mm512_set1_ps(bias_data[p * 16 + 13]);

  209.                         _sume = _mm512_set1_ps(bias_data[p * 16 + 14]);

  210.                         _sumf = _mm512_set1_ps(bias_data[p * 16 + 15]);

  211.                     }

  212. 

  213.                     for (int i = 0; i < num_input; i++)

  214.                     {

  215.                         __m512 _val = _mm512_loadu_ps(m);

  216.                         _sum0 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[0]), _sum0);

  217.                         _sum1 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[1]), _sum1);

  218.                         _sum2 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[2]), _sum2);

  219.                         _sum3 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[3]), _sum3);

  220.                         _sum4 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[4]), _sum4);

  221.                         _sum5 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[5]), _sum5);

  222.                         _sum6 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[6]), _sum6);

  223.                         _sum7 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[7]), _sum7);

  224.                         _sum8 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[8]), _sum8);

  225.                         _sum9 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[9]), _sum9);

  226.                         _suma = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[10]), _suma);

  227.                         _sumb = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[11]), _sumb);

  228.                         _sumc = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[12]), _sumc);

  229.                         _sumd = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[13]), _sumd);

  230.                         _sume = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[14]), _sume);

  231.                         _sumf = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[15]), _sumf);

  232. 

  233.                         m += 16;

  234.                         kptr += 16;

  235.                     }

  236. 

  237.                     _sum0 = activation_avx512(_sum0, activation_type, activation_params);

  238.                     _sum1 = activation_avx512(_sum1, activation_type, activation_params);

  239.                     _sum2 = activation_avx512(_sum2, activation_type, activation_params);

  240.                     _sum3 = activation_avx512(_sum3, activation_type, activation_params);

  241.                     _sum4 = activation_avx512(_sum4, activation_type, activation_params);

  242.                     _sum5 = activation_avx512(_sum5, activation_type, activation_params);

  243.                     _sum6 = activation_avx512(_sum6, activation_type, activation_params);

  244.                     _sum7 = activation_avx512(_sum7, activation_type, activation_params);

  245.                     _sum8 = activation_avx512(_sum8, activation_type, activation_params);

  246.                     _sum9 = activation_avx512(_sum9, activation_type, activation_params);

  247.                     _suma = activation_avx512(_suma, activation_type, activation_params);

  248.                     _sumb = activation_avx512(_sumb, activation_type, activation_params);

  249.                     _sumc = activation_avx512(_sumc, activation_type, activation_params);

  250.                     _sumd = activation_avx512(_sumd, activation_type, activation_params);

  251.                     _sume = activation_avx512(_sume, activation_type, activation_params);

  252.                     _sumf = activation_avx512(_sumf, activation_type, activation_params);

  253. 

  254.                     _mm512_storeu_ps(outptr, _sum0);

  255.                     _mm512_storeu_ps(outptr + 16, _sum1);

  256.                     _mm512_storeu_ps(outptr + 16 * 2, _sum2);

  257.                     _mm512_storeu_ps(outptr + 16 * 3, _sum3);

  258.                     _mm512_storeu_ps(outptr + 16 * 4, _sum4);

  259.                     _mm512_storeu_ps(outptr + 16 * 5, _sum5);

  260.                     _mm512_storeu_ps(outptr + 16 * 6, _sum6);

  261.                     _mm512_storeu_ps(outptr + 16 * 7, _sum7);

  262.                     _mm512_storeu_ps(outptr + 16 * 8, _sum8);

  263.                     _mm512_storeu_ps(outptr + 16 * 9, _sum9);

  264.                     _mm512_storeu_ps(outptr + 16 * 10, _suma);

  265.                     _mm512_storeu_ps(outptr + 16 * 11, _sumb);

  266.                     _mm512_storeu_ps(outptr + 16 * 12, _sumc);

  267.                     _mm512_storeu_ps(outptr + 16 * 13, _sumd);

  268.                     _mm512_storeu_ps(outptr + 16 * 14, _sume);

  269.                     _mm512_storeu_ps(outptr + 16 * 15, _sumf);

  270.                     outptr += 256;

  271.                 }

  272.             }

  273. 

  274.             if (elempack == 1 && num_output_elempack == 16)

  275.             {

  276.                 float* outptr = top_blob.row(j);

  277. 

  278.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  279.                 {

  280.                     const float* kptr = weight_data_tm.row(p);

  281.                     const float* m = bottom_blob.row(j);

  282. 

  283.                     __m512 _sum = _mm512_set1_ps(0.f);

  284. 

  285.                     if (bias_term)

  286.                     {

  287.                         _sum = _mm512_loadu_ps((const float*)bias_data + p * 16);

  288.                     }

  289. 

  290.                     int i = 0;

  291.                     for (; i < num_input; i++)

  292.                     {

  293.                         __m512 _val = _mm512_set1_ps(m[0]);

  294.                         __m512 _w = _mm512_loadu_ps(kptr);

  295.                         _sum = _mm512_fmadd_ps(_val, _w, _sum);

  296. 

  297.                         m += 1;

  298.                         kptr += 16;

  299.                     }

  300. 

  301.                     _sum = activation_avx512(_sum, activation_type, activation_params);

  302. 

  303.                     _mm512_storeu_ps(outptr, _sum);

  304.                     outptr += 16;

  305.                 }

  306.             }

  307. 

  308.             if (elempack == 4 && num_output_elempack == 16)

  309.             {

  310.                 float* outptr = top_blob.row(j);

  311. 

  312.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  313.                 {

  314.                     const float* kptr = weight_data_tm.row(p);

  315.                     const float* m = bottom_blob.row(j);

  316. 

  317.                     __m128 _sum0 = _mm_set1_ps(0.f);

  318.                     __m128 _sum1 = _mm_set1_ps(0.f);

  319.                     __m128 _sum2 = _mm_set1_ps(0.f);

  320.                     __m128 _sum3 = _mm_set1_ps(0.f);

  321.                     __m128 _sum4 = _mm_set1_ps(0.f);

  322.                     __m128 _sum5 = _mm_set1_ps(0.f);

  323.                     __m128 _sum6 = _mm_set1_ps(0.f);

  324.                     __m128 _sum7 = _mm_set1_ps(0.f);

  325.                     __m128 _sum8 = _mm_set1_ps(0.f);

  326.                     __m128 _sum9 = _mm_set1_ps(0.f);

  327.                     __m128 _suma = _mm_set1_ps(0.f);

  328.                     __m128 _sumb = _mm_set1_ps(0.f);

  329.                     __m128 _sumc = _mm_set1_ps(0.f);

  330.                     __m128 _sumd = _mm_set1_ps(0.f);

  331.                     __m128 _sume = _mm_set1_ps(0.f);

  332.                     __m128 _sumf = _mm_set1_ps(0.f);

  333. 

  334.                     if (bias_term)

  335.                     {

  336.                         _sum0 = _mm_set1_ps(bias_data[p * 16 + 0]);

  337.                         _sum1 = _mm_set1_ps(bias_data[p * 16 + 1]);

  338.                         _sum2 = _mm_set1_ps(bias_data[p * 16 + 2]);

  339.                         _sum3 = _mm_set1_ps(bias_data[p * 16 + 3]);

  340.                         _sum4 = _mm_set1_ps(bias_data[p * 16 + 4]);

  341.                         _sum5 = _mm_set1_ps(bias_data[p * 16 + 5]);

  342.                         _sum6 = _mm_set1_ps(bias_data[p * 16 + 6]);

  343.                         _sum7 = _mm_set1_ps(bias_data[p * 16 + 7]);

  344.                         _sum8 = _mm_set1_ps(bias_data[p * 16 + 8]);

  345.                         _sum9 = _mm_set1_ps(bias_data[p * 16 + 9]);

  346.                         _suma = _mm_set1_ps(bias_data[p * 16 + 10]);

  347.                         _sumb = _mm_set1_ps(bias_data[p * 16 + 11]);

  348.                         _sumc = _mm_set1_ps(bias_data[p * 16 + 12]);

  349.                         _sumd = _mm_set1_ps(bias_data[p * 16 + 13]);

  350.                         _sume = _mm_set1_ps(bias_data[p * 16 + 14]);

  351.                         _sumf = _mm_set1_ps(bias_data[p * 16 + 15]);

  352.                     }

  353. 

  354.                     int i = 0;

  355.                     for (; i < num_input; i++)

  356.                     {

  357.                         __m128 _val = _mm_loadu_ps(m);

  358.                         _sum0 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[0]), _sum0);

  359.                         _sum1 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[1]), _sum1);

  360.                         _sum2 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[2]), _sum2);

  361.                         _sum3 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[3]), _sum3);

  362.                         _sum4 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[4]), _sum4);

  363.                         _sum5 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[5]), _sum5);

  364.                         _sum6 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[6]), _sum6);

  365.                         _sum7 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[7]), _sum7);

  366.                         _sum8 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[8]), _sum8);

  367.                         _sum9 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[9]), _sum9);

  368.                         _suma = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[10]), _suma);

  369.                         _sumb = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[11]), _sumb);

  370.                         _sumc = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[12]), _sumc);

  371.                         _sumd = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[13]), _sumd);

  372.                         _sume = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[14]), _sume);

  373.                         _sumf = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[15]), _sumf);

  374. 

  375.                         m += 4;

  376.                         kptr += 16;

  377.                     }

  378. 

  379.                     _sum0 = activation_sse(_sum0, activation_type, activation_params);

  380.                     _sum1 = activation_sse(_sum1, activation_type, activation_params);

  381.                     _sum2 = activation_sse(_sum2, activation_type, activation_params);

  382.                     _sum3 = activation_sse(_sum3, activation_type, activation_params);

  383.                     _sum4 = activation_sse(_sum4, activation_type, activation_params);

  384.                     _sum5 = activation_sse(_sum5, activation_type, activation_params);

  385.                     _sum6 = activation_sse(_sum6, activation_type, activation_params);

  386.                     _sum7 = activation_sse(_sum7, activation_type, activation_params);

  387.                     _sum8 = activation_sse(_sum8, activation_type, activation_params);

  388.                     _sum9 = activation_sse(_sum9, activation_type, activation_params);

  389.                     _suma = activation_sse(_suma, activation_type, activation_params);

  390.                     _sumb = activation_sse(_sumb, activation_type, activation_params);

  391.                     _sumc = activation_sse(_sumc, activation_type, activation_params);

  392.                     _sumd = activation_sse(_sumd, activation_type, activation_params);

  393.                     _sume = activation_sse(_sume, activation_type, activation_params);

  394.                     _sumf = activation_sse(_sumf, activation_type, activation_params);

  395. 

  396.                     _mm_storeu_ps(outptr, _sum0);

  397.                     _mm_storeu_ps(outptr + 4, _sum1);

  398.                     _mm_storeu_ps(outptr + 4 * 2, _sum2);

  399.                     _mm_storeu_ps(outptr + 4 * 3, _sum3);

  400.                     _mm_storeu_ps(outptr + 4 * 4, _sum4);

  401.                     _mm_storeu_ps(outptr + 4 * 5, _sum5);

  402.                     _mm_storeu_ps(outptr + 4 * 6, _sum6);

  403.                     _mm_storeu_ps(outptr + 4 * 7, _sum7);

  404.                     _mm_storeu_ps(outptr + 4 * 8, _sum8);

  405.                     _mm_storeu_ps(outptr + 4 * 9, _sum9);

  406.                     _mm_storeu_ps(outptr + 4 * 10, _suma);

  407.                     _mm_storeu_ps(outptr + 4 * 11, _sumb);

  408.                     _mm_storeu_ps(outptr + 4 * 12, _sumc);

  409.                     _mm_storeu_ps(outptr + 4 * 13, _sumd);

  410.                     _mm_storeu_ps(outptr + 4 * 14, _sume);

  411.                     _mm_storeu_ps(outptr + 4 * 15, _sumf);

  412.                     outptr += 64;

  413.                 }

  414.             }

  415. 

  416.             if (elempack == 8 && num_output_elempack == 16)

  417.             {

  418.                 float* outptr = top_blob.row(j);

  419. 

  420.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  421.                 {

  422.                     const float* kptr = weight_data_tm.row(p);

  423.                     const float* m = bottom_blob.row(j);

  424. 

  425.                     __m256 _sum0 = _mm256_set1_ps(0.f);

  426.                     __m256 _sum1 = _mm256_set1_ps(0.f);

  427.                     __m256 _sum2 = _mm256_set1_ps(0.f);

  428.                     __m256 _sum3 = _mm256_set1_ps(0.f);

  429.                     __m256 _sum4 = _mm256_set1_ps(0.f);

  430.                     __m256 _sum5 = _mm256_set1_ps(0.f);

  431.                     __m256 _sum6 = _mm256_set1_ps(0.f);

  432.                     __m256 _sum7 = _mm256_set1_ps(0.f);

  433.                     __m256 _sum8 = _mm256_set1_ps(0.f);

  434.                     __m256 _sum9 = _mm256_set1_ps(0.f);

  435.                     __m256 _suma = _mm256_set1_ps(0.f);

  436.                     __m256 _sumb = _mm256_set1_ps(0.f);

  437.                     __m256 _sumc = _mm256_set1_ps(0.f);

  438.                     __m256 _sumd = _mm256_set1_ps(0.f);

  439.                     __m256 _sume = _mm256_set1_ps(0.f);

  440.                     __m256 _sumf = _mm256_set1_ps(0.f);

  441. 

  442.                     if (bias_term)

  443.                     {

  444.                         _sum0 = _mm256_set1_ps(bias_data[p * 16 + 0]);

  445.                         _sum1 = _mm256_set1_ps(bias_data[p * 16 + 1]);

  446.                         _sum2 = _mm256_set1_ps(bias_data[p * 16 + 2]);

  447.                         _sum3 = _mm256_set1_ps(bias_data[p * 16 + 3]);

  448.                         _sum4 = _mm256_set1_ps(bias_data[p * 16 + 4]);

  449.                         _sum5 = _mm256_set1_ps(bias_data[p * 16 + 5]);

  450.                         _sum6 = _mm256_set1_ps(bias_data[p * 16 + 6]);

  451.                         _sum7 = _mm256_set1_ps(bias_data[p * 16 + 7]);

  452.                         _sum8 = _mm256_set1_ps(bias_data[p * 16 + 8]);

  453.                         _sum9 = _mm256_set1_ps(bias_data[p * 16 + 9]);

  454.                         _suma = _mm256_set1_ps(bias_data[p * 16 + 10]);

  455.                         _sumb = _mm256_set1_ps(bias_data[p * 16 + 11]);

  456.                         _sumc = _mm256_set1_ps(bias_data[p * 16 + 12]);

  457.                         _sumd = _mm256_set1_ps(bias_data[p * 16 + 13]);

  458.                         _sume = _mm256_set1_ps(bias_data[p * 16 + 14]);

  459.                         _sumf = _mm256_set1_ps(bias_data[p * 16 + 15]);

  460.                     }

  461. 

  462.                     int i = 0;

  463.                     for (; i < num_input; i++)

  464.                     {

  465.                         __m256 _val = _mm256_loadu_ps(m);

  466.                         _sum0 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[0]), _sum0);

  467.                         _sum1 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[1]), _sum1);

  468.                         _sum2 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[2]), _sum2);

  469.                         _sum3 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[3]), _sum3);

  470.                         _sum4 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[4]), _sum4);

  471.                         _sum5 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[5]), _sum5);

  472.                         _sum6 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[6]), _sum6);

  473.                         _sum7 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[7]), _sum7);

  474.                         _sum8 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[8]), _sum8);

  475.                         _sum9 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[9]), _sum9);

  476.                         _suma = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[10]), _suma);

  477.                         _sumb = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[11]), _sumb);

  478.                         _sumc = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[12]), _sumc);

  479.                         _sumd = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[13]), _sumd);

  480.                         _sume = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[14]), _sume);

  481.                         _sumf = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[15]), _sumf);

  482. 

  483.                         m += 8;

  484.                         kptr += 16;

  485.                     }

  486. 

  487.                     _sum0 = activation_avx(_sum0, activation_type, activation_params);

  488.                     _sum1 = activation_avx(_sum1, activation_type, activation_params);

  489.                     _sum2 = activation_avx(_sum2, activation_type, activation_params);

  490.                     _sum3 = activation_avx(_sum3, activation_type, activation_params);

  491.                     _sum4 = activation_avx(_sum4, activation_type, activation_params);

  492.                     _sum5 = activation_avx(_sum5, activation_type, activation_params);

  493.                     _sum6 = activation_avx(_sum6, activation_type, activation_params);

  494.                     _sum7 = activation_avx(_sum7, activation_type, activation_params);

  495.                     _sum8 = activation_avx(_sum8, activation_type, activation_params);

  496.                     _sum9 = activation_avx(_sum9, activation_type, activation_params);

  497.                     _suma = activation_avx(_suma, activation_type, activation_params);

  498.                     _sumb = activation_avx(_sumb, activation_type, activation_params);

  499.                     _sumc = activation_avx(_sumc, activation_type, activation_params);

  500.                     _sumd = activation_avx(_sumd, activation_type, activation_params);

  501.                     _sume = activation_avx(_sume, activation_type, activation_params);

  502.                     _sumf = activation_avx(_sumf, activation_type, activation_params);

  503. 

  504.                     _mm256_storeu_ps(outptr, _sum0);

  505.                     _mm256_storeu_ps(outptr + 8, _sum1);

  506.                     _mm256_storeu_ps(outptr + 8 * 2, _sum2);

  507.                     _mm256_storeu_ps(outptr + 8 * 3, _sum3);

  508.                     _mm256_storeu_ps(outptr + 8 * 4, _sum4);

  509.                     _mm256_storeu_ps(outptr + 8 * 5, _sum5);

  510.                     _mm256_storeu_ps(outptr + 8 * 6, _sum6);

  511.                     _mm256_storeu_ps(outptr + 8 * 7, _sum7);

  512.                     _mm256_storeu_ps(outptr + 8 * 8, _sum8);

  513.                     _mm256_storeu_ps(outptr + 8 * 9, _sum9);

  514.                     _mm256_storeu_ps(outptr + 8 * 10, _suma);

  515.                     _mm256_storeu_ps(outptr + 8 * 11, _sumb);

  516.                     _mm256_storeu_ps(outptr + 8 * 12, _sumc);

  517.                     _mm256_storeu_ps(outptr + 8 * 13, _sumd);

  518.                     _mm256_storeu_ps(outptr + 8 * 14, _sume);

  519.                     _mm256_storeu_ps(outptr + 8 * 15, _sumf);

  520.                     outptr += 128;

  521.                 }

  522.             }

  523. 

  524.             if (elempack == 16 && num_output_elempack == 1)

  525.             {

  526.                 float* outptr = top_blob.row(j);

  527. 

  528.                 for (int p = 0; p < num_output; p++)

  529.                 {

  530.                     const float* kptr = (const float*)weight_data_tm + num_input * p;

  531.                     const float* m = bottom_blob.row(j);

  532. 

  533.                     __m512 _sum0 = _mm512_set1_ps(0.f);

  534. 

  535.                     if (bias_term)

  536.                     {

  537.                         _sum0 = _mm512_set1_ps(bias_data[p]);

  538.                     }

  539. 

  540.                     int i = 0;

  541.                     for (; i < num_input; i++)

  542.                     {

  543.                         __m512 _val = _mm512_loadu_ps(m);

  544.                         __m512 _k = _mm512_set1_ps(kptr[0]);

  545.                         _sum0 = _mm512_fmadd_ps(_val, _k, _sum0);

  546. 

  547.                         m += 16;

  548.                         kptr += 1;

  549.                     }

  550. 

  551.                     _sum0 = activation_avx512(_sum0, activation_type, activation_params);

  552. 

  553.                     _mm512_storeu_ps(outptr, _sum0);

  554.                     outptr += 16;

  555.                 }

  556.             }

  557. 

  558.             if (elempack == 16 && num_output_elempack == 4)

  559.             {

  560.                 float* outptr = top_blob.row(j);

  561. 

  562.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  563.                 {

  564.                     const float* kptr = weight_data_tm.row(p);

  565.                     const float* m = bottom_blob.row(j);

  566. 

  567.                     __m512 _sum0 = _mm512_set1_ps(0.f);

  568.                     __m512 _sum1 = _mm512_set1_ps(0.f);

  569.                     __m512 _sum2 = _mm512_set1_ps(0.f);

  570.                     __m512 _sum3 = _mm512_set1_ps(0.f);

  571. 

  572.                     if (bias_term)

  573.                     {

  574.                         _sum0 = _mm512_set1_ps(bias_data[p * 4 + 0]);

  575.                         _sum1 = _mm512_set1_ps(bias_data[p * 4 + 1]);

  576.                         _sum2 = _mm512_set1_ps(bias_data[p * 4 + 2]);

  577.                         _sum3 = _mm512_set1_ps(bias_data[p * 4 + 3]);

  578.                     }

  579. 

  580.                     int i = 0;

  581.                     for (; i < num_input; i++)

  582.                     {

  583.                         __m512 _val = _mm512_loadu_ps(m);

  584.                         _sum0 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[0]), _sum0);

  585.                         _sum1 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[1]), _sum1);

  586.                         _sum2 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[2]), _sum2);

  587.                         _sum3 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[3]), _sum3);

  588. 

  589.                         m += 16;

  590.                         kptr += 4;

  591.                     }

  592. 

  593.                     _sum0 = activation_avx512(_sum0, activation_type, activation_params);

  594.                     _sum1 = activation_avx512(_sum1, activation_type, activation_params);

  595.                     _sum2 = activation_avx512(_sum2, activation_type, activation_params);

  596.                     _sum3 = activation_avx512(_sum3, activation_type, activation_params);

  597. 

  598.                     _mm512_storeu_ps(outptr, _sum0);

  599.                     _mm512_storeu_ps(outptr + 16, _sum1);

  600.                     _mm512_storeu_ps(outptr + 32, _sum2);

  601.                     _mm512_storeu_ps(outptr + 48, _sum3);

  602.                     outptr += 64;

  603.                 }

  604.             }

  605. 

  606.             if (elempack == 16 && num_output_elempack == 8)

  607.             {

  608.                 float* outptr = top_blob.row(j);

  609. 

  610.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  611.                 {

  612.                     const float* kptr = weight_data_tm.row(p);

  613.                     const float* m = bottom_blob.row(j);

  614. 

  615.                     __m512 _sum0 = _mm512_set1_ps(0.f);

  616.                     __m512 _sum1 = _mm512_set1_ps(0.f);

  617.                     __m512 _sum2 = _mm512_set1_ps(0.f);

  618.                     __m512 _sum3 = _mm512_set1_ps(0.f);

  619.                     __m512 _sum4 = _mm512_set1_ps(0.f);

  620.                     __m512 _sum5 = _mm512_set1_ps(0.f);

  621.                     __m512 _sum6 = _mm512_set1_ps(0.f);

  622.                     __m512 _sum7 = _mm512_set1_ps(0.f);

  623. 

  624.                     if (bias_term)

  625.                     {

  626.                         _sum0 = _mm512_set1_ps(bias_data[p * 8 + 0]);

  627.                         _sum1 = _mm512_set1_ps(bias_data[p * 8 + 1]);

  628.                         _sum2 = _mm512_set1_ps(bias_data[p * 8 + 2]);

  629.                         _sum3 = _mm512_set1_ps(bias_data[p * 8 + 3]);

  630.                         _sum4 = _mm512_set1_ps(bias_data[p * 8 + 4]);

  631.                         _sum5 = _mm512_set1_ps(bias_data[p * 8 + 5]);

  632.                         _sum6 = _mm512_set1_ps(bias_data[p * 8 + 6]);

  633.                         _sum7 = _mm512_set1_ps(bias_data[p * 8 + 7]);

  634.                     }

  635. 

  636.                     int i = 0;

  637.                     for (; i < num_input; i++)

  638.                     {

  639.                         __m512 _val = _mm512_loadu_ps(m);

  640.                         _sum0 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[0]), _sum0);

  641.                         _sum1 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[1]), _sum1);

  642.                         _sum2 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[2]), _sum2);

  643.                         _sum3 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[3]), _sum3);

  644.                         _sum4 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[4]), _sum4);

  645.                         _sum5 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[5]), _sum5);

  646.                         _sum6 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[6]), _sum6);

  647.                         _sum7 = _mm512_fmadd_ps(_val, _mm512_set1_ps(kptr[7]), _sum7);

  648. 

  649.                         m += 16;

  650.                         kptr += 8;

  651.                     }

  652. 

  653.                     _sum0 = activation_avx512(_sum0, activation_type, activation_params);

  654.                     _sum1 = activation_avx512(_sum1, activation_type, activation_params);

  655.                     _sum2 = activation_avx512(_sum2, activation_type, activation_params);

  656.                     _sum3 = activation_avx512(_sum3, activation_type, activation_params);

  657.                     _sum4 = activation_avx512(_sum4, activation_type, activation_params);

  658.                     _sum5 = activation_avx512(_sum5, activation_type, activation_params);

  659.                     _sum6 = activation_avx512(_sum6, activation_type, activation_params);

  660.                     _sum7 = activation_avx512(_sum7, activation_type, activation_params);

  661. 

  662.                     _mm512_storeu_ps(outptr, _sum0);

  663.                     _mm512_storeu_ps(outptr + 16, _sum1);

  664.                     _mm512_storeu_ps(outptr + 16 * 2, _sum2);

  665.                     _mm512_storeu_ps(outptr + 16 * 3, _sum3);

  666.                     _mm512_storeu_ps(outptr + 16 * 4, _sum4);

  667.                     _mm512_storeu_ps(outptr + 16 * 5, _sum5);

  668.                     _mm512_storeu_ps(outptr + 16 * 6, _sum6);

  669.                     _mm512_storeu_ps(outptr + 16 * 7, _sum7);

  670.                     outptr += 128;

  671.                 }

  672.             }

  673. 

  674. #endif // __AVX512F__

  675. 

  676.             if (elempack == 8 && num_output_elempack == 8)

  677.             {

  678.                 float* outptr = top_blob.row(j);

  679. 

  680.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  681.                 {

  682.                     const float* kptr = weight_data_tm.row(p);

  683.                     const float* m = bottom_blob.row(j);

  684. 

  685.                     __m256 _sum0 = _mm256_set1_ps(0.f);

  686.                     __m256 _sum1 = _mm256_set1_ps(0.f);

  687.                     __m256 _sum2 = _mm256_set1_ps(0.f);

  688.                     __m256 _sum3 = _mm256_set1_ps(0.f);

  689.                     __m256 _sum4 = _mm256_set1_ps(0.f);

  690.                     __m256 _sum5 = _mm256_set1_ps(0.f);

  691.                     __m256 _sum6 = _mm256_set1_ps(0.f);

  692.                     __m256 _sum7 = _mm256_set1_ps(0.f);

  693. 

  694.                     if (bias_term)

  695.                     {

  696.                         _sum0 = _mm256_set1_ps(bias_data[p * 8 + 0]);

  697.                         _sum1 = _mm256_set1_ps(bias_data[p * 8 + 1]);

  698.                         _sum2 = _mm256_set1_ps(bias_data[p * 8 + 2]);

  699.                         _sum3 = _mm256_set1_ps(bias_data[p * 8 + 3]);

  700.                         _sum4 = _mm256_set1_ps(bias_data[p * 8 + 4]);

  701.                         _sum5 = _mm256_set1_ps(bias_data[p * 8 + 5]);

  702.                         _sum6 = _mm256_set1_ps(bias_data[p * 8 + 6]);

  703.                         _sum7 = _mm256_set1_ps(bias_data[p * 8 + 7]);

  704.                     }

  705. 

  706.                     for (int i = 0; i < num_input; i++)

  707.                     {

  708.                         __m256 _val = _mm256_loadu_ps(m);

  709.                         __m256 _k0 = _mm256_set1_ps(kptr[0]);

  710.                         __m256 _k1 = _mm256_set1_ps(kptr[1]);

  711.                         __m256 _k2 = _mm256_set1_ps(kptr[2]);

  712.                         __m256 _k3 = _mm256_set1_ps(kptr[3]);

  713.                         __m256 _k4 = _mm256_set1_ps(kptr[4]);

  714.                         __m256 _k5 = _mm256_set1_ps(kptr[5]);

  715.                         __m256 _k6 = _mm256_set1_ps(kptr[6]);

  716.                         __m256 _k7 = _mm256_set1_ps(kptr[7]);

  717.                         _sum0 = _mm256_comp_fmadd_ps(_val, _k0, _sum0);

  718.                         _sum1 = _mm256_comp_fmadd_ps(_val, _k1, _sum1);

  719.                         _sum2 = _mm256_comp_fmadd_ps(_val, _k2, _sum2);

  720.                         _sum3 = _mm256_comp_fmadd_ps(_val, _k3, _sum3);

  721.                         _sum4 = _mm256_comp_fmadd_ps(_val, _k4, _sum4);

  722.                         _sum5 = _mm256_comp_fmadd_ps(_val, _k5, _sum5);

  723.                         _sum6 = _mm256_comp_fmadd_ps(_val, _k6, _sum6);

  724.                         _sum7 = _mm256_comp_fmadd_ps(_val, _k7, _sum7);

  725. 

  726.                         m += 8;

  727.                         kptr += 8;

  728.                     }

  729. 

  730.                     _sum0 = activation_avx(_sum0, activation_type, activation_params);

  731.                     _sum1 = activation_avx(_sum1, activation_type, activation_params);

  732.                     _sum2 = activation_avx(_sum2, activation_type, activation_params);

  733.                     _sum3 = activation_avx(_sum3, activation_type, activation_params);

  734.                     _sum4 = activation_avx(_sum4, activation_type, activation_params);

  735.                     _sum5 = activation_avx(_sum5, activation_type, activation_params);

  736.                     _sum6 = activation_avx(_sum6, activation_type, activation_params);

  737.                     _sum7 = activation_avx(_sum7, activation_type, activation_params);

  738. 

  739.                     _mm256_storeu_ps(outptr, _sum0);

  740.                     _mm256_storeu_ps(outptr + 8, _sum1);

  741.                     _mm256_storeu_ps(outptr + 16, _sum2);

  742.                     _mm256_storeu_ps(outptr + 24, _sum3);

  743.                     _mm256_storeu_ps(outptr + 32, _sum4);

  744.                     _mm256_storeu_ps(outptr + 40, _sum5);

  745.                     _mm256_storeu_ps(outptr + 48, _sum6);

  746.                     _mm256_storeu_ps(outptr + 56, _sum7);

  747.                     outptr += 64;

  748.                 }

  749.             }

  750. 

  751.             if (elempack == 1 && num_output_elempack == 8)

  752.             {

  753.                 float* outptr = top_blob.row(j);

  754. 

  755.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  756.                 {

  757.                     const float* kptr = weight_data_tm.row(p);

  758.                     const float* m = bottom_blob.row(j);

  759. 

  760.                     __m256 _sum = _mm256_set1_ps(0.f);

  761. 

  762.                     if (bias_term)

  763.                     {

  764.                         _sum = _mm256_loadu_ps((const float*)bias_data + p * 8);

  765.                     }

  766. 

  767.                     int i = 0;

  768.                     for (; i + 7 < num_input; i += 8)

  769.                     {

  770.                         __m256 _val0 = _mm256_broadcast_ss(m);

  771.                         __m256 _val1 = _mm256_broadcast_ss(m + 1);

  772.                         __m256 _val2 = _mm256_broadcast_ss(m + 2);

  773.                         __m256 _val3 = _mm256_broadcast_ss(m + 3);

  774.                         __m256 _val4 = _mm256_broadcast_ss(m + 4);

  775.                         __m256 _val5 = _mm256_broadcast_ss(m + 5);

  776.                         __m256 _val6 = _mm256_broadcast_ss(m + 6);

  777.                         __m256 _val7 = _mm256_broadcast_ss(m + 7);

  778. 

  779.                         __m256 _w0 = _mm256_loadu_ps(kptr);

  780.                         _sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum);

  781.                         __m256 _w1 = _mm256_loadu_ps(kptr + 8);

  782.                         _sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum);

  783.                         __m256 _w2 = _mm256_loadu_ps(kptr + 16);

  784.                         _sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum);

  785.                         __m256 _w3 = _mm256_loadu_ps(kptr + 24);

  786.                         _sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum);

  787.                         __m256 _w4 = _mm256_loadu_ps(kptr + 32);

  788.                         _sum = _mm256_comp_fmadd_ps(_val4, _w4, _sum);

  789.                         __m256 _w5 = _mm256_loadu_ps(kptr + 40);

  790.                         _sum = _mm256_comp_fmadd_ps(_val5, _w5, _sum);

  791.                         __m256 _w6 = _mm256_loadu_ps(kptr + 48);

  792.                         _sum = _mm256_comp_fmadd_ps(_val6, _w6, _sum);

  793.                         __m256 _w7 = _mm256_loadu_ps(kptr + 56);

  794.                         _sum = _mm256_comp_fmadd_ps(_val7, _w7, _sum);

  795. 

  796.                         m += 8;

  797.                         kptr += 64;

  798.                     }

  799.                     for (; i + 3 < num_input; i += 4)

  800.                     {

  801.                         __m256 _val0 = _mm256_broadcast_ss(m);

  802.                         __m256 _val1 = _mm256_broadcast_ss(m + 1);

  803.                         __m256 _val2 = _mm256_broadcast_ss(m + 2);

  804.                         __m256 _val3 = _mm256_broadcast_ss(m + 3);

  805. 

  806.                         __m256 _w0 = _mm256_loadu_ps(kptr);

  807.                         _sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum);

  808.                         __m256 _w1 = _mm256_loadu_ps(kptr + 8);

  809.                         _sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum);

  810.                         __m256 _w2 = _mm256_loadu_ps(kptr + 16);

  811.                         _sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum);

  812.                         __m256 _w3 = _mm256_loadu_ps(kptr + 24);

  813.                         _sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum);

  814. 

  815.                         m += 4;

  816.                         kptr += 32;

  817.                     }

  818.                     for (; i < num_input; i++)

  819.                     {

  820.                         __m256 _val = _mm256_set1_ps(m[0]);

  821.                         __m256 _w = _mm256_loadu_ps(kptr);

  822.                         _sum = _mm256_comp_fmadd_ps(_val, _w, _sum);

  823. 

  824.                         m += 1;

  825.                         kptr += 8;

  826.                     }

  827. 

  828.                     _sum = activation_avx(_sum, activation_type, activation_params);

  829. 

  830.                     _mm256_storeu_ps(outptr, _sum);

  831.                     outptr += 8;

  832.                 }

  833.             }

  834. 

  835.             if (elempack == 4 && num_output_elempack == 8)

  836.             {

  837.                 float* outptr = top_blob.row(j);

  838. 

  839.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  840.                 {

  841.                     const float* kptr = weight_data_tm.row(p);

  842.                     const float* m = bottom_blob.row(j);

  843. 

  844.                     __m128 _sum0 = _mm_set1_ps(0.f);

  845.                     __m128 _sum1 = _mm_set1_ps(0.f);

  846.                     __m128 _sum2 = _mm_set1_ps(0.f);

  847.                     __m128 _sum3 = _mm_set1_ps(0.f);

  848.                     __m128 _sum4 = _mm_set1_ps(0.f);

  849.                     __m128 _sum5 = _mm_set1_ps(0.f);

  850.                     __m128 _sum6 = _mm_set1_ps(0.f);

  851.                     __m128 _sum7 = _mm_set1_ps(0.f);

  852. 

  853.                     if (bias_term)

  854.                     {

  855.                         _sum0 = _mm_set1_ps(bias_data[p * 8 + 0]);

  856.                         _sum1 = _mm_set1_ps(bias_data[p * 8 + 1]);

  857.                         _sum2 = _mm_set1_ps(bias_data[p * 8 + 2]);

  858.                         _sum3 = _mm_set1_ps(bias_data[p * 8 + 3]);

  859.                         _sum4 = _mm_set1_ps(bias_data[p * 8 + 4]);

  860.                         _sum5 = _mm_set1_ps(bias_data[p * 8 + 5]);

  861.                         _sum6 = _mm_set1_ps(bias_data[p * 8 + 6]);

  862.                         _sum7 = _mm_set1_ps(bias_data[p * 8 + 7]);

  863.                     }

  864. 

  865.                     int i = 0;

  866.                     for (; i < num_input; i++)

  867.                     {

  868.                         __m128 _val = _mm_loadu_ps(m);

  869.                         _sum0 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[0]), _sum0);

  870.                         _sum1 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[1]), _sum1);

  871.                         _sum2 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[2]), _sum2);

  872.                         _sum3 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[3]), _sum3);

  873.                         _sum4 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[4]), _sum4);

  874.                         _sum5 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[5]), _sum5);

  875.                         _sum6 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[6]), _sum6);

  876.                         _sum7 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[7]), _sum7);

  877. 

  878.                         m += 4;

  879.                         kptr += 8;

  880.                     }

  881. 

  882.                     _sum0 = activation_sse(_sum0, activation_type, activation_params);

  883.                     _sum1 = activation_sse(_sum1, activation_type, activation_params);

  884.                     _sum2 = activation_sse(_sum2, activation_type, activation_params);

  885.                     _sum3 = activation_sse(_sum3, activation_type, activation_params);

  886.                     _sum4 = activation_sse(_sum4, activation_type, activation_params);

  887.                     _sum5 = activation_sse(_sum5, activation_type, activation_params);

  888.                     _sum6 = activation_sse(_sum6, activation_type, activation_params);

  889.                     _sum7 = activation_sse(_sum7, activation_type, activation_params);

  890. 

  891.                     _mm_storeu_ps(outptr, _sum0);

  892.                     _mm_storeu_ps(outptr + 4, _sum1);

  893.                     _mm_storeu_ps(outptr + 8, _sum2);

  894.                     _mm_storeu_ps(outptr + 12, _sum3);

  895.                     _mm_storeu_ps(outptr + 16, _sum4);

  896.                     _mm_storeu_ps(outptr + 20, _sum5);

  897.                     _mm_storeu_ps(outptr + 24, _sum6);

  898.                     _mm_storeu_ps(outptr + 28, _sum7);

  899.                     outptr += 32;

  900.                 }

  901.             }

  902. 

  903.             if (elempack == 8 && num_output_elempack == 1)

  904.             {

  905.                 float* outptr = top_blob.row(j);

  906. 

  907.                 for (int p = 0; p < num_output; p++)

  908.                 {

  909.                     const float* kptr = (const float*)weight_data_tm + num_input * p;

  910.                     const float* m = bottom_blob.row(j);

  911. 

  912.                     __m256 _sum0 = _mm256_set1_ps(0.f);

  913.                     __m256 _sum1 = _mm256_set1_ps(0.f);

  914.                     __m256 _sum2 = _mm256_set1_ps(0.f);

  915.                     __m256 _sum3 = _mm256_set1_ps(0.f);

  916. 

  917.                     if (bias_term)

  918.                     {

  919.                         _sum0 = _mm256_set1_ps(bias_data[p]);

  920.                     }

  921. 

  922.                     int i = 0;

  923.                     for (; i + 7 < num_input; i += 8)

  924.                     {

  925.                         __m256 _val0 = _mm256_loadu_ps(m);

  926.                         __m256 _val1 = _mm256_loadu_ps(m + 8);

  927.                         __m256 _val2 = _mm256_loadu_ps(m + 16);

  928.                         __m256 _val3 = _mm256_loadu_ps(m + 24);

  929.                         __m256 _val4 = _mm256_loadu_ps(m + 32);

  930.                         __m256 _val5 = _mm256_loadu_ps(m + 40);

  931.                         __m256 _val6 = _mm256_loadu_ps(m + 48);

  932.                         __m256 _val7 = _mm256_loadu_ps(m + 56);

  933.                         _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0);

  934.                         _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1);

  935.                         _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2);

  936.                         _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3);

  937.                         _sum0 = _mm256_comp_fmadd_ps(_val4, _mm256_set1_ps(kptr[4]), _sum0);

  938.                         _sum1 = _mm256_comp_fmadd_ps(_val5, _mm256_set1_ps(kptr[5]), _sum1);

  939.                         _sum2 = _mm256_comp_fmadd_ps(_val6, _mm256_set1_ps(kptr[6]), _sum2);

  940.                         _sum3 = _mm256_comp_fmadd_ps(_val7, _mm256_set1_ps(kptr[7]), _sum3);

  941. 

  942.                         m += 64;

  943.                         kptr += 8;

  944.                     }

  945.                     for (; i + 3 < num_input; i += 4)

  946.                     {

  947.                         __m256 _val0 = _mm256_loadu_ps(m);

  948.                         __m256 _val1 = _mm256_loadu_ps(m + 8);

  949.                         __m256 _val2 = _mm256_loadu_ps(m + 16);

  950.                         __m256 _val3 = _mm256_loadu_ps(m + 24);

  951.                         _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0);

  952.                         _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1);

  953.                         _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2);

  954.                         _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3);

  955. 

  956.                         m += 32;

  957.                         kptr += 4;

  958.                     }

  959.                     for (; i < num_input; i++)

  960.                     {

  961.                         __m256 _val = _mm256_loadu_ps(m);

  962.                         __m256 _k = _mm256_set1_ps(kptr[0]);

  963.                         _sum0 = _mm256_comp_fmadd_ps(_val, _k, _sum0);

  964. 

  965.                         m += 8;

  966.                         kptr += 1;

  967.                     }

  968. 

  969.                     _sum0 = _mm256_add_ps(_sum0, _sum1);

  970.                     _sum2 = _mm256_add_ps(_sum2, _sum3);

  971.                     _sum0 = _mm256_add_ps(_sum0, _sum2);

  972. 

  973.                     _sum0 = activation_avx(_sum0, activation_type, activation_params);

  974. 

  975.                     _mm256_storeu_ps(outptr, _sum0);

  976.                     outptr += 8;

  977.                 }

  978.             }

  979. 

  980.             if (elempack == 8 && num_output_elempack == 4)

  981.             {

  982.                 float* outptr = top_blob.row(j);

  983. 

  984.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  985.                 {

  986.                     const float* kptr = weight_data_tm.row(p);

  987.                     const float* m = bottom_blob.row(j);

  988. 

  989.                     __m256 _sum0 = _mm256_set1_ps(0.f);

  990.                     __m256 _sum1 = _mm256_set1_ps(0.f);

  991.                     __m256 _sum2 = _mm256_set1_ps(0.f);

  992.                     __m256 _sum3 = _mm256_set1_ps(0.f);

  993. 

  994.                     if (bias_term)

  995.                     {

  996.                         _sum0 = _mm256_set1_ps(bias_data[p * 4 + 0]);

  997.                         _sum1 = _mm256_set1_ps(bias_data[p * 4 + 1]);

  998.                         _sum2 = _mm256_set1_ps(bias_data[p * 4 + 2]);

  999.                         _sum3 = _mm256_set1_ps(bias_data[p * 4 + 3]);

  1000.                     }

  1001. 

  1002.                     int i = 0;

  1003.                     for (; i + 3 < num_input; i += 4)

  1004.                     {

  1005.                         __m256 _val0 = _mm256_loadu_ps(m);

  1006.                         __m256 _val1 = _mm256_loadu_ps(m + 8);

  1007.                         __m256 _val2 = _mm256_loadu_ps(m + 16);

  1008.                         __m256 _val3 = _mm256_loadu_ps(m + 24);

  1009.                         _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0);

  1010.                         _sum1 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[1]), _sum1);

  1011.                         _sum2 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[2]), _sum2);

  1012.                         _sum3 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[3]), _sum3);

  1013.                         _sum0 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[4]), _sum0);

  1014.                         _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[5]), _sum1);

  1015.                         _sum2 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[6]), _sum2);

  1016.                         _sum3 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[7]), _sum3);

  1017.                         kptr += 8;

  1018. 

  1019.                         _sum0 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[0]), _sum0);

  1020.                         _sum1 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[1]), _sum1);

  1021.                         _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2);

  1022.                         _sum3 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[3]), _sum3);

  1023.                         _sum0 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[4]), _sum0);

  1024.                         _sum1 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[5]), _sum1);

  1025.                         _sum2 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[6]), _sum2);

  1026.                         _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[7]), _sum3);

  1027. 

  1028.                         m += 32;

  1029.                         kptr += 8;

  1030.                     }

  1031.                     for (; i < num_input; i++)

  1032.                     {

  1033.                         __m256 _val = _mm256_loadu_ps(m);

  1034.                         _sum0 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[0]), _sum0);

  1035.                         _sum1 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[1]), _sum1);

  1036.                         _sum2 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[2]), _sum2);

  1037.                         _sum3 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[3]), _sum3);

  1038. 

  1039.                         m += 8;

  1040.                         kptr += 4;

  1041.                     }

  1042. 

  1043.                     _sum0 = activation_avx(_sum0, activation_type, activation_params);

  1044.                     _sum1 = activation_avx(_sum1, activation_type, activation_params);

  1045.                     _sum2 = activation_avx(_sum2, activation_type, activation_params);

  1046.                     _sum3 = activation_avx(_sum3, activation_type, activation_params);

  1047. 

  1048.                     _mm256_storeu_ps(outptr, _sum0);

  1049.                     _mm256_storeu_ps(outptr + 8, _sum1);

  1050.                     _mm256_storeu_ps(outptr + 16, _sum2);

  1051.                     _mm256_storeu_ps(outptr + 24, _sum3);

  1052.                     outptr += 32;

  1053.                 }

  1054.             }

  1055. #endif // __AVX__

  1056. 

  1057.             if (elempack == 4 && num_output_elempack == 4)

  1058.             {

  1059.                 float* outptr = top_blob.row(j);

  1060. 

  1061.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  1062.                 {

  1063.                     const float* kptr = weight_data_tm.row(p);

  1064.                     const float* m = bottom_blob.row(j);

  1065. 

  1066.                     __m128 _sum0 = _mm_set1_ps(0.f);

  1067.                     __m128 _sum1 = _mm_set1_ps(0.f);

  1068.                     __m128 _sum2 = _mm_set1_ps(0.f);

  1069.                     __m128 _sum3 = _mm_set1_ps(0.f);

  1070. 

  1071.                     if (bias_term)

  1072.                     {

  1073.                         _sum0 = _mm_set1_ps(bias_data[p * 4 + 0]);

  1074.                         _sum1 = _mm_set1_ps(bias_data[p * 4 + 1]);

  1075.                         _sum2 = _mm_set1_ps(bias_data[p * 4 + 2]);

  1076.                         _sum3 = _mm_set1_ps(bias_data[p * 4 + 3]);

  1077.                     }

  1078. 

  1079.                     int i = 0;

  1080.                     for (; i + 3 < num_input; i += 4)

  1081.                     {

  1082.                         __m128 _val0 = _mm_loadu_ps(m);

  1083.                         __m128 _val1 = _mm_loadu_ps(m + 4);

  1084.                         __m128 _val2 = _mm_loadu_ps(m + 8);

  1085.                         __m128 _val3 = _mm_loadu_ps(m + 12);

  1086.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[0])), _sum0);

  1087.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[1])), _sum1);

  1088.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[2])), _sum2);

  1089.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[3])), _sum3);

  1090.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[4])), _sum0);

  1091.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[5])), _sum1);

  1092.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[6])), _sum2);

  1093.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[7])), _sum3);

  1094.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[8])), _sum0);

  1095.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[9])), _sum1);

  1096.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[10])), _sum2);

  1097.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[11])), _sum3);

  1098.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[12])), _sum0);

  1099.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[13])), _sum1);

  1100.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[14])), _sum2);

  1101.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[15])), _sum3);

  1102. 

  1103.                         m += 16;

  1104.                         kptr += 16;

  1105.                     }

  1106.                     for (; i < num_input; i++)

  1107.                     {

  1108.                         __m128 _val = _mm_loadu_ps(m);

  1109.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val, _mm_set1_ps(kptr[0])), _sum0);

  1110.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val, _mm_set1_ps(kptr[1])), _sum1);

  1111.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val, _mm_set1_ps(kptr[2])), _sum2);

  1112.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val, _mm_set1_ps(kptr[3])), _sum3);

  1113. 

  1114.                         m += 4;

  1115.                         kptr += 4;

  1116.                     }

  1117. 

  1118.                     _sum0 = activation_sse(_sum0, activation_type, activation_params);

  1119.                     _sum1 = activation_sse(_sum1, activation_type, activation_params);

  1120.                     _sum2 = activation_sse(_sum2, activation_type, activation_params);

  1121.                     _sum3 = activation_sse(_sum3, activation_type, activation_params);

  1122. 

  1123.                     _mm_storeu_ps(outptr, _sum0);

  1124.                     _mm_storeu_ps(outptr + 4, _sum1);

  1125.                     _mm_storeu_ps(outptr + 8, _sum2);

  1126.                     _mm_storeu_ps(outptr + 12, _sum3);

  1127.                     outptr += 16;

  1128.                 }

  1129.             }

  1130. 

  1131.             if (elempack == 1 && num_output_elempack == 4)

  1132.             {

  1133.                 float* outptr = top_blob.row(j);

  1134. 

  1135.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  1136.                 {

  1137.                     const float* kptr = weight_data_tm.row(p);

  1138.                     const float* m = bottom_blob.row(j);

  1139. 

  1140.                     __m128 _sum = _mm_set1_ps(0.f);

  1141. 

  1142.                     if (bias_term)

  1143.                     {

  1144.                         _sum = _mm_loadu_ps((const float*)bias_data + p * 4);

  1145.                     }

  1146. 

  1147.                     int i = 0;

  1148. #if __AVX__

  1149.                     for (; i + 7 < num_input; i += 8)

  1150.                     {

  1151.                         __m128 _val0 = _mm_broadcast_ss(m);

  1152.                         __m128 _val1 = _mm_broadcast_ss(m + 1);

  1153.                         __m128 _val2 = _mm_broadcast_ss(m + 2);

  1154.                         __m128 _val3 = _mm_broadcast_ss(m + 3);

  1155.                         __m128 _val4 = _mm_broadcast_ss(m + 4);

  1156.                         __m128 _val5 = _mm_broadcast_ss(m + 5);

  1157.                         __m128 _val6 = _mm_broadcast_ss(m + 6);

  1158.                         __m128 _val7 = _mm_broadcast_ss(m + 7);

  1159. 

  1160.                         __m128 _w0 = _mm_loadu_ps(kptr);

  1161.                         _sum = _mm_comp_fmadd_ps(_val0, _w0, _sum);

  1162.                         __m128 _w1 = _mm_loadu_ps(kptr + 4);

  1163.                         _sum = _mm_comp_fmadd_ps(_val1, _w1, _sum);

  1164.                         __m128 _w2 = _mm_loadu_ps(kptr + 8);

  1165.                         _sum = _mm_comp_fmadd_ps(_val2, _w2, _sum);

  1166.                         __m128 _w3 = _mm_loadu_ps(kptr + 12);

  1167.                         _sum = _mm_comp_fmadd_ps(_val3, _w3, _sum);

  1168.                         __m128 _w4 = _mm_loadu_ps(kptr + 16);

  1169.                         _sum = _mm_comp_fmadd_ps(_val4, _w4, _sum);

  1170.                         __m128 _w5 = _mm_loadu_ps(kptr + 20);

  1171.                         _sum = _mm_comp_fmadd_ps(_val5, _w5, _sum);

  1172.                         __m128 _w6 = _mm_loadu_ps(kptr + 24);

  1173.                         _sum = _mm_comp_fmadd_ps(_val6, _w6, _sum);

  1174.                         __m128 _w7 = _mm_loadu_ps(kptr + 28);

  1175.                         _sum = _mm_comp_fmadd_ps(_val7, _w7, _sum);

  1176. 

  1177.                         m += 8;

  1178.                         kptr += 32;

  1179.                     }

  1180. #endif // __AVX__

  1181.                     for (; i + 3 < num_input; i += 4)

  1182.                     {

  1183.                         __m128 _val0 = _mm_set1_ps(m[0]);

  1184.                         __m128 _val1 = _mm_set1_ps(m[1]);

  1185.                         __m128 _val2 = _mm_set1_ps(m[2]);

  1186.                         __m128 _val3 = _mm_set1_ps(m[3]);

  1187. 

  1188.                         __m128 _w0 = _mm_loadu_ps(kptr);

  1189.                         _sum = _mm_add_ps(_mm_mul_ps(_val0, _w0), _sum);

  1190.                         __m128 _w1 = _mm_loadu_ps(kptr + 4);

  1191.                         _sum = _mm_add_ps(_mm_mul_ps(_val1, _w1), _sum);

  1192.                         __m128 _w2 = _mm_loadu_ps(kptr + 8);

  1193.                         _sum = _mm_add_ps(_mm_mul_ps(_val2, _w2), _sum);

  1194.                         __m128 _w3 = _mm_loadu_ps(kptr + 12);

  1195.                         _sum = _mm_add_ps(_mm_mul_ps(_val3, _w3), _sum);

  1196. 

  1197.                         m += 4;

  1198.                         kptr += 16;

  1199.                     }

  1200.                     for (; i < num_input; i++)

  1201.                     {

  1202.                         __m128 _val = _mm_set1_ps(m[0]);

  1203.                         __m128 _k = _mm_loadu_ps(kptr);

  1204.                         _sum = _mm_add_ps(_mm_mul_ps(_val, _k), _sum);

  1205. 

  1206.                         m += 1;

  1207.                         kptr += 4;

  1208.                     }

  1209. 

  1210.                     _sum = activation_sse(_sum, activation_type, activation_params);

  1211. 

  1212.                     _mm_storeu_ps(outptr, _sum);

  1213.                     outptr += 4;

  1214.                 }

  1215.             }

  1216. 

  1217.             if (elempack == 4 && num_output_elempack == 1)

  1218.             {

  1219.                 float* outptr = top_blob.row(j);

  1220. 

  1221.                 for (int p = 0; p < num_output; p++)

  1222.                 {

  1223.                     const float* kptr = (const float*)weight_data_tm + num_input * p;

  1224.                     const float* m = bottom_blob.row(j);

  1225. 

  1226.                     __m128 _sum0 = _mm_set1_ps(0.f);

  1227.                     __m128 _sum1 = _mm_set1_ps(0.f);

  1228.                     __m128 _sum2 = _mm_set1_ps(0.f);

  1229.                     __m128 _sum3 = _mm_set1_ps(0.f);

  1230. 

  1231.                     if (bias_term)

  1232.                     {

  1233.                         _sum0 = _mm_set1_ps(bias_data[p]);

  1234.                     }

  1235. 

  1236.                     int i = 0;

  1237.                     for (; i + 7 < num_input; i += 8)

  1238.                     {

  1239.                         __m128 _val0 = _mm_loadu_ps(m);

  1240.                         __m128 _val1 = _mm_loadu_ps(m + 4);

  1241.                         __m128 _val2 = _mm_loadu_ps(m + 8);

  1242.                         __m128 _val3 = _mm_loadu_ps(m + 12);

  1243.                         __m128 _val4 = _mm_loadu_ps(m + 16);

  1244.                         __m128 _val5 = _mm_loadu_ps(m + 20);

  1245.                         __m128 _val6 = _mm_loadu_ps(m + 24);

  1246.                         __m128 _val7 = _mm_loadu_ps(m + 28);

  1247.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[0])), _sum0);

  1248.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[1])), _sum1);

  1249.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[2])), _sum2);

  1250.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[3])), _sum3);

  1251.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val4, _mm_set1_ps(kptr[4])), _sum0);

  1252.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val5, _mm_set1_ps(kptr[5])), _sum1);

  1253.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val6, _mm_set1_ps(kptr[6])), _sum2);

  1254.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val7, _mm_set1_ps(kptr[7])), _sum3);

  1255. 

  1256.                         m += 32;

  1257.                         kptr += 8;

  1258.                     }

  1259.                     for (; i + 3 < num_input; i += 4)

  1260.                     {

  1261.                         __m128 _val0 = _mm_loadu_ps(m);

  1262.                         __m128 _val1 = _mm_loadu_ps(m + 4);

  1263.                         __m128 _val2 = _mm_loadu_ps(m + 8);

  1264.                         __m128 _val3 = _mm_loadu_ps(m + 12);

  1265.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val0, _mm_set1_ps(kptr[0])), _sum0);

  1266.                         _sum1 = _mm_add_ps(_mm_mul_ps(_val1, _mm_set1_ps(kptr[1])), _sum1);

  1267.                         _sum2 = _mm_add_ps(_mm_mul_ps(_val2, _mm_set1_ps(kptr[2])), _sum2);

  1268.                         _sum3 = _mm_add_ps(_mm_mul_ps(_val3, _mm_set1_ps(kptr[3])), _sum3);

  1269. 

  1270.                         m += 16;

  1271.                         kptr += 4;

  1272.                     }

  1273.                     for (; i < num_input; i++)

  1274.                     {

  1275.                         __m128 _val = _mm_loadu_ps(m);

  1276.                         __m128 _k = _mm_set1_ps(kptr[0]);

  1277.                         _sum0 = _mm_add_ps(_mm_mul_ps(_val, _k), _sum0);

  1278. 

  1279.                         m += 4;

  1280.                         kptr += 1;

  1281.                     }

  1282. 

  1283.                     _sum0 = _mm_add_ps(_sum0, _sum1);

  1284.                     _sum2 = _mm_add_ps(_sum2, _sum3);

  1285.                     _sum0 = _mm_add_ps(_sum0, _sum2);

  1286. 

  1287.                     _sum0 = activation_sse(_sum0, activation_type, activation_params);

  1288. 

  1289.                     _mm_storeu_ps(outptr, _sum0);

  1290.                     outptr += 4;

  1291.                 }

  1292.             }

  1293. #endif // __SSE2__

  1294. 

  1295.             if (elempack == 1 && num_output_elempack == 1)

  1296.             {

  1297.                 float* outptr = top_blob.row(j);

  1298. 

  1299.                 for (int p = 0; p < num_output; p++)

  1300.                 {

  1301.                     const float* kptr = (const float*)weight_data_tm + num_input * p;

  1302.                     const float* m = bottom_blob.row(j);

  1303. 

  1304.                     float sum = 0.f;

  1305. 

  1306.                     if (bias_term)

  1307.                     {

  1308.                         sum = bias_data[p];

  1309.                     }

  1310. 

  1311.                     int i = 0;

  1312. #if __SSE2__

  1313. #if __AVX__

  1314.                     __m256 _sum = _mm256_set1_ps(0.f);

  1315.                     for (; i + 7 < num_input; i += 8)

  1316.                     {

  1317.                         __m256 _m = _mm256_loadu_ps(m);

  1318.                         __m256 _w = _mm256_loadu_ps(kptr);

  1319.                         _sum = _mm256_comp_fmadd_ps(_m, _w, _sum);

  1320. 

  1321.                         m += 8;

  1322.                         kptr += 8;

  1323.                     }

  1324. #endif // __AVX__

  1325.                     __m128 _suml = _mm_set1_ps(0.f);

  1326.                     for (; i + 3 < num_input; i += 4)

  1327.                     {

  1328.                         __m128 _val = _mm_loadu_ps(m);

  1329.                         __m128 _k = _mm_loadu_ps(kptr);

  1330.                         _suml = _mm_add_ps(_mm_mul_ps(_val, _k), _suml);

  1331. 

  1332.                         m += 4;

  1333.                         kptr += 4;

  1334.                     }

  1335. #endif // __SSE2__

  1336.                     for (; i < num_input; i++)

  1337.                     {

  1338.                         sum += *m++ * *kptr++;

  1339.                     }

  1340. 

  1341. #if __SSE2__

  1342. #if __AVX__

  1343.                     sum += _mm256_reduce_add_ps(_sum);

  1344. #endif // __AVX__

  1345.                     sum += _mm_reduce_add_ps(_suml);

  1346. #endif // __SSE2__

  1347. 

  1348.                     sum = activation_ss(sum, activation_type, activation_params);

  1349. 

  1350.                     outptr[0] = sum;

  1351.                     outptr += 1;

  1352.                 }

  1353.             }

  1354.         }

  1355. 

  1356.         return 0;

  1357.     }

  1358. 

  1359.     // flatten

  1360.     Mat bottom_blob_flattened = bottom_blob;

  1361.     if (bottom_blob.dims != 1)

  1362.     {

  1363.         Option opt_flatten = opt;

  1364.         opt_flatten.blob_allocator = opt.workspace_allocator;

  1365. 

  1366.         flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);

  1367.     }

  1368. 

  1369.     size_t elemsize = bottom_blob_flattened.elemsize;

  1370.     int elempack = bottom_blob_flattened.elempack;

  1371. 

  1372.     int out_elempack = 1;

  1373. #if __SSE2__

  1374.     if (opt.use_packing_layout)

  1375.     {

  1376. #if __AVX512F__

  1377.         out_elempack = num_output % 16 == 0 ? 16 : num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  1378. #elif __AVX__

  1379.         out_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;

  1380. #else

  1381.         out_elempack = num_output % 4 == 0 ? 4 : 1;

  1382. #endif

  1383.     }

  1384. #endif // __SSE2__

  1385.     size_t out_elemsize = elemsize / elempack * out_elempack;

  1386. 

  1387.     top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);

  1388.     if (top_blob.empty())

  1389.         return -100;

  1390. 

  1391. #if __SSE2__

  1392. #if __AVX__

  1393. #if __AVX512F__

  1394.     if (out_elempack == 16)

  1395.     {

  1396.         #pragma omp parallel for num_threads(opt.num_threads)

  1397.         for (int p = 0; p < num_output / out_elempack; p++)

  1398.         {

  1399.             __m512 _sum0 = _mm512_set1_ps(0.f);

  1400.             __m512 _sum1 = _mm512_set1_ps(0.f);

  1401.             __m512 _sum2 = _mm512_set1_ps(0.f);

  1402.             __m512 _sum3 = _mm512_set1_ps(0.f);

  1403.             __m512 _sum4 = _mm512_set1_ps(0.f);

  1404.             __m512 _sum5 = _mm512_set1_ps(0.f);

  1405.             __m512 _sum6 = _mm512_set1_ps(0.f);

  1406.             __m512 _sum7 = _mm512_set1_ps(0.f);

  1407. 

  1408.             if (bias_term)

  1409.             {

  1410.                 _sum0 = _mm512_loadu_ps((const float*)bias_data + p * 16);

  1411.             }

  1412. 

  1413.             const float* kptr = weight_data_tm.row(p);

  1414. 

  1415.             const float* sptr = bottom_blob_flattened;

  1416. 

  1417.             int i = 0;

  1418.             for (; i + 7 < num_input; i += 8)

  1419.             {

  1420.                 __m512 _val0 = _mm512_set1_ps(sptr[0]);

  1421.                 __m512 _val1 = _mm512_set1_ps(sptr[1]);

  1422.                 __m512 _val2 = _mm512_set1_ps(sptr[2]);

  1423.                 __m512 _val3 = _mm512_set1_ps(sptr[3]);

  1424.                 __m512 _val4 = _mm512_set1_ps(sptr[4]);

  1425.                 __m512 _val5 = _mm512_set1_ps(sptr[5]);

  1426.                 __m512 _val6 = _mm512_set1_ps(sptr[6]);

  1427.                 __m512 _val7 = _mm512_set1_ps(sptr[7]);

  1428. 

  1429.                 __m512 _w0 = _mm512_loadu_ps(kptr + 16 * 0);

  1430.                 __m512 _w1 = _mm512_loadu_ps(kptr + 16 * 1);

  1431.                 __m512 _w2 = _mm512_loadu_ps(kptr + 16 * 2);

  1432.                 __m512 _w3 = _mm512_loadu_ps(kptr + 16 * 3);

  1433.                 __m512 _w4 = _mm512_loadu_ps(kptr + 16 * 4);

  1434.                 __m512 _w5 = _mm512_loadu_ps(kptr + 16 * 5);

  1435.                 __m512 _w6 = _mm512_loadu_ps(kptr + 16 * 6);

  1436.                 __m512 _w7 = _mm512_loadu_ps(kptr + 16 * 7);

  1437. 

  1438.                 _sum0 = _mm512_fmadd_ps(_val0, _w0, _sum0);

  1439.                 _sum1 = _mm512_fmadd_ps(_val1, _w1, _sum1);

  1440.                 _sum2 = _mm512_fmadd_ps(_val2, _w2, _sum2);

  1441.                 _sum3 = _mm512_fmadd_ps(_val3, _w3, _sum3);

  1442.                 _sum4 = _mm512_fmadd_ps(_val4, _w4, _sum4);

  1443.                 _sum5 = _mm512_fmadd_ps(_val5, _w5, _sum5);

  1444.                 _sum6 = _mm512_fmadd_ps(_val6, _w6, _sum6);

  1445.                 _sum7 = _mm512_fmadd_ps(_val7, _w7, _sum7);

  1446. 

  1447.                 sptr += 8;

  1448.                 kptr += 128;

  1449.             }

  1450.             for (; i + 3 < num_input; i += 4)

  1451.             {

  1452.                 __m512 _val0 = _mm512_set1_ps(sptr[0]);

  1453.                 __m512 _val1 = _mm512_set1_ps(sptr[1]);

  1454.                 __m512 _val2 = _mm512_set1_ps(sptr[2]);

  1455.                 __m512 _val3 = _mm512_set1_ps(sptr[3]);

  1456. 

  1457.                 __m512 _w0 = _mm512_loadu_ps(kptr);

  1458.                 __m512 _w1 = _mm512_loadu_ps(kptr + 16);

  1459.                 __m512 _w2 = _mm512_loadu_ps(kptr + 32);

  1460.                 __m512 _w3 = _mm512_loadu_ps(kptr + 48);

  1461.                 _sum0 = _mm512_fmadd_ps(_val0, _w0, _sum0);

  1462.                 _sum1 = _mm512_fmadd_ps(_val1, _w1, _sum1);

  1463.                 _sum2 = _mm512_fmadd_ps(_val2, _w2, _sum2);

  1464.                 _sum3 = _mm512_fmadd_ps(_val3, _w3, _sum3);

  1465. 

  1466.                 sptr += 4;

  1467.                 kptr += 64;

  1468.             }

  1469.             for (; i < num_input; i++)

  1470.             {

  1471.                 __m512 _val = _mm512_set1_ps(sptr[0]);

  1472.                 __m512 _w = _mm512_loadu_ps(kptr);

  1473.                 _sum0 = _mm512_fmadd_ps(_val, _w, _sum0);

  1474. 

  1475.                 sptr += 1;

  1476.                 kptr += 16;

  1477.             }

  1478. 

  1479.             _sum0 = _mm512_add_ps(_sum0, _sum1);

  1480.             _sum2 = _mm512_add_ps(_sum2, _sum3);

  1481.             _sum4 = _mm512_add_ps(_sum4, _sum5);

  1482.             _sum6 = _mm512_add_ps(_sum6, _sum7);

  1483.             _sum0 = _mm512_add_ps(_sum0, _sum2);

  1484.             _sum4 = _mm512_add_ps(_sum4, _sum6);

  1485.             _sum0 = _mm512_add_ps(_sum0, _sum4);

  1486. 

  1487.             _sum0 = activation_avx512(_sum0, activation_type, activation_params);

  1488. 

  1489.             float* outptr = top_blob;

  1490.             _mm512_storeu_ps(outptr + p * 16, _sum0);

  1491.         }

  1492.     }

  1493. 

  1494. #endif // __AVX512F__

  1495. 

  1496.     if (out_elempack == 8)

  1497.     {

  1498.         #pragma omp parallel for num_threads(opt.num_threads)

  1499.         for (int p = 0; p < num_output / out_elempack; p++)

  1500.         {

  1501.             __m256 _sum0 = _mm256_set1_ps(0.f);

  1502.             __m256 _sum1 = _mm256_set1_ps(0.f);

  1503.             __m256 _sum2 = _mm256_set1_ps(0.f);

  1504.             __m256 _sum3 = _mm256_set1_ps(0.f);

  1505.             __m256 _sum4 = _mm256_set1_ps(0.f);

  1506.             __m256 _sum5 = _mm256_set1_ps(0.f);

  1507.             __m256 _sum6 = _mm256_set1_ps(0.f);

  1508.             __m256 _sum7 = _mm256_set1_ps(0.f);

  1509. 

  1510.             if (bias_term)

  1511.             {

  1512.                 _sum0 = _mm256_loadu_ps((const float*)bias_data + p * 8);

  1513.             }

  1514. 

  1515.             const float* kptr = weight_data_tm.row(p);

  1516. 

  1517.             const float* sptr = bottom_blob_flattened;

  1518. 

  1519.             int i = 0;

  1520.             for (; i + 7 < num_input; i += 8)

  1521.             {

  1522.                 __m256 _val0 = _mm256_broadcast_ss(sptr);

  1523.                 __m256 _val1 = _mm256_broadcast_ss(sptr + 1);

  1524.                 __m256 _val2 = _mm256_broadcast_ss(sptr + 2);

  1525.                 __m256 _val3 = _mm256_broadcast_ss(sptr + 3);

  1526.                 __m256 _val4 = _mm256_broadcast_ss(sptr + 4);

  1527.                 __m256 _val5 = _mm256_broadcast_ss(sptr + 5);

  1528.                 __m256 _val6 = _mm256_broadcast_ss(sptr + 6);

  1529.                 __m256 _val7 = _mm256_broadcast_ss(sptr + 7);

  1530. 

  1531.                 __m256 _w0 = _mm256_loadu_ps(kptr);

  1532.                 _sum0 = _mm256_comp_fmadd_ps(_val0, _w0, _sum0);

  1533.                 __m256 _w1 = _mm256_loadu_ps(kptr + 8);

  1534.                 _sum1 = _mm256_comp_fmadd_ps(_val1, _w1, _sum1);

  1535.                 __m256 _w2 = _mm256_loadu_ps(kptr + 16);

  1536.                 _sum2 = _mm256_comp_fmadd_ps(_val2, _w2, _sum2);

  1537.                 __m256 _w3 = _mm256_loadu_ps(kptr + 24);

  1538.                 _sum3 = _mm256_comp_fmadd_ps(_val3, _w3, _sum3);

  1539.                 __m256 _w4 = _mm256_loadu_ps(kptr + 32);

  1540.                 _sum4 = _mm256_comp_fmadd_ps(_val4, _w4, _sum4);

  1541.                 __m256 _w5 = _mm256_loadu_ps(kptr + 40);

  1542.                 _sum5 = _mm256_comp_fmadd_ps(_val5, _w5, _sum5);

  1543.                 __m256 _w6 = _mm256_loadu_ps(kptr + 48);

  1544.                 _sum6 = _mm256_comp_fmadd_ps(_val6, _w6, _sum6);

  1545.                 __m256 _w7 = _mm256_loadu_ps(kptr + 56);

  1546.                 _sum7 = _mm256_comp_fmadd_ps(_val7, _w7, _sum7);

  1547. 

  1548.                 sptr += 8;

  1549.                 kptr += 64;

  1550.             }

  1551.             for (; i + 3 < num_input; i += 4)

  1552.             {

  1553.                 __m256 _val0 = _mm256_broadcast_ss(sptr);

  1554.                 __m256 _val1 = _mm256_broadcast_ss(sptr + 1);

  1555.                 __m256 _val2 = _mm256_broadcast_ss(sptr + 2);

  1556.                 __m256 _val3 = _mm256_broadcast_ss(sptr + 3);

  1557. 

  1558.                 __m256 _w0 = _mm256_loadu_ps(kptr);

  1559.                 _sum0 = _mm256_comp_fmadd_ps(_val0, _w0, _sum0);

  1560.                 __m256 _w1 = _mm256_loadu_ps(kptr + 8);

  1561.                 _sum1 = _mm256_comp_fmadd_ps(_val1, _w1, _sum1);

  1562.                 __m256 _w2 = _mm256_loadu_ps(kptr + 16);

  1563.                 _sum2 = _mm256_comp_fmadd_ps(_val2, _w2, _sum2);

  1564.                 __m256 _w3 = _mm256_loadu_ps(kptr + 24);

  1565.                 _sum3 = _mm256_comp_fmadd_ps(_val3, _w3, _sum3);

  1566. 

  1567.                 sptr += 4;

  1568.                 kptr += 32;

  1569.             }

  1570.             for (; i < num_input; i++)

  1571.             {

  1572.                 __m256 _val = _mm256_set1_ps(sptr[0]);

  1573.                 __m256 _w = _mm256_loadu_ps(kptr);

  1574.                 _sum0 = _mm256_comp_fmadd_ps(_val, _w, _sum0);

  1575. 

  1576.                 sptr += 1;

  1577.                 kptr += 8;

  1578.             }

  1579. 

  1580.             _sum0 = _mm256_add_ps(_sum0, _sum1);

  1581.             _sum2 = _mm256_add_ps(_sum2, _sum3);

  1582.             _sum4 = _mm256_add_ps(_sum4, _sum5);

  1583.             _sum6 = _mm256_add_ps(_sum6, _sum7);

  1584.             _sum0 = _mm256_add_ps(_sum0, _sum2);

  1585.             _sum4 = _mm256_add_ps(_sum4, _sum6);

  1586.             _sum0 = _mm256_add_ps(_sum0, _sum4);

  1587. 

  1588.             _sum0 = activation_avx(_sum0, activation_type, activation_params);

  1589. 

  1590.             float* outptr = top_blob;

  1591.             _mm256_storeu_ps(outptr + p * 8, _sum0);

  1592.         }

  1593.     }

  1594. #endif // __AVX__

  1595. 

  1596.     if (out_elempack == 4)

  1597.     {

  1598.         #pragma omp parallel for num_threads(opt.num_threads)

  1599.         for (int p = 0; p < num_output / out_elempack; p++)

  1600.         {

  1601.             __m128 _sum0 = _mm_set1_ps(0.f);

  1602.             __m128 _sum1 = _mm_set1_ps(0.f);

  1603.             __m128 _sum2 = _mm_set1_ps(0.f);

  1604.             __m128 _sum3 = _mm_set1_ps(0.f);

  1605. #if __AVX__

  1606.             __m128 _sum4 = _mm_set1_ps(0.f);

  1607.             __m128 _sum5 = _mm_set1_ps(0.f);

  1608.             __m128 _sum6 = _mm_set1_ps(0.f);

  1609.             __m128 _sum7 = _mm_set1_ps(0.f);

  1610. #endif

  1611. 

  1612.             if (bias_term)

  1613.             {

  1614.                 _sum0 = _mm_loadu_ps((const float*)bias_data + p * 4);

  1615.             }

  1616. 

  1617.             const float* kptr = weight_data_tm.row(p);

  1618. 

  1619.             const float* sptr = bottom_blob_flattened;

  1620. 

  1621.             int i = 0;

  1622. #if __AVX__

  1623.             for (; i + 7 < num_input; i += 8)

  1624.             {

  1625.                 __m128 _val0 = _mm_broadcast_ss(sptr);

  1626.                 __m128 _val1 = _mm_broadcast_ss(sptr + 1);

  1627.                 __m128 _val2 = _mm_broadcast_ss(sptr + 2);

  1628.                 __m128 _val3 = _mm_broadcast_ss(sptr + 3);

  1629.                 __m128 _val4 = _mm_broadcast_ss(sptr + 4);

  1630.                 __m128 _val5 = _mm_broadcast_ss(sptr + 5);

  1631.                 __m128 _val6 = _mm_broadcast_ss(sptr + 6);

  1632.                 __m128 _val7 = _mm_broadcast_ss(sptr + 7);

  1633. 

  1634.                 __m128 _w0 = _mm_loadu_ps(kptr);

  1635.                 _sum0 = _mm_comp_fmadd_ps(_val0, _w0, _sum0);

  1636.                 __m128 _w1 = _mm_loadu_ps(kptr + 4);

  1637.                 _sum1 = _mm_comp_fmadd_ps(_val1, _w1, _sum1);

  1638.                 __m128 _w2 = _mm_loadu_ps(kptr + 8);

  1639.                 _sum2 = _mm_comp_fmadd_ps(_val2, _w2, _sum2);

  1640.                 __m128 _w3 = _mm_loadu_ps(kptr + 12);

  1641.                 _sum3 = _mm_comp_fmadd_ps(_val3, _w3, _sum3);

  1642.                 __m128 _w4 = _mm_loadu_ps(kptr + 16);

  1643.                 _sum4 = _mm_comp_fmadd_ps(_val4, _w4, _sum4);

  1644.                 __m128 _w5 = _mm_loadu_ps(kptr + 20);

  1645.                 _sum5 = _mm_comp_fmadd_ps(_val5, _w5, _sum5);

  1646.                 __m128 _w6 = _mm_loadu_ps(kptr + 24);

  1647.                 _sum6 = _mm_comp_fmadd_ps(_val6, _w6, _sum6);

  1648.                 __m128 _w7 = _mm_loadu_ps(kptr + 28);

  1649.                 _sum7 = _mm_comp_fmadd_ps(_val7, _w7, _sum7);

  1650. 

  1651.                 sptr += 8;

  1652.                 kptr += 32;

  1653.             }

  1654. #endif

  1655.             for (; i + 3 < num_input; i += 4)

  1656.             {

  1657.                 __m128 _val0 = _mm_set1_ps(sptr[0]);

  1658.                 __m128 _val1 = _mm_set1_ps(sptr[1]);

  1659.                 __m128 _val2 = _mm_set1_ps(sptr[2]);

  1660.                 __m128 _val3 = _mm_set1_ps(sptr[3]);

  1661. 

  1662.                 __m128 _w0 = _mm_loadu_ps(kptr);

  1663.                 _sum0 = _mm_add_ps(_mm_mul_ps(_val0, _w0), _sum0);

  1664.                 __m128 _w1 = _mm_loadu_ps(kptr + 4);

  1665.                 _sum1 = _mm_add_ps(_mm_mul_ps(_val1, _w1), _sum1);

  1666.                 __m128 _w2 = _mm_loadu_ps(kptr + 8);

  1667.                 _sum2 = _mm_add_ps(_mm_mul_ps(_val2, _w2), _sum2);

  1668.                 __m128 _w3 = _mm_loadu_ps(kptr + 12);

  1669.                 _sum3 = _mm_add_ps(_mm_mul_ps(_val3, _w3), _sum3);

  1670. 

  1671.                 sptr += 4;

  1672.                 kptr += 16;

  1673.             }

  1674.             for (; i < num_input; i++)

  1675.             {

  1676.                 __m128 _val = _mm_set1_ps(sptr[0]);

  1677.                 __m128 _w = _mm_loadu_ps(kptr);

  1678.                 _sum0 = _mm_add_ps(_mm_mul_ps(_val, _w), _sum0);

  1679. 

  1680.                 sptr += 1;

  1681.                 kptr += 4;

  1682.             }

  1683. 

  1684.             _sum0 = _mm_add_ps(_sum0, _sum1);

  1685.             _sum2 = _mm_add_ps(_sum2, _sum3);

  1686. #if __AVX__

  1687.             _sum4 = _mm_add_ps(_sum4, _sum5);

  1688.             _sum6 = _mm_add_ps(_sum6, _sum7);

  1689. #endif

  1690.             _sum0 = _mm_add_ps(_sum0, _sum2);

  1691. #if __AVX__

  1692.             _sum4 = _mm_add_ps(_sum4, _sum6);

  1693.             _sum0 = _mm_add_ps(_sum0, _sum4);

  1694. #endif

  1695. 

  1696.             _sum0 = activation_sse(_sum0, activation_type, activation_params);

  1697. 

  1698.             float* outptr = top_blob;

  1699.             _mm_storeu_ps(outptr + p * 4, _sum0);

  1700.         }

  1701.     }

  1702. #endif // __SSE2__

  1703. 

  1704.     if (out_elempack == 1)

  1705.     {

  1706. #if __SSE2__

  1707. #if __AVX__

  1708.         int remain_num_output_start = 0;

  1709.         int nn_num_output = num_output >> 3;

  1710. 

  1711.         #pragma omp parallel for num_threads(opt.num_threads)

  1712.         for (int pp = 0; pp < nn_num_output; pp++)

  1713.         {

  1714.             int p = pp * 8;

  1715. 

  1716.             float sums[8] = {0.0f};

  1717.             if (bias_term)

  1718.             {

  1719.                 sums[0] = bias_data[p];

  1720.                 sums[1] = bias_data[p + 1];

  1721.                 sums[2] = bias_data[p + 2];

  1722.                 sums[3] = bias_data[p + 3];

  1723.                 sums[4] = bias_data[p + 4];

  1724.                 sums[5] = bias_data[p + 5];

  1725.                 sums[6] = bias_data[p + 6];

  1726.                 sums[7] = bias_data[p + 7];

  1727.             }

  1728. 

  1729.             const float* w0 = (const float*)weight_data_tm + num_input * p;

  1730.             const float* w1 = (const float*)weight_data_tm + num_input * (p + 1);

  1731.             const float* w2 = (const float*)weight_data_tm + num_input * (p + 2);

  1732.             const float* w3 = (const float*)weight_data_tm + num_input * (p + 3);

  1733.             const float* w4 = (const float*)weight_data_tm + num_input * (p + 4);

  1734.             const float* w5 = (const float*)weight_data_tm + num_input * (p + 5);

  1735.             const float* w6 = (const float*)weight_data_tm + num_input * (p + 6);

  1736.             const float* w7 = (const float*)weight_data_tm + num_input * (p + 7);

  1737. 

  1738.             const float* m = bottom_blob_flattened;

  1739. 

  1740.             __m256 _sum0 = _mm256_set1_ps(0.f);

  1741.             __m256 _sum1 = _mm256_set1_ps(0.f);

  1742.             __m256 _sum2 = _mm256_set1_ps(0.f);

  1743.             __m256 _sum3 = _mm256_set1_ps(0.f);

  1744.             __m256 _sum4 = _mm256_set1_ps(0.f);

  1745.             __m256 _sum5 = _mm256_set1_ps(0.f);

  1746.             __m256 _sum6 = _mm256_set1_ps(0.f);

  1747.             __m256 _sum7 = _mm256_set1_ps(0.f);

  1748. 

  1749.             int i = 0;

  1750.             for (; i + 7 < num_input; i += 8)

  1751.             {

  1752.                 __m256 _m = _mm256_loadu_ps(m);

  1753. 

  1754.                 __m256 _w0 = _mm256_loadu_ps(w0);

  1755.                 _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0);

  1756.                 __m256 _w1 = _mm256_loadu_ps(w1);

  1757.                 _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1);

  1758.                 __m256 _w2 = _mm256_loadu_ps(w2);

  1759.                 _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2);

  1760.                 __m256 _w3 = _mm256_loadu_ps(w3);

  1761.                 _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3);

  1762.                 __m256 _w4 = _mm256_loadu_ps(w4);

  1763.                 _sum4 = _mm256_comp_fmadd_ps(_m, _w4, _sum4);

  1764.                 __m256 _w5 = _mm256_loadu_ps(w5);

  1765.                 _sum5 = _mm256_comp_fmadd_ps(_m, _w5, _sum5);

  1766.                 __m256 _w6 = _mm256_loadu_ps(w6);

  1767.                 _sum6 = _mm256_comp_fmadd_ps(_m, _w6, _sum6);

  1768.                 __m256 _w7 = _mm256_loadu_ps(w7);

  1769.                 _sum7 = _mm256_comp_fmadd_ps(_m, _w7, _sum7);

  1770. 

  1771.                 m += 8;

  1772.                 w0 += 8;

  1773.                 w1 += 8;

  1774.                 w2 += 8;

  1775.                 w3 += 8;

  1776.                 w4 += 8;

  1777.                 w5 += 8;

  1778.                 w6 += 8;

  1779.                 w7 += 8;

  1780.             }

  1781.             for (; i < num_input; i++)

  1782.             {

  1783.                 sums[0] += *m * *w0;

  1784.                 sums[1] += *m * *w1;

  1785.                 sums[2] += *m * *w2;

  1786.                 sums[3] += *m * *w3;

  1787.                 sums[4] += *m * *w4;

  1788.                 sums[5] += *m * *w5;

  1789.                 sums[6] += *m * *w6;

  1790.                 sums[7] += *m * *w7;

  1791. 

  1792.                 m++;

  1793.                 w0++;

  1794.                 w1++;

  1795.                 w2++;

  1796.                 w3++;

  1797.                 w4++;

  1798.                 w5++;

  1799.                 w6++;

  1800.                 w7++;

  1801.             }

  1802. 

  1803.             __m256 _sums = HorizontalSums(_sum0, _sum1, _sum2, _sum3, _sum4, _sum5, _sum6, _sum7);

  1804.             __m256 _sums_f = _mm256_loadu_ps(sums);

  1805.             _sums = _mm256_add_ps(_sums_f, _sums);

  1806.             _sums = activation_avx(_sums, activation_type, activation_params);

  1807. 

  1808.             float* outptr = top_blob;

  1809.             _mm256_storeu_ps(outptr + p, _sums);

  1810.         }

  1811. 

  1812.         remain_num_output_start += (nn_num_output << 3);

  1813.         nn_num_output = (num_output - remain_num_output_start) >> 2;

  1814. #else

  1815.         int remain_num_output_start = 0;

  1816.         int nn_num_output = num_output >> 2;

  1817. #endif // __AVX__

  1818. 

  1819.         #pragma omp parallel for num_threads(opt.num_threads)

  1820.         for (int pp = 0; pp < nn_num_output; pp++)

  1821.         {

  1822.             int p = remain_num_output_start + (pp * 4);

  1823. 

  1824.             float sums[4] = {0.0f};

  1825.             if (bias_term)

  1826.             {

  1827.                 sums[0] = bias_data[p];

  1828.                 sums[1] = bias_data[p + 1];

  1829.                 sums[2] = bias_data[p + 2];

  1830.                 sums[3] = bias_data[p + 3];

  1831.             }

  1832. 

  1833.             const float* w0 = (const float*)weight_data_tm + num_input * p;

  1834.             const float* w1 = (const float*)weight_data_tm + num_input * (p + 1);

  1835.             const float* w2 = (const float*)weight_data_tm + num_input * (p + 2);

  1836.             const float* w3 = (const float*)weight_data_tm + num_input * (p + 3);

  1837. 

  1838.             const float* m = bottom_blob_flattened;

  1839. 

  1840.             int i = 0;

  1841. #if __AVX__

  1842.             __m256 _sum0 = _mm256_set1_ps(0.f);

  1843.             __m256 _sum1 = _mm256_set1_ps(0.f);

  1844.             __m256 _sum2 = _mm256_set1_ps(0.f);

  1845.             __m256 _sum3 = _mm256_set1_ps(0.f);

  1846.             for (; i + 7 < num_input; i += 8)

  1847.             {

  1848.                 __m256 _m = _mm256_loadu_ps(m);

  1849. 

  1850.                 __m256 _w0 = _mm256_loadu_ps(w0);

  1851.                 _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0);

  1852.                 __m256 _w1 = _mm256_loadu_ps(w1);

  1853.                 _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1);

  1854.                 __m256 _w2 = _mm256_loadu_ps(w2);

  1855.                 _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2);

  1856.                 __m256 _w3 = _mm256_loadu_ps(w3);

  1857.                 _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3);

  1858. 

  1859.                 m += 8;

  1860.                 w0 += 8;

  1861.                 w1 += 8;

  1862.                 w2 += 8;

  1863.                 w3 += 8;

  1864.             }

  1865. #endif // __AVX__

  1866.             __m128 _sum0l = _mm_set1_ps(0.f);

  1867.             __m128 _sum1l = _mm_set1_ps(0.f);

  1868.             __m128 _sum2l = _mm_set1_ps(0.f);

  1869.             __m128 _sum3l = _mm_set1_ps(0.f);

  1870.             for (; i + 3 < num_input; i += 4)

  1871.             {

  1872.                 __m128 _m = _mm_loadu_ps(m);

  1873. 

  1874.                 __m128 _w0 = _mm_loadu_ps(w0);

  1875.                 _sum0l = _mm_add_ps(_mm_mul_ps(_m, _w0), _sum0l);

  1876.                 __m128 _w1 = _mm_loadu_ps(w1);

  1877.                 _sum1l = _mm_add_ps(_mm_mul_ps(_m, _w1), _sum1l);

  1878.                 __m128 _w2 = _mm_loadu_ps(w2);

  1879.                 _sum2l = _mm_add_ps(_mm_mul_ps(_m, _w2), _sum2l);

  1880.                 __m128 _w3 = _mm_loadu_ps(w3);

  1881.                 _sum3l = _mm_add_ps(_mm_mul_ps(_m, _w3), _sum3l);

  1882. 

  1883.                 m += 4;

  1884.                 w0 += 4;

  1885.                 w1 += 4;

  1886.                 w2 += 4;

  1887.                 w3 += 4;

  1888.             }

  1889.             for (; i < num_input; i++)

  1890.             {

  1891.                 sums[0] += *m * *w0;

  1892.                 sums[1] += *m * *w1;

  1893.                 sums[2] += *m * *w2;

  1894.                 sums[3] += *m * *w3;

  1895. 

  1896.                 m++;

  1897.                 w0++;

  1898.                 w1++;

  1899.                 w2++;

  1900.                 w3++;

  1901.             }

  1902. 

  1903.             __m128 _sums = _mm_loadu_ps(sums);

  1904. #if __AVX__

  1905.             _sums = _mm_add_ps(HorizontalSums(_sum0, _sum1, _sum2, _sum3), _sums);

  1906. #endif

  1907.             _MM_TRANSPOSE4_PS(_sum0l, _sum1l, _sum2l, _sum3l);

  1908.             _sums = _mm_add_ps(_sum0l, _sums);

  1909.             _sums = _mm_add_ps(_sum1l, _sums);

  1910.             _sums = _mm_add_ps(_sum2l, _sums);

  1911.             _sums = _mm_add_ps(_sum3l, _sums);

  1912.             _sums = activation_sse(_sums, activation_type, activation_params);

  1913. 

  1914.             float* outptr = top_blob;

  1915.             _mm_storeu_ps(outptr + p, _sums);

  1916.         }

  1917. 

  1918.         remain_num_output_start += (nn_num_output << 2);

  1919. #else

  1920.         int remain_num_output_start = 0;

  1921. #endif // __SSE2__

  1922. 

  1923.         #pragma omp parallel for num_threads(opt.num_threads)

  1924.         for (int p = remain_num_output_start; p < num_output; p++)

  1925.         {

  1926.             float sum = 0.f;

  1927. 

  1928.             if (bias_term)

  1929.                 sum = bias_data[p];

  1930. 

  1931.             const float* w = (const float*)weight_data_tm + num_input * p;

  1932. 

  1933.             const float* m = bottom_blob_flattened;

  1934. 

  1935.             int i = 0;

  1936. #if __SSE2__

  1937. #if __AVX__

  1938.             __m256 _sum = _mm256_set1_ps(0.f);

  1939.             for (; i + 7 < num_input; i += 8)

  1940.             {

  1941.                 __m256 _m = _mm256_loadu_ps(m);

  1942. 

  1943.                 __m256 _w = _mm256_loadu_ps(w);

  1944.                 _sum = _mm256_comp_fmadd_ps(_m, _w, _sum);

  1945. 

  1946.                 m += 8;

  1947.                 w += 8;

  1948.             }

  1949. #endif // __AVX__

  1950.             __m128 _suml = _mm_set1_ps(0.f);

  1951.             for (; i + 3 < num_input; i += 4)

  1952.             {

  1953.                 __m128 _m = _mm_loadu_ps(m);

  1954. 

  1955.                 __m128 _w = _mm_loadu_ps(w);

  1956.                 _suml = _mm_add_ps(_mm_mul_ps(_m, _w), _suml);

  1957. 

  1958.                 m += 4;

  1959.                 w += 4;

  1960.             }

  1961. #endif // __SSE2__

  1962.             for (; i < num_input; i++)

  1963.             {

  1964.                 sum += *m * *w;

  1965.                 m++;

  1966.                 w++;

  1967.             }

  1968. 

  1969. #if __SSE2__

  1970. #if __AVX__

  1971.             sum += _mm256_reduce_add_ps(_sum);

  1972. #endif

  1973.             sum += _mm_reduce_add_ps(_suml);

  1974. #endif // __SSE2__

  1975. 

  1976.             sum = activation_ss(sum, activation_type, activation_params);

  1977. 

  1978.             float* outptr = top_blob;

  1979.             outptr[p] = sum;

  1980.         }

  1981.     }

  1982. 

  1983.     return 0;

  1984. }

  1985. 

  1986. #if NCNN_INT8

  1987. int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt)

  1988. {

  1989.     const int num_input = weight_data_size / num_output;

  1990. 

  1991.     int out_elempack = 1;

  1992. #if __SSE2__

  1993.     if (opt.use_packing_layout)

  1994.     {

  1995.         out_elempack = num_output % 8 == 0 ? 8 : 1;

  1996.     }

  1997. #endif // __SSE2__

  1998. 

  1999.     // src = inch-outch

  2000.     // dst = pb-inch-outch/pb

  2001.     {

  2002.         Mat weight_data_r2 = weight_data.reshape(num_input, num_output);

  2003. 

  2004.         weight_data_tm.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack);

  2005. 

  2006.         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)

  2007.         {

  2008.             signed char* g0 = weight_data_tm.row<signed char>(q / out_elempack);

  2009. 

  2010.             for (int p = 0; p < num_input; p++)

  2011.             {

  2012.                 for (int j = 0; j < out_elempack; j++)

  2013.                 {

  2014.                     *g0++ = weight_data_r2.row<signed char>(q + j)[p];

  2015.                 }

  2016.             }

  2017.         }

  2018.     }

  2019. 

  2020.     scale_in_data.create(num_output);

  2021.     for (int p = 0; p < num_output; p++)

  2022.     {

  2023.         // dequantize

  2024.         float scale_in;

  2025.         if (weight_data_int8_scales[p] == 0)

  2026.             scale_in = 0;

  2027.         else

  2028.             scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);

  2029. 

  2030.         scale_in_data[p] = scale_in;

  2031.     }

  2032. 

  2033.     if (opt.lightmode)

  2034.     {

  2035.         weight_data.release();

  2036.     }

  2037. 

  2038.     return 0;

  2039. }

  2040. 

  2041. int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  2042. {

  2043.     const int num_input = weight_data_size / num_output;

  2044. 

  2045.     int elembits = bottom_blob.elembits();

  2046. 

  2047.     Mat bottom_blob_int8 = bottom_blob;

  2048.     if (elembits != 8)

  2049.     {

  2050.         Option opt_q = opt;

  2051.         opt_q.blob_allocator = opt.workspace_allocator;

  2052.         quantize_to_int8(bottom_blob, bottom_blob_int8, bottom_blob_int8_scales, opt_q);

  2053.     }

  2054. 

  2055.     if (bottom_blob_int8.dims == 2 && bottom_blob_int8.w == num_input && bottom_blob_int8.h * bottom_blob_int8.elempack > 1)

  2056.     {

  2057.         // gemm

  2058.         Mat bottom_blob_int8_unpacked;

  2059.         Option opt_unpack = opt;

  2060.         opt_unpack.blob_allocator = opt.workspace_allocator;

  2061.         convert_packing(bottom_blob_int8, bottom_blob_int8_unpacked, 1, opt_unpack);

  2062. 

  2063.         int h = bottom_blob_int8_unpacked.h;

  2064. 

  2065.         int out_elempack = 1;

  2066. #if __SSE2__

  2067.         if (opt.use_packing_layout)

  2068.         {

  2069.             out_elempack = h % 4 == 0 ? 4 : 1;

  2070.         }

  2071. #endif

  2072. 

  2073.         int outh = h / out_elempack;

  2074. 

  2075.         top_blob.create(num_output, outh, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator);

  2076.         if (top_blob.empty())

  2077.             return -100;

  2078. 

  2079.         int num_output_elempack = 1;

  2080. #if __SSE2__

  2081.         if (opt.use_packing_layout)

  2082.         {

  2083.             num_output_elempack = num_output % 8 == 0 ? 8 : 1;

  2084.         }

  2085. #endif

  2086. 

  2087. #if __SSE2__

  2088.         if (num_output_elempack == 8 && out_elempack == 4)

  2089.         {

  2090.             #pragma omp parallel for num_threads(opt.num_threads)

  2091.             for (int j = 0; j < outh; j++)

  2092.             {

  2093.                 float* outptr = top_blob.row(j);

  2094. 

  2095.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  2096.                 {

  2097.                     const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2098.                     const signed char* m0 = bottom_blob_int8_unpacked.row<const signed char>(j * 4);

  2099.                     const signed char* m1 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 1);

  2100.                     const signed char* m2 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 2);

  2101.                     const signed char* m3 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 3);

  2102. 

  2103.                     __m128i _sum00 = _mm_setzero_si128();

  2104.                     __m128i _sum01 = _mm_setzero_si128();

  2105.                     __m128i _sum10 = _mm_setzero_si128();

  2106.                     __m128i _sum11 = _mm_setzero_si128();

  2107.                     __m128i _sum20 = _mm_setzero_si128();

  2108.                     __m128i _sum21 = _mm_setzero_si128();

  2109.                     __m128i _sum30 = _mm_setzero_si128();

  2110.                     __m128i _sum31 = _mm_setzero_si128();

  2111. 

  2112.                     int i = 0;

  2113.                     for (; i < num_input; i++)

  2114.                     {

  2115.                         // TODO use _mm_cvtepi8_epi16 on sse4.1

  2116.                         __m128i _w = _mm_loadl_epi64((const __m128i*)kptr);

  2117.                         _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w));

  2118. 

  2119.                         __m128i _val0 = _mm_set1_epi16((short)m0[0]);

  2120.                         __m128i _val1 = _mm_set1_epi16((short)m1[0]);

  2121.                         __m128i _val2 = _mm_set1_epi16((short)m2[0]);

  2122.                         __m128i _val3 = _mm_set1_epi16((short)m3[0]);

  2123. 

  2124.                         __m128i _s0l = _mm_mullo_epi16(_val0, _w);

  2125.                         __m128i _s0h = _mm_mulhi_epi16(_val0, _w);

  2126.                         __m128i _s1l = _mm_mullo_epi16(_val1, _w);

  2127.                         __m128i _s1h = _mm_mulhi_epi16(_val1, _w);

  2128.                         __m128i _s2l = _mm_mullo_epi16(_val2, _w);

  2129.                         __m128i _s2h = _mm_mulhi_epi16(_val2, _w);

  2130.                         __m128i _s3l = _mm_mullo_epi16(_val3, _w);

  2131.                         __m128i _s3h = _mm_mulhi_epi16(_val3, _w);

  2132.                         __m128i _s00 = _mm_unpacklo_epi16(_s0l, _s0h);

  2133.                         __m128i _s01 = _mm_unpackhi_epi16(_s0l, _s0h);

  2134.                         __m128i _s10 = _mm_unpacklo_epi16(_s1l, _s1h);

  2135.                         __m128i _s11 = _mm_unpackhi_epi16(_s1l, _s1h);

  2136.                         __m128i _s20 = _mm_unpacklo_epi16(_s2l, _s2h);

  2137.                         __m128i _s21 = _mm_unpackhi_epi16(_s2l, _s2h);

  2138.                         __m128i _s30 = _mm_unpacklo_epi16(_s3l, _s3h);

  2139.                         __m128i _s31 = _mm_unpackhi_epi16(_s3l, _s3h);

  2140. 

  2141.                         _sum00 = _mm_add_epi32(_sum00, _s00);

  2142.                         _sum01 = _mm_add_epi32(_sum01, _s01);

  2143.                         _sum10 = _mm_add_epi32(_sum10, _s10);

  2144.                         _sum11 = _mm_add_epi32(_sum11, _s11);

  2145.                         _sum20 = _mm_add_epi32(_sum20, _s20);

  2146.                         _sum21 = _mm_add_epi32(_sum21, _s21);

  2147.                         _sum30 = _mm_add_epi32(_sum30, _s30);

  2148.                         _sum31 = _mm_add_epi32(_sum31, _s31);

  2149. 

  2150.                         m0++;

  2151.                         m1++;

  2152.                         m2++;

  2153.                         m3++;

  2154.                         kptr += 8;

  2155.                     }

  2156. 

  2157.                     // dequantize and relu

  2158.                     __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);

  2159.                     __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);

  2160. 

  2161.                     __m128 _sumfp32_00 = _mm_cvtepi32_ps(_sum00);

  2162.                     __m128 _sumfp32_01 = _mm_cvtepi32_ps(_sum01);

  2163.                     __m128 _sumfp32_10 = _mm_cvtepi32_ps(_sum10);

  2164.                     __m128 _sumfp32_11 = _mm_cvtepi32_ps(_sum11);

  2165.                     __m128 _sumfp32_20 = _mm_cvtepi32_ps(_sum20);

  2166.                     __m128 _sumfp32_21 = _mm_cvtepi32_ps(_sum21);

  2167.                     __m128 _sumfp32_30 = _mm_cvtepi32_ps(_sum30);

  2168.                     __m128 _sumfp32_31 = _mm_cvtepi32_ps(_sum31);

  2169.                     if (bias_term)

  2170.                     {

  2171.                         __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);

  2172.                         __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);

  2173.                         _sumfp32_00 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_00, _scale_in0));

  2174.                         _sumfp32_01 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_01, _scale_in1));

  2175.                         _sumfp32_10 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_10, _scale_in0));

  2176.                         _sumfp32_11 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_11, _scale_in1));

  2177.                         _sumfp32_20 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_20, _scale_in0));

  2178.                         _sumfp32_21 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_21, _scale_in1));

  2179.                         _sumfp32_30 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_30, _scale_in0));

  2180.                         _sumfp32_31 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_31, _scale_in1));

  2181.                     }

  2182.                     else

  2183.                     {

  2184.                         _sumfp32_00 = _mm_mul_ps(_sumfp32_00, _scale_in0);

  2185.                         _sumfp32_01 = _mm_mul_ps(_sumfp32_01, _scale_in1);

  2186.                         _sumfp32_10 = _mm_mul_ps(_sumfp32_10, _scale_in0);

  2187.                         _sumfp32_11 = _mm_mul_ps(_sumfp32_11, _scale_in1);

  2188.                         _sumfp32_20 = _mm_mul_ps(_sumfp32_20, _scale_in0);

  2189.                         _sumfp32_21 = _mm_mul_ps(_sumfp32_21, _scale_in1);

  2190.                         _sumfp32_30 = _mm_mul_ps(_sumfp32_30, _scale_in0);

  2191.                         _sumfp32_31 = _mm_mul_ps(_sumfp32_31, _scale_in1);

  2192.                     }

  2193. 

  2194.                     _sumfp32_00 = activation_sse(_sumfp32_00, activation_type, activation_params);

  2195.                     _sumfp32_01 = activation_sse(_sumfp32_01, activation_type, activation_params);

  2196.                     _sumfp32_10 = activation_sse(_sumfp32_10, activation_type, activation_params);

  2197.                     _sumfp32_11 = activation_sse(_sumfp32_11, activation_type, activation_params);

  2198.                     _sumfp32_20 = activation_sse(_sumfp32_20, activation_type, activation_params);

  2199.                     _sumfp32_21 = activation_sse(_sumfp32_21, activation_type, activation_params);

  2200.                     _sumfp32_30 = activation_sse(_sumfp32_30, activation_type, activation_params);

  2201.                     _sumfp32_31 = activation_sse(_sumfp32_31, activation_type, activation_params);

  2202. 

  2203.                     // transpose 4x8

  2204.                     _MM_TRANSPOSE4_PS(_sumfp32_00, _sumfp32_10, _sumfp32_20, _sumfp32_30);

  2205.                     _MM_TRANSPOSE4_PS(_sumfp32_01, _sumfp32_11, _sumfp32_21, _sumfp32_31);

  2206. 

  2207.                     _mm_storeu_ps(outptr, _sumfp32_00);

  2208.                     _mm_storeu_ps(outptr + 4, _sumfp32_10);

  2209.                     _mm_storeu_ps(outptr + 8, _sumfp32_20);

  2210.                     _mm_storeu_ps(outptr + 12, _sumfp32_30);

  2211.                     _mm_storeu_ps(outptr + 16, _sumfp32_01);

  2212.                     _mm_storeu_ps(outptr + 20, _sumfp32_11);

  2213.                     _mm_storeu_ps(outptr + 24, _sumfp32_21);

  2214.                     _mm_storeu_ps(outptr + 28, _sumfp32_31);

  2215. 

  2216.                     outptr += 32;

  2217.                 }

  2218.             }

  2219.         }

  2220. 

  2221.         if (num_output_elempack == 1 && out_elempack == 4)

  2222.         {

  2223.             #pragma omp parallel for num_threads(opt.num_threads)

  2224.             for (int j = 0; j < outh; j++)

  2225.             {

  2226.                 float* outptr = top_blob.row(j);

  2227. 

  2228.                 for (int p = 0; p < num_output; p++)

  2229.                 {

  2230.                     const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2231.                     const signed char* m0 = bottom_blob_int8_unpacked.row<const signed char>(j * 4);

  2232.                     const signed char* m1 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 1);

  2233.                     const signed char* m2 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 2);

  2234.                     const signed char* m3 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 3);

  2235. 

  2236.                     int sum0 = 0;

  2237.                     int sum1 = 0;

  2238.                     int sum2 = 0;

  2239.                     int sum3 = 0;

  2240. 

  2241.                     int i = 0;

  2242.                     for (; i < num_input; i++)

  2243.                     {

  2244.                         sum0 += *m0++ * kptr[0];

  2245.                         sum1 += *m1++ * kptr[0];

  2246.                         sum2 += *m2++ * kptr[0];

  2247.                         sum3 += *m3++ * kptr[0];

  2248.                         kptr += 1;

  2249.                     }

  2250. 

  2251.                     // dequantize and relu

  2252.                     float sumfp32_0 = sum0 * scale_in_data[p];

  2253.                     float sumfp32_1 = sum1 * scale_in_data[p];

  2254.                     float sumfp32_2 = sum2 * scale_in_data[p];

  2255.                     float sumfp32_3 = sum3 * scale_in_data[p];

  2256. 

  2257.                     if (bias_term)

  2258.                     {

  2259.                         sumfp32_0 += bias_data[p];

  2260.                         sumfp32_1 += bias_data[p];

  2261.                         sumfp32_2 += bias_data[p];

  2262.                         sumfp32_3 += bias_data[p];

  2263.                     }

  2264. 

  2265.                     outptr[0] = activation_ss(sumfp32_0, activation_type, activation_params);

  2266.                     outptr[1] = activation_ss(sumfp32_1, activation_type, activation_params);

  2267.                     outptr[2] = activation_ss(sumfp32_2, activation_type, activation_params);

  2268.                     outptr[3] = activation_ss(sumfp32_3, activation_type, activation_params);

  2269.                     outptr += 4;

  2270.                 }

  2271.             }

  2272.         }

  2273. 

  2274.         if (num_output_elempack == 8 && out_elempack == 1)

  2275.         {

  2276.             #pragma omp parallel for num_threads(opt.num_threads)

  2277.             for (int j = 0; j < outh; j++)

  2278.             {

  2279.                 float* outptr = top_blob.row(j);

  2280. 

  2281.                 for (int p = 0; p < num_output / num_output_elempack; p++)

  2282.                 {

  2283.                     const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2284.                     const signed char* m = bottom_blob_int8_unpacked.row<const signed char>(j);

  2285. 

  2286.                     __m128i _sum0 = _mm_setzero_si128();

  2287.                     __m128i _sum1 = _mm_setzero_si128();

  2288. 

  2289.                     int i = 0;

  2290.                     for (; i < num_input; i++)

  2291.                     {

  2292.                         __m128i _val = _mm_set1_epi16((short)m[0]);

  2293. 

  2294.                         // TODO use _mm_cvtepi8_epi16 on sse4.1

  2295.                         __m128i _w = _mm_loadl_epi64((const __m128i*)kptr);

  2296.                         _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w));

  2297. 

  2298.                         __m128i _sl = _mm_mullo_epi16(_val, _w);

  2299.                         __m128i _sh = _mm_mulhi_epi16(_val, _w);

  2300.                         __m128i _s0 = _mm_unpacklo_epi16(_sl, _sh);

  2301.                         __m128i _s1 = _mm_unpackhi_epi16(_sl, _sh);

  2302. 

  2303.                         _sum0 = _mm_add_epi32(_sum0, _s0);

  2304.                         _sum1 = _mm_add_epi32(_sum1, _s1);

  2305. 

  2306.                         m++;

  2307.                         kptr += 8;

  2308.                     }

  2309. 

  2310.                     // dequantize and relu

  2311.                     __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);

  2312.                     __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);

  2313. 

  2314.                     __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0);

  2315.                     __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1);

  2316. 

  2317.                     if (bias_term)

  2318.                     {

  2319.                         __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);

  2320.                         __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);

  2321.                         _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0));

  2322.                         _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1));

  2323.                     }

  2324.                     else

  2325.                     {

  2326.                         _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0);

  2327.                         _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1);

  2328.                     }

  2329. 

  2330.                     _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params);

  2331.                     _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params);

  2332. 

  2333.                     _mm_storeu_ps(outptr, _sumfp32_0);

  2334.                     _mm_storeu_ps(outptr + 4, _sumfp32_1);

  2335.                     outptr += 8;

  2336.                 }

  2337.             }

  2338.         }

  2339. #endif // __SSE2__

  2340. 

  2341.         if (num_output_elempack == 1 && out_elempack == 1)

  2342.         {

  2343.             #pragma omp parallel for num_threads(opt.num_threads)

  2344.             for (int j = 0; j < outh; j++)

  2345.             {

  2346.                 float* outptr = top_blob.row(j);

  2347. 

  2348.                 for (int p = 0; p < num_output; p++)

  2349.                 {

  2350.                     const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2351.                     const signed char* m = bottom_blob_int8_unpacked.row<const signed char>(j);

  2352. 

  2353.                     int sum = 0;

  2354. 

  2355.                     int i = 0;

  2356.                     for (; i < num_input; i++)

  2357.                     {

  2358.                         sum += *m++ * *kptr++;

  2359.                     }

  2360. 

  2361.                     // dequantize and relu

  2362.                     float sumfp32 = sum * scale_in_data[p];

  2363. 

  2364.                     if (bias_term)

  2365.                         sumfp32 += bias_data[p];

  2366. 

  2367.                     outptr[0] = activation_ss(sumfp32, activation_type, activation_params);

  2368.                     outptr += 1;

  2369.                 }

  2370.             }

  2371.         }

  2372. 

  2373.         return 0;

  2374.     }

  2375. 

  2376.     Mat bottom_blob_int8_flattened = bottom_blob_int8;

  2377.     if (bottom_blob_int8.dims != 1)

  2378.     {

  2379.         Option opt_flatten = opt;

  2380.         opt_flatten.blob_allocator = opt.workspace_allocator;

  2381.         flatten->forward(bottom_blob_int8, bottom_blob_int8_flattened, opt_flatten);

  2382.     }

  2383. 

  2384.     //     int elempack = bottom_blob_int8_flattened.elempack;

  2385. 

  2386.     int out_elempack = 1;

  2387. #if __SSE2__

  2388.     if (opt.use_packing_layout)

  2389.     {

  2390.         out_elempack = num_output % 8 == 0 ? 8 : 1;

  2391.     }

  2392. #endif // __SSE2__

  2393.     //     size_t out_elemsize = elemsize / elempack * out_elempack;

  2394. 

  2395.     top_blob.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator);

  2396.     if (top_blob.empty())

  2397.         return -100;

  2398. 

  2399. #if __SSE2__

  2400.     if (out_elempack == 8)

  2401.     {

  2402.         #pragma omp parallel for num_threads(opt.num_threads)

  2403.         for (int p = 0; p < num_output / out_elempack; p++)

  2404.         {

  2405.             __m128i _sum0 = _mm_setzero_si128();

  2406.             __m128i _sum1 = _mm_setzero_si128();

  2407. 

  2408.             const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2409.             const signed char* sptr = bottom_blob_int8_flattened;

  2410. 

  2411.             int i = 0;

  2412.             for (; i < num_input; i++)

  2413.             {

  2414.                 __m128i _val = _mm_set1_epi16((short)sptr[0]);

  2415. 

  2416.                 // TODO use _mm_cvtepi8_epi16 on sse4.1

  2417.                 __m128i _w = _mm_loadl_epi64((const __m128i*)kptr);

  2418.                 _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w));

  2419. 

  2420.                 __m128i _sl = _mm_mullo_epi16(_val, _w);

  2421.                 __m128i _sh = _mm_mulhi_epi16(_val, _w);

  2422.                 __m128i _s0 = _mm_unpacklo_epi16(_sl, _sh);

  2423.                 __m128i _s1 = _mm_unpackhi_epi16(_sl, _sh);

  2424. 

  2425.                 _sum0 = _mm_add_epi32(_sum0, _s0);

  2426.                 _sum1 = _mm_add_epi32(_sum1, _s1);

  2427. 

  2428.                 sptr += 1;

  2429.                 kptr += 8;

  2430.             }

  2431. 

  2432.             // dequantize and relu

  2433.             __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);

  2434.             __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);

  2435. 

  2436.             __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0);

  2437.             __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1);

  2438. 

  2439.             if (bias_term)

  2440.             {

  2441.                 __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);

  2442.                 __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);

  2443.                 _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0));

  2444.                 _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1));

  2445.             }

  2446.             else

  2447.             {

  2448.                 _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0);

  2449.                 _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1);

  2450.             }

  2451. 

  2452.             _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params);

  2453.             _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params);

  2454. 

  2455.             float* outptr = (float*)top_blob + p * 8;

  2456.             _mm_storeu_ps(outptr, _sumfp32_0);

  2457.             _mm_storeu_ps(outptr + 4, _sumfp32_1);

  2458.         }

  2459.     }

  2460. #endif // __SSE2__

  2461. 

  2462.     if (out_elempack == 1)

  2463.     {

  2464.         #pragma omp parallel for num_threads(opt.num_threads)

  2465.         for (int p = 0; p < num_output / out_elempack; p++)

  2466.         {

  2467.             int sum = 0;

  2468. 

  2469.             const signed char* kptr = weight_data_tm.row<const signed char>(p);

  2470.             const signed char* sptr = bottom_blob_int8_flattened;

  2471. 

  2472.             int i = 0;

  2473.             for (; i < num_input; i++)

  2474.             {

  2475.                 signed char val = sptr[0];

  2476. 

  2477.                 signed char w = kptr[0];

  2478. 

  2479.                 sum += val * w;

  2480. 

  2481.                 sptr += 1;

  2482.                 kptr += 1;

  2483.             }

  2484. 

  2485.             // dequantize and relu

  2486.             float sumfp32 = sum * scale_in_data[p];

  2487. 

  2488.             if (bias_term)

  2489.                 sumfp32 += bias_data[p];

  2490. 

  2491.             sumfp32 = activation_ss(sumfp32, activation_type, activation_params);

  2492. 

  2493.             top_blob[p] = sumfp32;

  2494.         }

  2495.     }

  2496. 

  2497.     return 0;

  2498. }

  2499. #endif // NCNN_INT8

  2500. 

  2501. } // namespace ncnn