nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 90e6be457b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '90e6be457b'
${ noResults }
ncnn/src/layer/arm/convolution_arm.cpp

1916 lines
70 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #include "convolution_arm.h"

  16. #include "benchmark.h"

  17. #include "cpu.h"

  18. 

  19. #include "layer_type.h"

  20. 

  21. #if __ARM_NEON

  22. #include <arm_neon.h>

  23. #include "neon_mathfun.h"

  24. #include "neon_activation.h"

  25. #endif // __ARM_NEON

  26. 

  27. namespace ncnn {

  28. 

  29. #include "convolution_1x1.h"

  30. #include "convolution_2x2.h"

  31. #include "convolution_3x3.h"

  32. #include "convolution_4x4.h"

  33. #include "convolution_5x5.h"

  34. #include "convolution_7x7.h"

  35. #include "convolution_sgemm.h"

  36. #include "convolution_sgemm_int8.h"

  37. #include "convolution_1x1_int8.h"

  38. #include "convolution_3x3_int8.h"

  39. 

  40. #include "convolution_1x1_bf16s.h"

  41. 

  42. #if __ARM_NEON

  43. #include "convolution_1x1_pack4.h"

  44. #include "convolution_1x1_pack4to1.h"

  45. #include "convolution_3x3_pack4.h"

  46. #include "convolution_3x3_pack1to4.h"

  47. #include "convolution_3x3_pack4to1.h"

  48. #include "convolution_5x5_pack4.h"

  49. #include "convolution_7x7_pack1to4.h"

  50. 

  51. #include "convolution_1x1_pack4_bf16s.h"

  52. #include "convolution_1x1_pack4to1_bf16s.h"

  53. #include "convolution_3x3_pack4_bf16s.h"

  54. #include "convolution_3x3_pack1to4_bf16s.h"

  55. #include "convolution_3x3_pack4to1_bf16s.h"

  56. #include "convolution_5x5_pack4_bf16s.h"

  57. #include "convolution_7x7_pack1to4_bf16s.h"

  58. #endif // __ARM_NEON

  59. 

  60. 

  61. DEFINE_LAYER_CREATOR(Convolution_arm)

  62. 

  63. Convolution_arm::Convolution_arm()

  64. {

  65. #if __ARM_NEON

  66.     support_packing = true;

  67. #endif // __ARM_NEON

  68. 

  69.     support_bf16_storage = true;

  70. 

  71.     activation = 0;

  72.     convolution_dilation1 = 0;

  73. }

  74. 

  75. int Convolution_arm::create_pipeline(const Option& opt)

  76. {

  77.     if (activation_type == 1)

  78.     {

  79.         activation = ncnn::create_layer(ncnn::LayerType::ReLU);

  80. 

  81.         ncnn::ParamDict pd;

  82.         activation->load_param(pd);

  83.     }

  84.     else if (activation_type == 2)

  85.     {

  86.         activation = ncnn::create_layer(ncnn::LayerType::ReLU);

  87. 

  88.         ncnn::ParamDict pd;

  89.         pd.set(0, activation_params[0]);// slope

  90.         activation->load_param(pd);

  91.     }

  92.     else if (activation_type == 3)

  93.     {

  94.         activation = ncnn::create_layer(ncnn::LayerType::Clip);

  95. 

  96.         ncnn::ParamDict pd;

  97.         pd.set(0, activation_params[0]);// min

  98.         pd.set(1, activation_params[1]);// max

  99.         activation->load_param(pd);

  100.     }

  101.     else if (activation_type == 4)

  102.     {

  103.         activation = ncnn::create_layer(ncnn::LayerType::Sigmoid);

  104. 

  105.         ncnn::ParamDict pd;

  106.         activation->load_param(pd);

  107.     }

  108. 

  109.     if (activation)

  110.     {

  111.         activation->create_pipeline(opt);

  112.     }

  113. 

  114.     if (opt.use_bf16_storage)

  115.     {

  116.         return create_pipeline_bf16s(opt);

  117.     }

  118. 

  119.     if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)

  120.     {

  121.         support_packing = false;

  122. 

  123.         return create_pipeline_int8_arm(opt);

  124.     }

  125. 

  126.     if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)

  127.     {

  128.         convolution_dilation1 = ncnn::create_layer(ncnn::LayerType::Convolution);

  129. 

  130.         // set param

  131.         ncnn::ParamDict pd;

  132.         pd.set(0, num_output);// num_output

  133.         pd.set(1, kernel_w);

  134.         pd.set(11, kernel_h);

  135.         pd.set(2, 1);

  136.         pd.set(12, 1);

  137.         pd.set(3, 1);// stride_w

  138.         pd.set(13, 1);// stride_h

  139.         pd.set(4, 0);// pad_w

  140.         pd.set(14, 0);// pad_h

  141.         pd.set(5, bias_term);

  142.         pd.set(6, weight_data_size);

  143. 

  144.         convolution_dilation1->load_param(pd);

  145. 

  146.         // set weights

  147.         if (bias_term)

  148.         {

  149.             ncnn::Mat weights[2];

  150.             weights[0] = weight_data;

  151.             weights[1] = bias_data;

  152. 

  153.             convolution_dilation1->load_model(ModelBinFromMatArray(weights));

  154.         }

  155.         else

  156.         {

  157.             ncnn::Mat weights[1];

  158.             weights[0] = weight_data;

  159. 

  160.             convolution_dilation1->load_model(ModelBinFromMatArray(weights));

  161.         }

  162. 

  163.         convolution_dilation1->create_pipeline(opt);

  164. 

  165.         return 0;

  166.     }

  167. 

  168.     const int maxk = kernel_w * kernel_h;

  169.     const int num_input = weight_data_size / maxk / num_output;

  170. 

  171.     int elempack = (opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1;

  172.     int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;

  173. 

  174. #if __ARM_NEON

  175.     // pack4

  176.     if (elempack == 4 && out_elempack == 4)

  177.     {

  178.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  179.         {

  180.             conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);

  181.         }

  182.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  183.         {

  184.             conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);

  185.         }

  186.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  187.         {

  188.             conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);

  189.         }

  190.         else

  191.         {

  192.             // src = kw-kh-inch-outch

  193.             // dst = 4b-4a-kw-kh-inch/4a-outch/4b

  194.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  195. 

  196.             weight_data_pack4.create(maxk, num_input/4, num_output/4, (size_t)4*16, 16);

  197. 

  198.             for (int q=0; q+3<num_output; q+=4)

  199.             {

  200.                 const Mat k0 = weight_data_r2.channel(q);

  201.                 const Mat k1 = weight_data_r2.channel(q+1);

  202.                 const Mat k2 = weight_data_r2.channel(q+2);

  203.                 const Mat k3 = weight_data_r2.channel(q+3);

  204. 

  205.                 Mat g0 = weight_data_pack4.channel(q/4);

  206. 

  207.                 for (int p=0; p+3<num_input; p+=4)

  208.                 {

  209.                     const float* k00 = k0.row(p);

  210.                     const float* k01 = k0.row(p+1);

  211.                     const float* k02 = k0.row(p+2);

  212.                     const float* k03 = k0.row(p+3);

  213. 

  214.                     const float* k10 = k1.row(p);

  215.                     const float* k11 = k1.row(p+1);

  216.                     const float* k12 = k1.row(p+2);

  217.                     const float* k13 = k1.row(p+3);

  218. 

  219.                     const float* k20 = k2.row(p);

  220.                     const float* k21 = k2.row(p+1);

  221.                     const float* k22 = k2.row(p+2);

  222.                     const float* k23 = k2.row(p+3);

  223. 

  224.                     const float* k30 = k3.row(p);

  225.                     const float* k31 = k3.row(p+1);

  226.                     const float* k32 = k3.row(p+2);

  227.                     const float* k33 = k3.row(p+3);

  228. 

  229.                     float* g00 = g0.row(p/4);

  230. 

  231.                     for (int k=0; k<maxk; k++)

  232.                     {

  233.                         g00[0] = k00[k];

  234.                         g00[1] = k10[k];

  235.                         g00[2] = k20[k];

  236.                         g00[3] = k30[k];

  237. 

  238.                         g00[4] = k01[k];

  239.                         g00[5] = k11[k];

  240.                         g00[6] = k21[k];

  241.                         g00[7] = k31[k];

  242. 

  243.                         g00[8] = k02[k];

  244.                         g00[9] = k12[k];

  245.                         g00[10] = k22[k];

  246.                         g00[11] = k32[k];

  247. 

  248.                         g00[12] = k03[k];

  249.                         g00[13] = k13[k];

  250.                         g00[14] = k23[k];

  251.                         g00[15] = k33[k];

  252. 

  253.                         g00 += 16;

  254.                     }

  255.                 }

  256.             }

  257.         }

  258.     }

  259. 

  260.     // pack1to4

  261.     if (elempack == 1 && out_elempack == 4)

  262.     {

  263.         // src = kw-kh-inch-outch

  264.         // dst = 4b-kw-kh-inch-outch/4b

  265.         {

  266.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  267. 

  268.             weight_data_pack1to4.create(maxk, num_input, num_output/4, (size_t)4*4, 4);

  269. 

  270.             for (int q=0; q+3<num_output; q+=4)

  271.             {

  272.                 const Mat k0 = weight_data_r2.channel(q);

  273.                 const Mat k1 = weight_data_r2.channel(q+1);

  274.                 const Mat k2 = weight_data_r2.channel(q+2);

  275.                 const Mat k3 = weight_data_r2.channel(q+3);

  276. 

  277.                 Mat g0 = weight_data_pack1to4.channel(q/4);

  278. 

  279.                 for (int p=0; p<num_input; p++)

  280.                 {

  281.                     const float* k00 = k0.row(p);

  282.                     const float* k10 = k1.row(p);

  283.                     const float* k20 = k2.row(p);

  284.                     const float* k30 = k3.row(p);

  285. 

  286.                     float* g00 = g0.row(p);

  287. 

  288.                     for (int k=0; k<maxk; k++)

  289.                     {

  290.                         g00[0] = k00[k];

  291.                         g00[1] = k10[k];

  292.                         g00[2] = k20[k];

  293.                         g00[3] = k30[k];

  294. 

  295.                         g00 += 4;

  296.                     }

  297.                 }

  298.             }

  299.         }

  300.     }

  301. 

  302.     // pack4to1

  303.     if (elempack == 4 && out_elempack == 1)

  304.     {

  305.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  306.         {

  307.             conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);

  308.         }

  309.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  310.         {

  311.             conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);

  312.         }

  313.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  314.         {

  315.             conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);

  316.         }

  317.         else

  318.         {

  319.             // src = kw-kh-inch-outch

  320.             // dst = 4a-kw-kh-inch/4a-outch

  321.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  322. 

  323.             weight_data_pack4to1.create(maxk, num_input/4, num_output, (size_t)4*4, 4);

  324. 

  325.             for (int q=0; q<num_output; q++)

  326.             {

  327.                 const Mat k0 = weight_data_r2.channel(q);

  328.                 Mat g0 = weight_data_pack4to1.channel(q);

  329. 

  330.                 for (int p=0; p+3<num_input; p+=4)

  331.                 {

  332.                     const float* k00 = k0.row(p);

  333.                     const float* k01 = k0.row(p+1);

  334.                     const float* k02 = k0.row(p+2);

  335.                     const float* k03 = k0.row(p+3);

  336. 

  337.                     float* g00 = g0.row(p/4);

  338. 

  339.                     for (int k=0; k<maxk; k++)

  340.                     {

  341.                         g00[0] = k00[k];

  342.                         g00[1] = k01[k];

  343.                         g00[2] = k02[k];

  344.                         g00[3] = k03[k];

  345. 

  346.                         g00 += 4;

  347.                     }

  348.                 }

  349.             }

  350.         }

  351.     }

  352. #endif // __ARM_NEON

  353. 

  354.     // pack1

  355.     if (elempack == 1 && out_elempack == 1)

  356.     {

  357.         use_winograd3x3 = false;

  358.         use_sgemm1x1 = false;

  359. 

  360.         if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  361.         {

  362.             // winograd is slow on small channel count

  363.             if (num_input >= 16 && num_output >= 16)

  364.                 use_winograd3x3 = true;

  365. 

  366.             if (use_winograd3x3)

  367.             {

  368. //                 conv3x3s1_winograd64_transform_kernel_neon(weight_data, weight_3x3_winograd64_data, num_input, num_output);

  369.                 conv3x3s1_winograd64_transform_kernel_neon5(weight_data, weight_3x3_winograd64_data, num_input, num_output);

  370.             }

  371.         }

  372. 

  373.         // TODO assume more proper condition

  374.         if (opt.use_sgemm_convolution && kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  375.         {

  376.             if (num_input >= 64 && num_output >= 64)

  377.                 use_sgemm1x1 = true;

  378. 

  379.             if (use_sgemm1x1)

  380.             {

  381.                 conv1x1s1_sgemm_transform_kernel_neon(weight_data, weight_1x1_sgemm_data, num_input, num_output);

  382.             }

  383.         }

  384. 

  385.         if (impl_type > 0 && impl_type < 6 && impl_type != 4)

  386.         {

  387.             switch (impl_type)

  388.             {

  389.                 case 1:

  390.                     // winograd

  391.                     conv3x3s1_winograd64_transform_kernel_neon5(weight_data, weight_3x3_winograd64_data, num_input, num_output);

  392.                     break;

  393.                 case 2:

  394.                     // pointwise

  395.                     conv1x1s1_sgemm_transform_kernel_neon(weight_data, weight_1x1_sgemm_data, num_input, num_output);

  396.                     break;

  397.                 case 3:

  398.                     // im2col

  399.                     conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk);

  400.                     break;

  401. //                 case 4:

  402. //                     // direct

  403. //                     break;

  404.                 case 5:

  405.                     // conv3x3s2

  406.                     conv3x3s2_transform_kernel_neon(weight_data, weight_3x3s2_data, num_input, num_output);

  407.                     break;

  408.             }

  409.         }

  410. 

  411.         if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  412.         {

  413.             conv3x3s2_transform_kernel_neon(weight_data, weight_3x3s2_data, num_input, num_output);

  414.         }

  415. 

  416.         if (opt.use_sgemm_convolution && kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  417.         {

  418.             conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk);

  419.         }

  420. 

  421.         if (opt.use_sgemm_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  422.         {

  423.             conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk);

  424.         }

  425.     }

  426. 

  427.     return 0;

  428. }

  429. 

  430. int Convolution_arm::destroy_pipeline(const Option& opt)

  431. {

  432.     if (activation)

  433.     {

  434.         activation->destroy_pipeline(opt);

  435.         delete activation;

  436.         activation = 0;

  437.     }

  438. 

  439.     if (convolution_dilation1)

  440.     {

  441.         convolution_dilation1->destroy_pipeline(opt);

  442.         delete convolution_dilation1;

  443.         convolution_dilation1 = 0;

  444.     }

  445. 

  446.     return 0;

  447. }

  448. 

  449. int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  450. {

  451.     if (bottom_blob.dims != 3)

  452.     {

  453.         return Convolution::forward(bottom_blob, top_blob, opt);

  454.     }

  455. 

  456.     if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)

  457.     {

  458.         return forward_int8_arm(bottom_blob, top_blob, opt);

  459.     }

  460. 

  461.     if (opt.use_bf16_storage)

  462.         return forward_bf16s(bottom_blob, top_blob, opt);

  463. 

  464.     int w = bottom_blob.w;

  465.     int h = bottom_blob.h;

  466.     int channels = bottom_blob.c;

  467.     size_t elemsize = bottom_blob.elemsize;

  468.     int elempack = bottom_blob.elempack;

  469. 

  470. //     fprintf(stderr, "Convolution input %d x %d  pad = %d %d  ksize=%d %d  stride=%d %d\n", w, h, pad_w, pad_h, kernel_w, kernel_h, stride_w, stride_h);

  471. 

  472.     const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;

  473.     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;

  474. 

  475.     Mat bottom_blob_bordered;

  476.     make_padding(bottom_blob, bottom_blob_bordered, opt);

  477.     if (bottom_blob_bordered.empty())

  478.         return -100;

  479. 

  480.     w = bottom_blob_bordered.w;

  481.     h = bottom_blob_bordered.h;

  482. 

  483.     int outw = (w - kernel_extent_w) / stride_w + 1;

  484.     int outh = (h - kernel_extent_h) / stride_h + 1;

  485.     int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;

  486.     size_t out_elemsize = elemsize / elempack * out_elempack;

  487. 

  488.     top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);

  489.     if (top_blob.empty())

  490.         return -100;

  491. 

  492.     if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)

  493.     {

  494.         return forwardDilation_arm(bottom_blob_bordered, top_blob, opt);

  495.     }

  496. 

  497.     const int maxk = kernel_w * kernel_h;

  498. 

  499.     // kernel offsets

  500.     std::vector<int> _space_ofs(maxk);

  501.     int* space_ofs = &_space_ofs[0];

  502.     {

  503.         int p1 = 0;

  504.         int p2 = 0;

  505.         int gap = w * dilation_h - kernel_w * dilation_w;

  506.         for (int i = 0; i < kernel_h; i++)

  507.         {

  508.             for (int j = 0; j < kernel_w; j++)

  509.             {

  510.                 space_ofs[p1] = p2;

  511.                 p1++;

  512.                 p2 += dilation_w;

  513.             }

  514.             p2 += gap;

  515.         }

  516.     }

  517. 

  518. #if __ARM_NEON

  519.     if (elempack == 4 && out_elempack == 4)

  520.     {

  521.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  522.         {

  523.             conv1x1s1_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  524. 

  525.             if (activation)

  526.             {

  527.                 activation->forward_inplace(top_blob, opt);

  528.             }

  529.         }

  530.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  531.         {

  532.             conv1x1s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  533. 

  534.             if (activation)

  535.             {

  536.                 activation->forward_inplace(top_blob, opt);

  537.             }

  538.         }

  539.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  540.         {

  541.             conv3x3s1_winograd64_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  542. 

  543.             if (activation)

  544.             {

  545.                 activation->forward_inplace(top_blob, opt);

  546.             }

  547.         }

  548.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  549.         {

  550.             conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  551. 

  552.             if (activation)

  553.             {

  554.                 activation->forward_inplace(top_blob, opt);

  555.             }

  556.         }

  557.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  558.         {

  559.             conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  560. 

  561.             if (activation)

  562.             {

  563.                 activation->forward_inplace(top_blob, opt);

  564.             }

  565.         }

  566.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  567.         {

  568.             conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);

  569. 

  570.             if (activation)

  571.             {

  572.                 activation->forward_inplace(top_blob, opt);

  573.             }

  574.         }

  575.         else

  576.         {

  577.             // num_output

  578.             #pragma omp parallel for num_threads(opt.num_threads)

  579.             for (int p=0; p<num_output / out_elempack; p++)

  580.             {

  581.                 float* outptr = top_blob.channel(p);

  582. 

  583.                 for (int i = 0; i < outh; i++)

  584.                 {

  585.                     for (int j = 0; j < outw; j++)

  586.                     {

  587.                         float32x4_t _sum = vdupq_n_f32(0.f);

  588. 

  589.                         if (bias_term)

  590.                         {

  591.                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);

  592.                         }

  593. 

  594.                         const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16;

  595. 

  596.                         // channels

  597.                         for (int q=0; q<channels; q++)

  598.                         {

  599.                             const Mat m = bottom_blob_bordered.channel(q);

  600.                             const float* sptr = m.row(i*stride_h) + j*stride_w * 4;

  601. 

  602.                             for (int k = 0; k < maxk; k++) // 29.23

  603.                             {

  604.                                 float32x4_t _val = vld1q_f32( sptr + space_ofs[k] * 4 );

  605. 

  606.                                 float32x4_t _w0 = vld1q_f32( kptr );

  607.                                 float32x4_t _w1 = vld1q_f32( kptr + 4 );

  608.                                 float32x4_t _w2 = vld1q_f32( kptr + 8 );

  609.                                 float32x4_t _w3 = vld1q_f32( kptr + 12 );

  610. 

  611. #if __aarch64__

  612.                                 _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);

  613.                                 _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);

  614.                                 _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);

  615.                                 _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);

  616. #else

  617.                                 _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);

  618.                                 _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);

  619.                                 _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);

  620.                                 _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);

  621. #endif

  622. 

  623.                                 kptr += 16;

  624.                             }

  625.                         }

  626. 

  627.                         _sum = activation_ps(_sum, activation_type, activation_params);

  628. 

  629.                         vst1q_f32(outptr + j * 4, _sum);

  630.                     }

  631. 

  632.                     outptr += outw * 4;

  633.                 }

  634.             }

  635.         }

  636.     }

  637. 

  638.     if (elempack == 1 && out_elempack == 4)

  639.     {

  640.         if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  641.         {

  642.             conv3x3s1_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);

  643. 

  644.             if (activation)

  645.             {

  646.                 activation->forward_inplace(top_blob, opt);

  647.             }

  648.         }

  649.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  650.         {

  651.             conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);

  652. 

  653.             if (activation)

  654.             {

  655.                 activation->forward_inplace(top_blob, opt);

  656.             }

  657.         }

  658.         else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  659.         {

  660.             conv7x7s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);

  661. 

  662.             if (activation)

  663.             {

  664.                 activation->forward_inplace(top_blob, opt);

  665.             }

  666.         }

  667.         else

  668.         {

  669.             // num_output

  670.             #pragma omp parallel for num_threads(opt.num_threads)

  671.             for (int p=0; p<num_output / out_elempack; p++)

  672.             {

  673.                 float* outptr = top_blob.channel(p);

  674. 

  675.                 for (int i = 0; i < outh; i++)

  676.                 {

  677.                     for (int j = 0; j < outw; j++)

  678.                     {

  679.                         float32x4_t _sum = vdupq_n_f32(0.f);

  680. 

  681.                         if (bias_term)

  682.                         {

  683.                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);

  684.                         }

  685. 

  686.                         const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4;

  687. 

  688.                         // channels

  689.                         for (int q=0; q<channels; q++)

  690.                         {

  691.                             const Mat m = bottom_blob_bordered.channel(q);

  692.                             const float* sptr = m.row(i*stride_h) + j*stride_w;

  693. 

  694.                             for (int k = 0; k < maxk; k++) // 29.23

  695.                             {

  696.                                 float32x4_t _val = vdupq_n_f32( sptr[ space_ofs[k] ] );

  697.                                 float32x4_t _w = vld1q_f32( kptr );

  698.                                 _sum = vmlaq_f32(_sum, _val, _w);

  699. 

  700.                                 kptr += 4;

  701.                             }

  702.                         }

  703. 

  704.                         _sum = activation_ps(_sum, activation_type, activation_params);

  705. 

  706.                         vst1q_f32(outptr + j * 4, _sum);

  707.                     }

  708. 

  709.                     outptr += outw * 4;

  710.                 }

  711.             }

  712.         }

  713.     }

  714. 

  715.     if (elempack == 4 && out_elempack == 1)

  716.     {

  717.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  718.         {

  719.             conv1x1s1_sgemm_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);

  720. 

  721.             if (activation)

  722.             {

  723.                 activation->forward_inplace(top_blob, opt);

  724.             }

  725.         }

  726.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  727.         {

  728.             conv1x1s2_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);

  729. 

  730.             if (activation)

  731.             {

  732.                 activation->forward_inplace(top_blob, opt);

  733.             }

  734.         }

  735.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  736.         {

  737.             // TODO more proper condition

  738.             conv3x3s1_winograd64_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);

  739. 

  740. //             conv3x3s1_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);

  741. 

  742.             if (activation)

  743.             {

  744.                 activation->forward_inplace(top_blob, opt);

  745.             }

  746.         }

  747.         else

  748.         {

  749.             // num_output

  750.             #pragma omp parallel for num_threads(opt.num_threads)

  751.             for (int p=0; p<num_output; p++)

  752.             {

  753.                 float* outptr = top_blob.channel(p);

  754. 

  755.                 for (int i = 0; i < outh; i++)

  756.                 {

  757.                     for (int j = 0; j < outw; j++)

  758.                     {

  759.                         float sum = 0.f;

  760. 

  761.                         if (bias_term)

  762.                         {

  763.                             sum = bias_data[p];

  764.                         }

  765. 

  766.                         const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4;

  767. 

  768.                         // channels

  769.                         for (int q=0; q<channels; q++)

  770.                         {

  771.                             const Mat m = bottom_blob_bordered.channel(q);

  772.                             const float* sptr = m.row(i*stride_h) + j*stride_w * 4;

  773. 

  774.                             for (int k = 0; k < maxk; k++) // 29.23

  775.                             {

  776.                                 float32x4_t _val = vld1q_f32( sptr + space_ofs[k] * 4 );

  777.                                 float32x4_t _w = vld1q_f32( kptr );

  778.                                 float32x4_t _s4 = vmulq_f32(_val, _w);

  779. #if __aarch64__

  780.                                 sum += vaddvq_f32(_s4); // dot

  781. #else

  782.                                 float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));

  783.                                 _ss = vpadd_f32(_ss, _ss);

  784.                                 sum += vget_lane_f32(_ss, 0);

  785. #endif

  786. 

  787.                                 kptr += 4;

  788.                             }

  789.                         }

  790. 

  791.                         sum = activation_ss(sum, activation_type, activation_params);

  792. 

  793.                         outptr[j] = sum;

  794.                     }

  795. 

  796.                     outptr += outw;

  797.                 }

  798.             }

  799.         }

  800.     }

  801. #endif // __ARM_NEON

  802. 

  803.     if (elempack == 1 && out_elempack == 1)

  804.     {

  805.         if (impl_type > 0 && impl_type < 6 && impl_type != 4)

  806.         {

  807.             // engineering is magic.

  808.             switch (impl_type)

  809.             {

  810.                 case 1:

  811.                     conv3x3s1_winograd64_neon5(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data, opt);

  812.                     break;

  813.                 case 2:

  814.                     conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_1x1_sgemm_data, bias_data, opt);

  815.                     break;

  816.                 case 3:

  817.                     conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);

  818.                     break;

  819. //                 case 4: FIXME fallback to auto path

  820. //                     conv(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  821. //                     break;

  822.                 case 5:

  823.                     conv3x3s2_packed_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data, bias_data, opt);

  824.                     break;

  825.             }

  826. 

  827.             if (activation)

  828.             {

  829.                 activation->forward_inplace(top_blob, opt);

  830.             }

  831.         }

  832.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  833.         {

  834.             if (use_sgemm1x1)

  835.             {

  836.                 conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_1x1_sgemm_data, bias_data, opt);

  837.             }

  838.             else

  839.             {

  840.                 conv1x1s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  841.             }

  842. 

  843.             if (activation)

  844.             {

  845.                 activation->forward_inplace(top_blob, opt);

  846.             }

  847.         }

  848.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  849.         {

  850.             if (opt.use_sgemm_convolution)

  851.                 conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);

  852.             else

  853.                 conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  854. 

  855.             if (activation)

  856.             {

  857.                 activation->forward_inplace(top_blob, opt);

  858.             }

  859.         }

  860.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  861.         {

  862.             if (use_winograd3x3 && w <= 120 && h <= 120)

  863.             {

  864. //                 conv3x3s1_winograd64_neon4(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data, opt);

  865.                 conv3x3s1_winograd64_neon5(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data, opt);

  866.             }

  867.             else

  868.             {

  869.                 conv3x3s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  870.             }

  871. 

  872.             if (activation)

  873.             {

  874.                 activation->forward_inplace(top_blob, opt);

  875.             }

  876.         }

  877.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  878.         {

  879.             if (opt.use_sgemm_convolution && !(outw >=8 && outh >=8))

  880.                 conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);

  881.             else

  882.                 conv3x3s2_packed_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data, bias_data, opt);

  883. 

  884.             if (activation)

  885.             {

  886.                 activation->forward_inplace(top_blob, opt);

  887.             }

  888.         }

  889.         else if (kernel_w == 4 && kernel_h == 4 && dilation_w == 1 && dilation_h == 1 && stride_w == 4 && stride_h == 4)

  890.         {

  891.             conv4x4s4_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  892. 

  893.             if (activation)

  894.             {

  895.                 activation->forward_inplace(top_blob, opt);

  896.             }

  897.         }

  898.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  899.         {

  900.             conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  901. 

  902.             if (activation)

  903.             {

  904.                 activation->forward_inplace(top_blob, opt);

  905.             }

  906.         }

  907.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  908.         {

  909.             conv5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  910. 

  911.             if (activation)

  912.             {

  913.                 activation->forward_inplace(top_blob, opt);

  914.             }

  915.         }

  916.         else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  917.         {

  918.             conv7x7s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  919. 

  920.             if (activation)

  921.             {

  922.                 activation->forward_inplace(top_blob, opt);

  923.             }

  924.         }

  925.         else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  926.         {

  927.             conv7x7s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);

  928. 

  929.             if (activation)

  930.             {

  931.                 activation->forward_inplace(top_blob, opt);

  932.             }

  933.         }

  934.         else

  935.         {

  936.             // num_output

  937.             #pragma omp parallel for num_threads(opt.num_threads)

  938.             for (int p=0; p<num_output; p++)

  939.             {

  940.                 float* outptr = top_blob.channel(p);

  941. 

  942.                 for (int i = 0; i < outh; i++)

  943.                 {

  944.                     for (int j = 0; j < outw; j++)

  945.                     {

  946.                         float sum = 0.f;

  947. 

  948.                         if (bias_term)

  949.                         {

  950.                             sum = bias_data[p];

  951.                         }

  952. 

  953.                         const float* kptr = (const float*)weight_data + maxk * channels * p;

  954. 

  955.                         // channels

  956.                         for (int q=0; q<channels; q++)

  957.                         {

  958.                             const Mat m = bottom_blob_bordered.channel(q);

  959.                             const float* sptr = m.row(i*stride_h) + j*stride_w;

  960. 

  961.                             for (int k = 0; k < maxk; k++)

  962.                             {

  963.                                 float val = sptr[ space_ofs[k] ];

  964.                                 float w = kptr[ k ];

  965.                                 sum += val * w;

  966.                             }

  967. 

  968.                             kptr += maxk;

  969.                         }

  970. 

  971.                         if (activation_type == 1)

  972.                         {

  973.                             sum = std::max(sum, 0.f);

  974.                         }

  975.                         else if (activation_type == 2)

  976.                         {

  977.                             float slope = activation_params[0];

  978.                             sum = sum > 0.f ? sum : sum * slope;

  979.                         }

  980.                         else if (activation_type == 3)

  981.                         {

  982.                             float min = activation_params[0];

  983.                             float max = activation_params[1];

  984.                             if (sum < min)

  985.                                 sum = min;

  986.                             if (sum > max)

  987.                                 sum = max;

  988.                         }

  989.                         else if (activation_type == 4)

  990.                         {

  991.                             sum = static_cast<float>(1.f / (1.f + exp(-sum)));

  992.                         }

  993. 

  994.                         outptr[j] = sum;

  995.                     }

  996. 

  997.                     outptr += outw;

  998.                 }

  999.             }

  1000.         }

  1001.     }

  1002. 

  1003.     return 0;

  1004. }

  1005. 

  1006. int Convolution_arm::create_pipeline_bf16s(const Option& opt)

  1007. {

  1008.     const int maxk = kernel_w * kernel_h;

  1009.     const int num_input = weight_data_size / maxk / num_output;

  1010. 

  1011.     int elempack = (opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1;

  1012.     int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;

  1013. 

  1014. #if __ARM_NEON

  1015.     // pack4

  1016.     if (elempack == 4 && out_elempack == 4)

  1017.     {

  1018.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1019.         {

  1020.             conv1x1s1_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_pack4_bf16, num_input, num_output);

  1021.         }

  1022.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1023.         {

  1024.             conv1x1s1_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_pack4_bf16, num_input, num_output);

  1025.         }

  1026.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1027.         {

  1028.             conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4_bf16, num_input, num_output);

  1029.         }

  1030.         else

  1031.         {

  1032.             // src = kw-kh-inch-outch

  1033.             // dst = 4b-4a-kw-kh-inch/4a-outch/4b

  1034.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  1035. 

  1036.             weight_data_pack4_bf16.create(maxk, num_input/4, num_output/4, (size_t)2*16, 16);

  1037. 

  1038.             for (int q=0; q+3<num_output; q+=4)

  1039.             {

  1040.                 const Mat k0 = weight_data_r2.channel(q);

  1041.                 const Mat k1 = weight_data_r2.channel(q+1);

  1042.                 const Mat k2 = weight_data_r2.channel(q+2);

  1043.                 const Mat k3 = weight_data_r2.channel(q+3);

  1044. 

  1045.                 Mat g0 = weight_data_pack4_bf16.channel(q/4);

  1046. 

  1047.                 for (int p=0; p+3<num_input; p+=4)

  1048.                 {

  1049.                     const float* k00 = k0.row(p);

  1050.                     const float* k01 = k0.row(p+1);

  1051.                     const float* k02 = k0.row(p+2);

  1052.                     const float* k03 = k0.row(p+3);

  1053. 

  1054.                     const float* k10 = k1.row(p);

  1055.                     const float* k11 = k1.row(p+1);

  1056.                     const float* k12 = k1.row(p+2);

  1057.                     const float* k13 = k1.row(p+3);

  1058. 

  1059.                     const float* k20 = k2.row(p);

  1060.                     const float* k21 = k2.row(p+1);

  1061.                     const float* k22 = k2.row(p+2);

  1062.                     const float* k23 = k2.row(p+3);

  1063. 

  1064.                     const float* k30 = k3.row(p);

  1065.                     const float* k31 = k3.row(p+1);

  1066.                     const float* k32 = k3.row(p+2);

  1067.                     const float* k33 = k3.row(p+3);

  1068. 

  1069.                     unsigned short* g00 = g0.row<unsigned short>(p/4);

  1070. 

  1071.                     for (int k=0; k<maxk; k++)

  1072.                     {

  1073.                         g00[0] = float32_to_bfloat16(k00[k]);

  1074.                         g00[1] = float32_to_bfloat16(k10[k]);

  1075.                         g00[2] = float32_to_bfloat16(k20[k]);

  1076.                         g00[3] = float32_to_bfloat16(k30[k]);

  1077. 

  1078.                         g00[4] = float32_to_bfloat16(k01[k]);

  1079.                         g00[5] = float32_to_bfloat16(k11[k]);

  1080.                         g00[6] = float32_to_bfloat16(k21[k]);

  1081.                         g00[7] = float32_to_bfloat16(k31[k]);

  1082. 

  1083.                         g00[8] = float32_to_bfloat16(k02[k]);

  1084.                         g00[9] = float32_to_bfloat16(k12[k]);

  1085.                         g00[10] = float32_to_bfloat16(k22[k]);

  1086.                         g00[11] = float32_to_bfloat16(k32[k]);

  1087. 

  1088.                         g00[12] = float32_to_bfloat16(k03[k]);

  1089.                         g00[13] = float32_to_bfloat16(k13[k]);

  1090.                         g00[14] = float32_to_bfloat16(k23[k]);

  1091.                         g00[15] = float32_to_bfloat16(k33[k]);

  1092. 

  1093.                         g00 += 16;

  1094.                     }

  1095.                 }

  1096.             }

  1097.         }

  1098.     }

  1099. 

  1100.     // pack1to4

  1101.     if (elempack == 1 && out_elempack == 4)

  1102.     {

  1103.         // src = kw-kh-inch-outch

  1104.         // dst = 4b-kw-kh-inch-outch/4b

  1105.         {

  1106.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  1107. 

  1108.             weight_data_pack1to4_bf16.create(maxk, num_input, num_output/4, (size_t)2*4, 4);

  1109. 

  1110.             for (int q=0; q+3<num_output; q+=4)

  1111.             {

  1112.                 const Mat k0 = weight_data_r2.channel(q);

  1113.                 const Mat k1 = weight_data_r2.channel(q+1);

  1114.                 const Mat k2 = weight_data_r2.channel(q+2);

  1115.                 const Mat k3 = weight_data_r2.channel(q+3);

  1116. 

  1117.                 Mat g0 = weight_data_pack1to4_bf16.channel(q/4);

  1118. 

  1119.                 for (int p=0; p<num_input; p++)

  1120.                 {

  1121.                     const float* k00 = k0.row(p);

  1122.                     const float* k10 = k1.row(p);

  1123.                     const float* k20 = k2.row(p);

  1124.                     const float* k30 = k3.row(p);

  1125. 

  1126.                     unsigned short* g00 = g0.row<unsigned short>(p);

  1127. 

  1128.                     for (int k=0; k<maxk; k++)

  1129.                     {

  1130.                         g00[0] = float32_to_bfloat16(k00[k]);

  1131.                         g00[1] = float32_to_bfloat16(k10[k]);

  1132.                         g00[2] = float32_to_bfloat16(k20[k]);

  1133.                         g00[3] = float32_to_bfloat16(k30[k]);

  1134. 

  1135.                         g00 += 4;

  1136.                     }

  1137.                 }

  1138.             }

  1139.         }

  1140.     }

  1141. 

  1142.     // pack4to1

  1143.     if (elempack == 4 && out_elempack == 1)

  1144.     {

  1145.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1146.         {

  1147.             conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output);

  1148.         }

  1149.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1150.         {

  1151.             conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output);

  1152.         }

  1153.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1154.         {

  1155.             conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output);

  1156.         }

  1157.         else

  1158.         {

  1159.             // src = kw-kh-inch-outch

  1160.             // dst = 4a-kw-kh-inch/4a-outch

  1161.             Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

  1162. 

  1163.             weight_data_pack4to1_bf16.create(maxk, num_input/4, num_output, (size_t)2*4, 4);

  1164. 

  1165.             for (int q=0; q<num_output; q++)

  1166.             {

  1167.                 const Mat k0 = weight_data_r2.channel(q);

  1168.                 Mat g0 = weight_data_pack4to1_bf16.channel(q);

  1169. 

  1170.                 for (int p=0; p+3<num_input; p+=4)

  1171.                 {

  1172.                     const float* k00 = k0.row(p);

  1173.                     const float* k01 = k0.row(p+1);

  1174.                     const float* k02 = k0.row(p+2);

  1175.                     const float* k03 = k0.row(p+3);

  1176. 

  1177.                     unsigned short* g00 = g0.row<unsigned short>(p/4);

  1178. 

  1179.                     for (int k=0; k<maxk; k++)

  1180.                     {

  1181.                         g00[0] = float32_to_bfloat16(k00[k]);

  1182.                         g00[1] = float32_to_bfloat16(k01[k]);

  1183.                         g00[2] = float32_to_bfloat16(k02[k]);

  1184.                         g00[3] = float32_to_bfloat16(k03[k]);

  1185. 

  1186.                         g00 += 4;

  1187.                     }

  1188.                 }

  1189.             }

  1190.         }

  1191.     }

  1192. #endif // __ARM_NEON

  1193. 

  1194.     // pack1

  1195.     if (elempack == 1 && out_elempack == 1)

  1196.     {

  1197.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1198.         {

  1199.             conv1x1s1_sgemm_transform_kernel_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output);

  1200.         }

  1201.         else

  1202.         {

  1203.             ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt);

  1204.         }

  1205.     }

  1206. 

  1207.     return 0;

  1208. }

  1209. 

  1210. int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  1211. {

  1212.     int w = bottom_blob.w;

  1213.     int h = bottom_blob.h;

  1214.     int channels = bottom_blob.c;

  1215.     size_t elemsize = bottom_blob.elemsize;

  1216.     int elempack = bottom_blob.elempack;

  1217. 

  1218. //     fprintf(stderr, "Convolution input %d x %d  pad = %d %d  ksize=%d %d  stride=%d %d\n", w, h, pad_w, pad_h, kernel_w, kernel_h, stride_w, stride_h);

  1219. 

  1220.     const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;

  1221.     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;

  1222. 

  1223.     Mat bottom_blob_bordered;

  1224.     make_padding(bottom_blob, bottom_blob_bordered, opt);

  1225.     if (bottom_blob_bordered.empty())

  1226.         return -100;

  1227. 

  1228.     w = bottom_blob_bordered.w;

  1229.     h = bottom_blob_bordered.h;

  1230. 

  1231.     int outw = (w - kernel_extent_w) / stride_w + 1;

  1232.     int outh = (h - kernel_extent_h) / stride_h + 1;

  1233.     int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;

  1234.     size_t out_elemsize = elemsize / elempack * out_elempack;

  1235. 

  1236.     top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);

  1237.     if (top_blob.empty())

  1238.         return -100;

  1239. 

  1240.     // FIXME

  1241. //     if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)

  1242. //     {

  1243. //         return forwardDilation_arm(bottom_blob_bordered, top_blob, opt);

  1244. //     }

  1245. 

  1246.     const int maxk = kernel_w * kernel_h;

  1247. 

  1248.     // kernel offsets

  1249.     std::vector<int> _space_ofs(maxk);

  1250.     int* space_ofs = &_space_ofs[0];

  1251.     {

  1252.         int p1 = 0;

  1253.         int p2 = 0;

  1254.         int gap = w * dilation_h - kernel_w * dilation_w;

  1255.         for (int i = 0; i < kernel_h; i++)

  1256.         {

  1257.             for (int j = 0; j < kernel_w; j++)

  1258.             {

  1259.                 space_ofs[p1] = p2;

  1260.                 p1++;

  1261.                 p2 += dilation_w;

  1262.             }

  1263.             p2 += gap;

  1264.         }

  1265.     }

  1266. 

  1267. #if __ARM_NEON

  1268.     if (elempack == 4 && out_elempack == 4)

  1269.     {

  1270.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1271.         {

  1272.             conv1x1s1_sgemm_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1273. 

  1274.             if (activation)

  1275.             {

  1276.                 activation->forward_inplace(top_blob, opt);

  1277.             }

  1278.         }

  1279.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1280.         {

  1281.             conv1x1s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1282. 

  1283.             if (activation)

  1284.             {

  1285.                 activation->forward_inplace(top_blob, opt);

  1286.             }

  1287.         }

  1288.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1289.         {

  1290.             conv3x3s1_winograd64_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1291. 

  1292.             if (activation)

  1293.             {

  1294.                 activation->forward_inplace(top_blob, opt);

  1295.             }

  1296.         }

  1297.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1298.         {

  1299.             conv3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1300. 

  1301.             if (activation)

  1302.             {

  1303.                 activation->forward_inplace(top_blob, opt);

  1304.             }

  1305.         }

  1306.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1307.         {

  1308.             conv5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1309. 

  1310.             if (activation)

  1311.             {

  1312.                 activation->forward_inplace(top_blob, opt);

  1313.             }

  1314.         }

  1315.         else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1316.         {

  1317.             conv5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);

  1318. 

  1319.             if (activation)

  1320.             {

  1321.                 activation->forward_inplace(top_blob, opt);

  1322.             }

  1323.         }

  1324.         else

  1325.         {

  1326.             // num_output

  1327.             #pragma omp parallel for num_threads(opt.num_threads)

  1328.             for (int p=0; p<num_output / out_elempack; p++)

  1329.             {

  1330.                 unsigned short* outptr = top_blob.channel(p);

  1331. 

  1332.                 for (int i = 0; i < outh; i++)

  1333.                 {

  1334.                     for (int j = 0; j < outw; j++)

  1335.                     {

  1336.                         float32x4_t _sum = vdupq_n_f32(0.f);

  1337. 

  1338.                         if (bias_term)

  1339.                         {

  1340.                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);

  1341.                         }

  1342. 

  1343.                         const unsigned short* kptr = weight_data_pack4_bf16.channel(p);

  1344. 

  1345.                         // channels

  1346.                         for (int q=0; q<channels; q++)

  1347.                         {

  1348.                             const Mat m = bottom_blob_bordered.channel(q);

  1349.                             const unsigned short* sptr = m.row<const unsigned short>(i*stride_h) + j*stride_w * 4;

  1350. 

  1351.                             for (int k = 0; k < maxk; k++)

  1352.                             {

  1353.                                 float32x4_t _val = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( sptr + space_ofs[k] * 4 ), 16));

  1354. 

  1355.                                 float32x4_t _w0 = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr ), 16));

  1356.                                 float32x4_t _w1 = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr + 4 ), 16));

  1357.                                 float32x4_t _w2 = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr + 8 ), 16));

  1358.                                 float32x4_t _w3 = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr + 12 ), 16));

  1359. 

  1360. #if __aarch64__

  1361.                                 _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);

  1362.                                 _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);

  1363.                                 _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);

  1364.                                 _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);

  1365. #else

  1366.                                 _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);

  1367.                                 _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);

  1368.                                 _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);

  1369.                                 _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);

  1370. #endif

  1371. 

  1372.                                 kptr += 16;

  1373.                             }

  1374.                         }

  1375. 

  1376.                         _sum = activation_ps(_sum, activation_type, activation_params);

  1377. 

  1378.                         vst1_u16(outptr + j * 4, vshrn_n_u32(vreinterpretq_u32_f32(_sum), 16));

  1379.                     }

  1380. 

  1381.                     outptr += outw * 4;

  1382.                 }

  1383.             }

  1384.         }

  1385.     }

  1386. 

  1387.     if (elempack == 1 && out_elempack == 4)

  1388.     {

  1389.         if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1390.         {

  1391.             conv3x3s1_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt);

  1392. 

  1393.             if (activation)

  1394.             {

  1395.                 activation->forward_inplace(top_blob, opt);

  1396.             }

  1397.         }

  1398.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1399.         {

  1400.             conv3x3s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt);

  1401. 

  1402.             if (activation)

  1403.             {

  1404.                 activation->forward_inplace(top_blob, opt);

  1405.             }

  1406.         }

  1407.         else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1408.         {

  1409.             conv7x7s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt);

  1410. 

  1411.             if (activation)

  1412.             {

  1413.                 activation->forward_inplace(top_blob, opt);

  1414.             }

  1415.         }

  1416.         else

  1417.         {

  1418.             // num_output

  1419.             #pragma omp parallel for num_threads(opt.num_threads)

  1420.             for (int p=0; p<num_output / out_elempack; p++)

  1421.             {

  1422.                 unsigned short* outptr = top_blob.channel(p);

  1423. 

  1424.                 for (int i = 0; i < outh; i++)

  1425.                 {

  1426.                     for (int j = 0; j < outw; j++)

  1427.                     {

  1428.                         float32x4_t _sum = vdupq_n_f32(0.f);

  1429. 

  1430.                         if (bias_term)

  1431.                         {

  1432.                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);

  1433.                         }

  1434. 

  1435.                         const unsigned short* kptr = weight_data_pack1to4_bf16.channel(p);

  1436. 

  1437.                         // channels

  1438.                         for (int q=0; q<channels; q++)

  1439.                         {

  1440.                             const Mat m = bottom_blob_bordered.channel(q);

  1441.                             const unsigned short* sptr = m.row<const unsigned short>(i*stride_h) + j*stride_w;

  1442. 

  1443.                             for (int k = 0; k < maxk; k++)

  1444.                             {

  1445.                                 float32x4_t _val = vdupq_n_f32(bfloat16_to_float32( sptr[ space_ofs[k] ] ));

  1446.                                 float32x4_t _w = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr ), 16));

  1447.                                 _sum = vmlaq_f32(_sum, _val, _w);

  1448. 

  1449.                                 kptr += 4;

  1450.                             }

  1451.                         }

  1452. 

  1453.                         _sum = activation_ps(_sum, activation_type, activation_params);

  1454. 

  1455.                         vst1_u16(outptr + j * 4, vshrn_n_u32(vreinterpretq_u32_f32(_sum), 16));

  1456.                     }

  1457. 

  1458.                     outptr += outw * 4;

  1459.                 }

  1460.             }

  1461.         }

  1462.     }

  1463. 

  1464.     if (elempack == 4 && out_elempack == 1)

  1465.     {

  1466.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1467.         {

  1468.             conv1x1s1_sgemm_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt);

  1469. 

  1470.             if (activation)

  1471.             {

  1472.                 activation->forward_inplace(top_blob, opt);

  1473.             }

  1474.         }

  1475.         else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1476.         {

  1477.             conv1x1s2_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt);

  1478. 

  1479.             if (activation)

  1480.             {

  1481.                 activation->forward_inplace(top_blob, opt);

  1482.             }

  1483.         }

  1484.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1485.         {

  1486.             // TODO more proper condition

  1487.             conv3x3s1_winograd64_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt);

  1488. 

  1489. //             conv3x3s1_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt);

  1490. 

  1491.             if (activation)

  1492.             {

  1493.                 activation->forward_inplace(top_blob, opt);

  1494.             }

  1495.         }

  1496.         else

  1497.         {

  1498.             // num_output

  1499.             #pragma omp parallel for num_threads(opt.num_threads)

  1500.             for (int p=0; p<num_output; p++)

  1501.             {

  1502.                 unsigned short* outptr = top_blob.channel(p);

  1503. 

  1504.                 for (int i = 0; i < outh; i++)

  1505.                 {

  1506.                     for (int j = 0; j < outw; j++)

  1507.                     {

  1508.                         float sum = 0.f;

  1509. 

  1510.                         if (bias_term)

  1511.                         {

  1512.                             sum = bias_data[p];

  1513.                         }

  1514. 

  1515.                         const unsigned short* kptr = weight_data_pack4to1_bf16.channel(p);

  1516. 

  1517.                         // channels

  1518.                         for (int q=0; q<channels; q++)

  1519.                         {

  1520.                             const Mat m = bottom_blob_bordered.channel(q);

  1521.                             const unsigned short* sptr = m.row<const unsigned short>(i*stride_h) + j*stride_w * 4;

  1522. 

  1523.                             for (int k = 0; k < maxk; k++)

  1524.                             {

  1525.                                 float32x4_t _val = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( sptr + space_ofs[k] * 4 ), 16));

  1526.                                 float32x4_t _w = vreinterpretq_f32_u32(vshll_n_u16(vld1_u16( kptr ), 16));

  1527.                                 float32x4_t _s4 = vmulq_f32(_val, _w);

  1528. #if __aarch64__

  1529.                                 sum += vaddvq_f32(_s4); // dot

  1530. #else

  1531.                                 float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));

  1532.                                 _ss = vpadd_f32(_ss, _ss);

  1533.                                 sum += vget_lane_f32(_ss, 0);

  1534. #endif

  1535. 

  1536.                                 kptr += 4;

  1537.                             }

  1538.                         }

  1539. 

  1540.                         sum = activation_ss(sum, activation_type, activation_params);

  1541. 

  1542.                         outptr[j] = float32_to_bfloat16(sum);

  1543.                     }

  1544. 

  1545.                     outptr += outw;

  1546.                 }

  1547.             }

  1548.         }

  1549.     }

  1550. #endif // __ARM_NEON

  1551. 

  1552.     if (elempack == 1 && out_elempack == 1)

  1553.     {

  1554.         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1555.         {

  1556.             conv1x1s1_sgemm_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt);

  1557. 

  1558.             if (activation)

  1559.             {

  1560.                 activation->forward_inplace(top_blob, opt);

  1561.             }

  1562.         }

  1563.         else

  1564.         {

  1565.             // num_output

  1566.             #pragma omp parallel for num_threads(opt.num_threads)

  1567.             for (int p=0; p<num_output; p++)

  1568.             {

  1569.                 unsigned short* outptr = top_blob.channel(p);

  1570. 

  1571.                 for (int i = 0; i < outh; i++)

  1572.                 {

  1573.                     for (int j = 0; j < outw; j++)

  1574.                     {

  1575.                         float sum = 0.f;

  1576. 

  1577.                         if (bias_term)

  1578.                         {

  1579.                             sum = bias_data[p];

  1580.                         }

  1581. 

  1582.                         const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * channels * p;

  1583. 

  1584.                         // channels

  1585.                         for (int q=0; q<channels; q++)

  1586.                         {

  1587.                             const Mat m = bottom_blob_bordered.channel(q);

  1588.                             const unsigned short* sptr = m.row<unsigned short>(i*stride_h) + j*stride_w;

  1589. 

  1590.                             for (int k = 0; k < maxk; k++)

  1591.                             {

  1592.                                 float val = bfloat16_to_float32(sptr[ space_ofs[k] ]);

  1593.                                 float w = bfloat16_to_float32(kptr[ k ]);

  1594.                                 sum += val * w;

  1595.                             }

  1596. 

  1597.                             kptr += maxk;

  1598.                         }

  1599. 

  1600.                         if (activation_type == 1)

  1601.                         {

  1602.                             sum = std::max(sum, 0.f);

  1603.                         }

  1604.                         else if (activation_type == 2)

  1605.                         {

  1606.                             float slope = activation_params[0];

  1607.                             sum = sum > 0.f ? sum : sum * slope;

  1608.                         }

  1609.                         else if (activation_type == 3)

  1610.                         {

  1611.                             float min = activation_params[0];

  1612.                             float max = activation_params[1];

  1613.                             if (sum < min)

  1614.                                 sum = min;

  1615.                             if (sum > max)

  1616.                                 sum = max;

  1617.                         }

  1618.                         else if (activation_type == 4)

  1619.                         {

  1620.                             sum = static_cast<float>(1.f / (1.f + exp(-sum)));

  1621.                         }

  1622. 

  1623.                         outptr[j] = float32_to_bfloat16(sum);

  1624.                     }

  1625. 

  1626.                     outptr += outw;

  1627.                 }

  1628.             }

  1629.         }

  1630.     }

  1631. 

  1632.     return 0;

  1633. }

  1634. 

  1635. int Convolution_arm::create_pipeline_int8_arm(const Option& opt)

  1636. {

  1637.     const int maxk = kernel_w * kernel_h;

  1638.     const int num_input = weight_data_size / maxk / num_output;

  1639. 

  1640.     use_winograd3x3_int8 = false;

  1641.     use_sgemm1x1_int8 = false;

  1642. 

  1643.     if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1644.     {

  1645.         use_winograd3x3_int8 = true;

  1646. //         conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);

  1647.         conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);

  1648.     }

  1649. 

  1650.     if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1651.     {

  1652.         conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_data_int8, num_input, num_output);

  1653.     }

  1654.     else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)

  1655.     {

  1656.         use_sgemm1x1_int8 = true;

  1657.         conv1x1s1_sgemm_transform_kernel_int8_neon(weight_data, weight_1x1s1_sgemm_data_int8, num_input, num_output);

  1658.     }

  1659.     else

  1660.     {

  1661.         conv_im2col_sgemm_transform_kernel_int8_neon(weight_data, weight_sgemm_data_int8, num_input, num_output, maxk);

  1662.     }

  1663. 

  1664.     return 0;

  1665. }

  1666. 

  1667. int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  1668. {

  1669.     if (dilation_w > 1 || dilation_h > 1)

  1670.     {

  1671.         return Convolution::forward(bottom_blob, top_blob, opt);

  1672.     }

  1673. 

  1674.     int w = bottom_blob.w;

  1675.     int h = bottom_blob.h;

  1676.     // int channels = bottom_blob.c;

  1677.     size_t elemsize = bottom_blob.elemsize;

  1678. 

  1679. //     fprintf(stderr, "Convolution_arm input %d x %d  ksize=%d %d  stride=%d %d\n", w, h, kernel_w, kernel_h, stride_w, stride_h);

  1680. 

  1681.     const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;

  1682.     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;

  1683. 

  1684.     Mat bottom_blob_unbordered = bottom_blob;

  1685.     if (elemsize != 1)

  1686.     {

  1687.         Option opt_g = opt;

  1688.         opt_g.blob_allocator = opt.workspace_allocator;

  1689. 

  1690.         quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g);

  1691.     }

  1692. 

  1693.     Mat bottom_blob_bordered;

  1694.     make_padding(bottom_blob_unbordered, bottom_blob_bordered, opt);

  1695.     if (bottom_blob_bordered.empty())

  1696.         return -100;

  1697. 

  1698.     w = bottom_blob_bordered.w;

  1699.     h = bottom_blob_bordered.h;

  1700. 

  1701.     int outw = (w - kernel_extent_w) / stride_w + 1;

  1702.     int outh = (h - kernel_extent_h) / stride_h + 1;

  1703. 

  1704.     // int8

  1705.     size_t out_elemsize = use_int8_requantize ? 1u : 4u;

  1706. 

  1707.     top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);

  1708.     if (top_blob.empty())

  1709.         return -100;

  1710. 

  1711.     // int8

  1712.     if (use_int8_requantize == true)

  1713.     {

  1714.         Mat top_blob_tm;

  1715.         top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);

  1716.         if (top_blob_tm.empty())

  1717.             return -100;

  1718.         

  1719.         if (use_sgemm1x1_int8)

  1720.         {

  1721.             std::vector<float> requantize_scales;

  1722.             for (int p=0; p<num_output; p++)

  1723.             {

  1724.                 float scale_in;

  1725.                 if (weight_data_int8_scales[p] == 0)

  1726.                     scale_in = 0;

  1727.                 else

  1728.                     scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);

  1729. 

  1730.                 float scale_out = top_blob_int8_scale;

  1731. 

  1732.                 requantize_scales.push_back(scale_in);

  1733.                 requantize_scales.push_back(scale_out);

  1734.             }

  1735. 

  1736.             conv1x1s1_sgemm_int8_requant_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_data_int8, bias_data, requantize_scales, opt);

  1737. 

  1738.             if (activation)

  1739.             {

  1740.                 activation->forward_inplace(top_blob, opt);

  1741.             }

  1742. 

  1743.             return 0;

  1744.         }

  1745.         else if (use_winograd3x3_int8)

  1746.         {

  1747. //             conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);

  1748.             conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);

  1749.         }

  1750.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1751.         {

  1752.             conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3s2_data_int8, opt);

  1753.         }

  1754.         else

  1755.         {

  1756.             conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob_tm, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt);

  1757.         }

  1758. 

  1759.         // requantize, reverse scale inplace

  1760.         #pragma omp parallel for num_threads(opt.num_threads)

  1761.         for (int p=0; p<num_output; p++)

  1762.         {

  1763.             Option opt_g = opt;

  1764.             opt_g.num_threads = 1;

  1765.             opt_g.blob_allocator = top_blob.allocator;

  1766. 

  1767.             Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1);

  1768.             Mat top_blob_g = top_blob.channel_range(p, 1);

  1769. 

  1770.             // requantize and relu

  1771.             float scale_in;

  1772.             if (weight_data_int8_scales[p] == 0)

  1773.                 scale_in = 0;

  1774.             else

  1775.                 scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);

  1776. 

  1777.             float scale_out = top_blob_int8_scale;//FIXME load param

  1778. 

  1779.             requantize_int8_to_int8(top_blob_tm_g, top_blob_g, scale_in, scale_out, bias_term ? (const float*)bias_data + p : 0, bias_term ? 1 : 0, 0, opt_g);

  1780.         }

  1781.     }

  1782.     else

  1783.     {

  1784.         if (use_sgemm1x1_int8)

  1785.         {

  1786.             conv1x1s1_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_data_int8, opt);

  1787.         }

  1788.         else if (use_winograd3x3_int8)

  1789.         {

  1790. //             conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);

  1791.             conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);

  1792.         }

  1793.         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)

  1794.         {

  1795.             conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data_int8, opt);

  1796.         }

  1797.         else

  1798.         {

  1799.             conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt);

  1800.         }

  1801. 

  1802.         // dequantize, reverse scale inplace

  1803.         #pragma omp parallel for num_threads(opt.num_threads)

  1804.         for (int p=0; p<num_output; p++)

  1805.         {

  1806.             Option opt_g = opt;

  1807.             opt_g.num_threads = 1;

  1808.             opt_g.blob_allocator = top_blob.allocator;

  1809. 

  1810.             Mat top_blob_g = top_blob.channel_range(p, 1);

  1811. 

  1812.             // dequantize

  1813.             float scale_in;

  1814.             if (weight_data_int8_scales[p] == 0)

  1815.                 scale_in = 0;

  1816.             else

  1817.                 scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);

  1818. 

  1819.             dequantize_int32_to_float32(top_blob_g, scale_in, bias_term ? (const float*)bias_data + p : 0, bias_term ? 1 : 0, opt_g);

  1820.         }

  1821.     }

  1822. 

  1823.     if (activation)

  1824.     {

  1825.         activation->forward_inplace(top_blob, opt);

  1826.     }           

  1827. 

  1828.     return 0;

  1829. }

  1830. 

  1831. int Convolution_arm::forwardDilation_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  1832. {

  1833.     int w = bottom_blob.w;

  1834.     int h = bottom_blob.h;

  1835.     size_t elemsize = bottom_blob.elemsize;

  1836. 

  1837.     const int kernel_size = kernel_w;

  1838.     const int stride = stride_w;

  1839.     const int dilation = dilation_w;

  1840.     const int kernel_extent = dilation * (kernel_size - 1) + 1;

  1841. 

  1842.     int outw = (w - kernel_extent) / stride + 1;

  1843.     int outh = (h - kernel_extent) / stride + 1;

  1844. 

  1845.     top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);

  1846.     if (top_blob.empty())

  1847.         return -100;

  1848. 

  1849.     // Make (dilation * dilation) batches

  1850.     Mat inner_bottom_blob;

  1851.     Mat inner_top_blob;

  1852.     for (int x = 0; x < dilation; x ++)

  1853.     {

  1854.         for (int y = 0; y < dilation; y ++)

  1855.         {

  1856.             int inner_w = (w - y + dilation - 1) / dilation;

  1857.             int inner_h = (h - x + dilation - 1) / dilation;

  1858. 

  1859.             int inner_outw = (inner_w - kernel_size) / stride + 1;

  1860.             int inner_outh = (inner_h - kernel_size) / stride + 1;

  1861. 

  1862.             inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator);

  1863.             if (inner_bottom_blob.empty())

  1864.                 return -100;

  1865. 

  1866.             inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator);

  1867.             if (inner_top_blob.empty())

  1868.                 return -100;

  1869. 

  1870.             #pragma omp parallel for num_threads(opt.num_threads)

  1871.             for (int c = 0; c < bottom_blob.c; c ++)

  1872.             {

  1873.                 float *outptr = inner_bottom_blob.channel(c);

  1874. 

  1875.                 for (int i = 0; i < inner_h; i ++)

  1876.                 {

  1877.                     const float *ptr = (const float *) bottom_blob.channel(c) + dilation * i * w + x * w + y;

  1878.                     for (int j = 0; j < inner_w; j ++)

  1879.                     {

  1880.                         outptr[j] = ptr[j*dilation];

  1881.                     }

  1882.                     outptr += inner_w;

  1883.                 }

  1884.             }

  1885. 

  1886.             Option opt_g = opt;

  1887.             opt_g.blob_allocator = inner_top_blob.allocator;

  1888.             convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g);

  1889. 

  1890.             #pragma omp parallel for num_threads(opt.num_threads)

  1891.             for (int c = 0; c < num_output; c ++)

  1892.             {

  1893.                 float *outptr = (float *) top_blob.channel(c) + x * outw + y;

  1894.                 for (int i = 0; i < inner_outh; i ++)

  1895.                 {

  1896.                     const float *ptr = (const float *) inner_top_blob.channel(c) + i * inner_outw;

  1897.                     for (int j = 0; j < inner_outw; j ++)

  1898.                     {

  1899.                         outptr[j*dilation] = ptr[j];

  1900.                     }

  1901.                     outptr += dilation * outw;

  1902.                 }

  1903.             }

  1904.         }

  1905.     }

  1906. 

  1907.     if (activation)

  1908.     {

  1909.         activation->forward_inplace(top_blob, opt);

  1910.     }

  1911. 

  1912.     return 0;

  1913. }

  1914. 

  1915. } // namespace ncnn