wangwei
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 3ef308e71c
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3ef308e71c'
${ noResults }
MegEngine/dnn/src/arm_common/conv_bias/fp32/direct.cpp

912 lines
37 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/arm_common/conv_bias/fp32/direct.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #include <cstring>

  13. #include "include/megdnn/oprs.h"

  14. #include "midout.h"

  15. #include "src/arm_common/conv_bias/fp32/direct.h"

  16. #include "src/arm_common/simd_macro/marm_neon.h"

  17. #include "src/common/utils.h"

  18. #include "src/arm_common/conv_bias/postprocess_helper.h"

  19. #include "src/common/unroll_macro.h"

  20. MIDOUT_DECL(megdnn_arm_conv_f32)

  21. 

  22. using namespace megdnn;

  23. using namespace arm_common;

  24. using namespace fp32;

  25. using namespace conv_bias;

  26. 

  27. namespace {

  28. 

  29. template <int FH, int height, int width>

  30. struct do_pixel_proxy {

  31.     static void exec(const float* src, const float* filter, float* dst,

  32.                      const int IH, const int IW, const int OH, const int OW,

  33.                      const int FW, const int oh, const int ow);

  34. };

  35. 

  36. #define cb_load(i) data = vld1q_lane_f32(dst + i, data, i);

  37. #define LOAD_OUT                                               \

  38.     if (width < 4) {                                           \

  39.         auto load_less_4 = [](float* dst, float32x4_t& data) { \

  40.             if (width == 1u) {                                 \

  41.                 UNROLL_CALL_NOWRAPPER(1, cb_load);             \

  42.             } else if (width == 2u) {                          \

  43.                 UNROLL_CALL_NOWRAPPER(2, cb_load);             \

  44.             } else if (width == 3u) {                          \

  45.                 UNROLL_CALL_NOWRAPPER(3, cb_load);             \

  46.             }                                                  \

  47.         };                                                     \

  48.         if (height >= 1)                                       \

  49.             load_less_4(dst + 0 * OW, out0);                   \

  50.         if (height >= 2)                                       \

  51.             load_less_4(dst + 1 * OW, out1);                   \

  52.         if (height >= 3)                                       \

  53.             load_less_4(dst + 2 * OW, out2);                   \

  54.         if (height >= 4)                                       \

  55.             load_less_4(dst + 3 * OW, out3);                   \

  56.     } else {                                                   \

  57.         if (height > 0)                                        \

  58.             out0 = vld1q_f32(dst + 0 * OW);                    \

  59.         if (height > 1)                                        \

  60.             out1 = vld1q_f32(dst + 1 * OW);                    \

  61.         if (height > 2)                                        \

  62.             out2 = vld1q_f32(dst + 2 * OW);                    \

  63.         if (height > 3)                                        \

  64.             out3 = vld1q_f32(dst + 3 * OW);                    \

  65.     }

  66. #define cb_store(i) vst1q_lane_f32(dst + i, data, i);

  67. #define STORE_OUT                                               \

  68.     if (width < 4) {                                            \

  69.         auto store_less_4 = [](float* dst, float32x4_t& data) { \

  70.             if (width == 1u) {                                  \

  71.                 UNROLL_CALL_NOWRAPPER(1, cb_store);             \

  72.             } else if (width == 2u) {                           \

  73.                 UNROLL_CALL_NOWRAPPER(2, cb_store);             \

  74.             } else if (width == 3u) {                           \

  75.                 UNROLL_CALL_NOWRAPPER(3, cb_store);             \

  76.             }                                                   \

  77.         };                                                      \

  78.         if (height >= 1)                                        \

  79.             store_less_4(dst + 0 * OW, out0);                   \

  80.         if (height >= 2)                                        \

  81.             store_less_4(dst + 1 * OW, out1);                   \

  82.         if (height >= 3)                                        \

  83.             store_less_4(dst + 2 * OW, out2);                   \

  84.         if (height >= 4)                                        \

  85.             store_less_4(dst + 3 * OW, out3);                   \

  86.     } else {                                                    \

  87.         if (height >= 1)                                        \

  88.             vst1q_f32(dst + 0 * OW, out0);                      \

  89.         if (height >= 2)                                        \

  90.             vst1q_f32(dst + 1 * OW, out1);                      \

  91.         if (height >= 3)                                        \

  92.             vst1q_f32(dst + 2 * OW, out2);                      \

  93.         if (height >= 4)                                        \

  94.             vst1q_f32(dst + 3 * OW, out3);                      \

  95.     }

  96. 

  97. template <int height, int width>

  98. struct do_pixel_proxy<1, height, width> {

  99.     static void exec(const float* src, const float* filter, float* dst,

  100.                      const int IH, const int IW, const int OH, const int OW,

  101.                      const int FW, const int oh, const int ow) {

  102.         (void)IH;

  103.         (void)OH;

  104.         const int ih = oh, iw = ow;

  105.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, inp;

  106.         src += ih * IW + iw;

  107.         dst += oh * OW + ow;

  108.         LOAD_OUT;

  109.         for (int fw = 0; fw < FW; ++fw) {

  110.             const float* src_dd = src + fw;

  111.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  112. 

  113.             if (height > 0)

  114.                 inp = vld1q_f32(src_dd + 0 * IW);

  115.             if (height > 0)

  116.                 out0 = vmlaq_f32(out0, inp, kr0);

  117. 

  118.             if (height > 1)

  119.                 inp = vld1q_f32(src_dd + 1 * IW);

  120.             if (height > 1)

  121.                 out1 = vmlaq_f32(out1, inp, kr0);

  122. 

  123.             if (height > 2)

  124.                 inp = vld1q_f32(src_dd + 2 * IW);

  125.             if (height > 2)

  126.                 out2 = vmlaq_f32(out2, inp, kr0);

  127. 

  128.             if (height > 3)

  129.                 inp = vld1q_f32(src_dd + 3 * IW);

  130.             if (height > 3)

  131.                 out3 = vmlaq_f32(out3, inp, kr0);

  132.         }

  133.         STORE_OUT;

  134.     }

  135. };

  136. 

  137. template <int height, int width>

  138. struct do_pixel_proxy<2, height, width> {

  139.     static void exec(const float* src, const float* filter, float* dst,

  140.                      const int IH, const int IW, const int OH, const int OW,

  141.                      const int FW, const int oh, const int ow) {

  142.         (void)IH;

  143.         (void)OH;

  144.         const int ih = oh, iw = ow;

  145.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, inp;

  146.         src += ih * IW + iw;

  147.         dst += oh * OW + ow;

  148.         LOAD_OUT;

  149.         for (int fw = 0; fw < FW; ++fw) {

  150.             const float* src_dd = src + fw;

  151.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  152.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  153. 

  154.             if (height > 0)

  155.                 inp = vld1q_f32(src_dd + 0 * IW);

  156.             if (height > 0)

  157.                 out0 = vmlaq_f32(out0, inp, kr0);

  158. 

  159.             if (height > 0)

  160.                 inp = vld1q_f32(src_dd + 1 * IW);

  161.             if (height > 0)

  162.                 out0 = vmlaq_f32(out0, inp, kr1);

  163.             if (height > 1)

  164.                 out1 = vmlaq_f32(out1, inp, kr0);

  165. 

  166.             if (height > 1)

  167.                 inp = vld1q_f32(src_dd + 2 * IW);

  168.             if (height > 1)

  169.                 out1 = vmlaq_f32(out1, inp, kr1);

  170.             if (height > 2)

  171.                 out2 = vmlaq_f32(out2, inp, kr0);

  172. 

  173.             if (height > 2)

  174.                 inp = vld1q_f32(src_dd + 3 * IW);

  175.             if (height > 2)

  176.                 out2 = vmlaq_f32(out2, inp, kr1);

  177.             if (height > 3)

  178.                 out3 = vmlaq_f32(out3, inp, kr0);

  179. 

  180.             if (height > 3)

  181.                 inp = vld1q_f32(src_dd + 4 * IW);

  182.             if (height > 3)

  183.                 out3 = vmlaq_f32(out3, inp, kr1);

  184.         }

  185.         STORE_OUT;

  186.     }

  187. };

  188. 

  189. template <int height, int width>

  190. struct do_pixel_proxy<3, height, width> {

  191.     static void exec(const float* src, const float* filter, float* dst,

  192.                      const int IH, const int IW, const int OH, const int OW,

  193.                      const int FW, const int oh, const int ow) {

  194.         (void)IH;

  195.         (void)OH;

  196.         const int ih = oh, iw = ow;

  197.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, kr2, inp;

  198.         src += ih * IW + iw;

  199.         dst += oh * OW + ow;

  200.         LOAD_OUT;

  201.         for (int fw = 0; fw < FW; ++fw) {

  202.             const float* src_dd = src + fw;

  203.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  204.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  205.             kr2 = vdupq_n_f32(filter[2 * FW + fw]);

  206. 

  207.             if (height > 0)

  208.                 inp = vld1q_f32(src_dd + 0 * IW);

  209.             if (height > 0)

  210.                 out0 = vmlaq_f32(out0, inp, kr0);

  211. 

  212.             if (height > 0)

  213.                 inp = vld1q_f32(src_dd + 1 * IW);

  214.             if (height > 0)

  215.                 out0 = vmlaq_f32(out0, inp, kr1);

  216.             if (height > 1)

  217.                 out1 = vmlaq_f32(out1, inp, kr0);

  218. 

  219.             if (height > 0)

  220.                 inp = vld1q_f32(src_dd + 2 * IW);

  221.             if (height > 0)

  222.                 out0 = vmlaq_f32(out0, inp, kr2);

  223.             if (height > 1)

  224.                 out1 = vmlaq_f32(out1, inp, kr1);

  225.             if (height > 2)

  226.                 out2 = vmlaq_f32(out2, inp, kr0);

  227. 

  228.             if (height > 1)

  229.                 inp = vld1q_f32(src_dd + 3 * IW);

  230.             if (height > 1)

  231.                 out1 = vmlaq_f32(out1, inp, kr2);

  232.             if (height > 2)

  233.                 out2 = vmlaq_f32(out2, inp, kr1);

  234.             if (height > 3)

  235.                 out3 = vmlaq_f32(out3, inp, kr0);

  236. 

  237.             if (height > 2)

  238.                 inp = vld1q_f32(src_dd + 4 * IW);

  239.             if (height > 2)

  240.                 out2 = vmlaq_f32(out2, inp, kr2);

  241.             if (height > 3)

  242.                 out3 = vmlaq_f32(out3, inp, kr1);

  243. 

  244.             if (height > 3)

  245.                 inp = vld1q_f32(src_dd + 5 * IW);

  246.             if (height > 3)

  247.                 out3 = vmlaq_f32(out3, inp, kr2);

  248.         }

  249.         STORE_OUT;

  250.     }

  251. };

  252. 

  253. template <int height, int width>

  254. struct do_pixel_proxy<4, height, width> {

  255.     static void exec(const float* src, const float* filter, float* dst,

  256.                      const int IH, const int IW, const int OH, const int OW,

  257.                      const int FW, const int oh, const int ow) {

  258.         (void)IH;

  259.         (void)OH;

  260.         const int ih = oh, iw = ow;

  261.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, kr2, kr3, inp;

  262.         src += ih * IW + iw;

  263.         dst += oh * OW + ow;

  264.         LOAD_OUT;

  265.         for (int fw = 0; fw < FW; ++fw) {

  266.             const float* src_dd = src + fw;

  267.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  268.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  269.             kr2 = vdupq_n_f32(filter[2 * FW + fw]);

  270.             kr3 = vdupq_n_f32(filter[3 * FW + fw]);

  271. 

  272.             if (height > 0)

  273.                 inp = vld1q_f32(src_dd + 0 * IW);

  274.             if (height > 0)

  275.                 out0 = vmlaq_f32(out0, inp, kr0);

  276. 

  277.             if (height > 0)

  278.                 inp = vld1q_f32(src_dd + 1 * IW);

  279.             if (height > 0)

  280.                 out0 = vmlaq_f32(out0, inp, kr1);

  281.             if (height > 1)

  282.                 out1 = vmlaq_f32(out1, inp, kr0);

  283. 

  284.             if (height > 0)

  285.                 inp = vld1q_f32(src_dd + 2 * IW);

  286.             if (height > 0)

  287.                 out0 = vmlaq_f32(out0, inp, kr2);

  288.             if (height > 1)

  289.                 out1 = vmlaq_f32(out1, inp, kr1);

  290.             if (height > 2)

  291.                 out2 = vmlaq_f32(out2, inp, kr0);

  292. 

  293.             if (height > 0)

  294.                 inp = vld1q_f32(src_dd + 3 * IW);

  295.             if (height > 0)

  296.                 out0 = vmlaq_f32(out0, inp, kr3);

  297.             if (height > 1)

  298.                 out1 = vmlaq_f32(out1, inp, kr2);

  299.             if (height > 2)

  300.                 out2 = vmlaq_f32(out2, inp, kr1);

  301.             if (height > 3)

  302.                 out3 = vmlaq_f32(out3, inp, kr0);

  303. 

  304.             if (height > 1)

  305.                 inp = vld1q_f32(src_dd + 4 * IW);

  306.             if (height > 1)

  307.                 out1 = vmlaq_f32(out1, inp, kr3);

  308.             if (height > 2)

  309.                 out2 = vmlaq_f32(out2, inp, kr2);

  310.             if (height > 3)

  311.                 out3 = vmlaq_f32(out3, inp, kr1);

  312. 

  313.             if (height > 2)

  314.                 inp = vld1q_f32(src_dd + 5 * IW);

  315.             if (height > 2)

  316.                 out2 = vmlaq_f32(out2, inp, kr3);

  317.             if (height > 3)

  318.                 out3 = vmlaq_f32(out3, inp, kr2);

  319. 

  320.             if (height > 3)

  321.                 inp = vld1q_f32(src_dd + 6 * IW);

  322.             if (height > 3)

  323.                 out3 = vmlaq_f32(out3, inp, kr3);

  324.         }

  325.         STORE_OUT;

  326.     }

  327. };

  328. 

  329. template <int height, int width>

  330. struct do_pixel_proxy<5, height, width> {

  331.     static void exec(const float* src, const float* filter, float* dst,

  332.                      const int IH, const int IW, const int OH, const int OW,

  333.                      const int FW, const int oh, const int ow) {

  334.         (void)IH;

  335.         (void)OH;

  336.         const int ih = oh, iw = ow;

  337.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, kr2, kr3, kr4,

  338.                 inp;

  339.         src += ih * IW + iw;

  340.         dst += oh * OW + ow;

  341.         LOAD_OUT;

  342.         for (int fw = 0; fw < FW; ++fw) {

  343.             const float* src_dd = src + fw;

  344.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  345.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  346.             kr2 = vdupq_n_f32(filter[2 * FW + fw]);

  347.             kr3 = vdupq_n_f32(filter[3 * FW + fw]);

  348.             kr4 = vdupq_n_f32(filter[4 * FW + fw]);

  349. 

  350.             if (height > 0)

  351.                 inp = vld1q_f32(src_dd + 0 * IW);

  352.             if (height > 0)

  353.                 out0 = vmlaq_f32(out0, inp, kr0);

  354. 

  355.             if (height > 0)

  356.                 inp = vld1q_f32(src_dd + 1 * IW);

  357.             if (height > 0)

  358.                 out0 = vmlaq_f32(out0, inp, kr1);

  359.             if (height > 1)

  360.                 out1 = vmlaq_f32(out1, inp, kr0);

  361. 

  362.             if (height > 0)

  363.                 inp = vld1q_f32(src_dd + 2 * IW);

  364.             if (height > 0)

  365.                 out0 = vmlaq_f32(out0, inp, kr2);

  366.             if (height > 1)

  367.                 out1 = vmlaq_f32(out1, inp, kr1);

  368.             if (height > 2)

  369.                 out2 = vmlaq_f32(out2, inp, kr0);

  370. 

  371.             if (height > 0)

  372.                 inp = vld1q_f32(src_dd + 3 * IW);

  373.             if (height > 0)

  374.                 out0 = vmlaq_f32(out0, inp, kr3);

  375.             if (height > 1)

  376.                 out1 = vmlaq_f32(out1, inp, kr2);

  377.             if (height > 2)

  378.                 out2 = vmlaq_f32(out2, inp, kr1);

  379.             if (height > 3)

  380.                 out3 = vmlaq_f32(out3, inp, kr0);

  381. 

  382.             if (height > 0)

  383.                 inp = vld1q_f32(src_dd + 4 * IW);

  384.             if (height > 0)

  385.                 out0 = vmlaq_f32(out0, inp, kr4);

  386.             if (height > 1)

  387.                 out1 = vmlaq_f32(out1, inp, kr3);

  388.             if (height > 2)

  389.                 out2 = vmlaq_f32(out2, inp, kr2);

  390.             if (height > 3)

  391.                 out3 = vmlaq_f32(out3, inp, kr1);

  392. 

  393.             if (height > 1)

  394.                 inp = vld1q_f32(src_dd + 5 * IW);

  395.             if (height > 1)

  396.                 out1 = vmlaq_f32(out1, inp, kr4);

  397.             if (height > 2)

  398.                 out2 = vmlaq_f32(out2, inp, kr3);

  399.             if (height > 3)

  400.                 out3 = vmlaq_f32(out3, inp, kr2);

  401. 

  402.             if (height > 2)

  403.                 inp = vld1q_f32(src_dd + 6 * IW);

  404.             if (height > 2)

  405.                 out2 = vmlaq_f32(out2, inp, kr4);

  406.             if (height > 3)

  407.                 out3 = vmlaq_f32(out3, inp, kr3);

  408. 

  409.             if (height > 3)

  410.                 inp = vld1q_f32(src_dd + 7 * IW);

  411.             if (height > 3)

  412.                 out3 = vmlaq_f32(out3, inp, kr4);

  413.         }

  414.         STORE_OUT;

  415.     }

  416. };

  417. 

  418. template <int height, int width>

  419. struct do_pixel_proxy<6, height, width> {

  420.     static void exec(const float* src, const float* filter, float* dst,

  421.                      const int IH, const int IW, const int OH, const int OW,

  422.                      const int FW, const int oh, const int ow) {

  423.         (void)IH;

  424.         (void)OH;

  425.         const int ih = oh, iw = ow;

  426.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, kr2, kr3, kr4,

  427.                 kr5, inp;

  428.         src += ih * IW + iw;

  429.         dst += oh * OW + ow;

  430.         LOAD_OUT;

  431.         for (int fw = 0; fw < FW; ++fw) {

  432.             const float* src_dd = src + fw;

  433.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  434.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  435.             kr2 = vdupq_n_f32(filter[2 * FW + fw]);

  436.             kr3 = vdupq_n_f32(filter[3 * FW + fw]);

  437.             kr4 = vdupq_n_f32(filter[4 * FW + fw]);

  438.             kr5 = vdupq_n_f32(filter[5 * FW + fw]);

  439. 

  440.             if (height > 0)

  441.                 inp = vld1q_f32(src_dd + 0 * IW);

  442.             if (height > 0)

  443.                 out0 = vmlaq_f32(out0, inp, kr0);

  444. 

  445.             if (height > 0)

  446.                 inp = vld1q_f32(src_dd + 1 * IW);

  447.             if (height > 0)

  448.                 out0 = vmlaq_f32(out0, inp, kr1);

  449.             if (height > 1)

  450.                 out1 = vmlaq_f32(out1, inp, kr0);

  451. 

  452.             if (height > 0)

  453.                 inp = vld1q_f32(src_dd + 2 * IW);

  454.             if (height > 0)

  455.                 out0 = vmlaq_f32(out0, inp, kr2);

  456.             if (height > 1)

  457.                 out1 = vmlaq_f32(out1, inp, kr1);

  458.             if (height > 2)

  459.                 out2 = vmlaq_f32(out2, inp, kr0);

  460. 

  461.             if (height > 0)

  462.                 inp = vld1q_f32(src_dd + 3 * IW);

  463.             if (height > 0)

  464.                 out0 = vmlaq_f32(out0, inp, kr3);

  465.             if (height > 1)

  466.                 out1 = vmlaq_f32(out1, inp, kr2);

  467.             if (height > 2)

  468.                 out2 = vmlaq_f32(out2, inp, kr1);

  469.             if (height > 3)

  470.                 out3 = vmlaq_f32(out3, inp, kr0);

  471. 

  472.             if (height > 0)

  473.                 inp = vld1q_f32(src_dd + 4 * IW);

  474.             if (height > 0)

  475.                 out0 = vmlaq_f32(out0, inp, kr4);

  476.             if (height > 1)

  477.                 out1 = vmlaq_f32(out1, inp, kr3);

  478.             if (height > 2)

  479.                 out2 = vmlaq_f32(out2, inp, kr2);

  480.             if (height > 3)

  481.                 out3 = vmlaq_f32(out3, inp, kr1);

  482. 

  483.             if (height > 0)

  484.                 inp = vld1q_f32(src_dd + 5 * IW);

  485.             if (height > 0)

  486.                 out0 = vmlaq_f32(out0, inp, kr5);

  487.             if (height > 1)

  488.                 out1 = vmlaq_f32(out1, inp, kr4);

  489.             if (height > 2)

  490.                 out2 = vmlaq_f32(out2, inp, kr3);

  491.             if (height > 3)

  492.                 out3 = vmlaq_f32(out3, inp, kr2);

  493. 

  494.             if (height > 1)

  495.                 inp = vld1q_f32(src_dd + 6 * IW);

  496.             if (height > 1)

  497.                 out1 = vmlaq_f32(out1, inp, kr5);

  498.             if (height > 2)

  499.                 out2 = vmlaq_f32(out2, inp, kr4);

  500.             if (height > 3)

  501.                 out3 = vmlaq_f32(out3, inp, kr3);

  502. 

  503.             if (height > 2)

  504.                 inp = vld1q_f32(src_dd + 7 * IW);

  505.             if (height > 2)

  506.                 out2 = vmlaq_f32(out2, inp, kr5);

  507.             if (height > 3)

  508.                 out3 = vmlaq_f32(out3, inp, kr4);

  509. 

  510.             if (height > 3)

  511.                 inp = vld1q_f32(src_dd + 8 * IW);

  512.             if (height > 3)

  513.                 out3 = vmlaq_f32(out3, inp, kr5);

  514.         }

  515.         STORE_OUT;

  516.     }

  517. };

  518. 

  519. template <int height, int width>

  520. struct do_pixel_proxy<7, height, width> {

  521.     static void exec(const float* src, const float* filter, float* dst,

  522.                      const int IH, const int IW, const int OH, const int OW,

  523.                      const int FW, const int oh, const int ow) {

  524.         (void)IH;

  525.         (void)OH;

  526.         const int ih = oh, iw = ow;

  527.         float32x4_t out0{0}, out1{0}, out2{0}, out3{0}, kr0, kr1, kr2, kr3, kr4,

  528.                 kr5, kr6, inp;

  529.         src += ih * IW + iw;

  530.         dst += oh * OW + ow;

  531.         LOAD_OUT;

  532.         for (int fw = 0; fw < FW; ++fw) {

  533.             const float* src_dd = src + fw;

  534.             kr0 = vdupq_n_f32(filter[0 * FW + fw]);

  535.             kr1 = vdupq_n_f32(filter[1 * FW + fw]);

  536.             kr2 = vdupq_n_f32(filter[2 * FW + fw]);

  537.             kr3 = vdupq_n_f32(filter[3 * FW + fw]);

  538.             kr4 = vdupq_n_f32(filter[4 * FW + fw]);

  539.             kr5 = vdupq_n_f32(filter[5 * FW + fw]);

  540.             kr6 = vdupq_n_f32(filter[6 * FW + fw]);

  541. 

  542.             if (height > 0)

  543.                 inp = vld1q_f32(src_dd + 0 * IW);

  544.             if (height > 0)

  545.                 out0 = vmlaq_f32(out0, inp, kr0);

  546. 

  547.             if (height > 0)

  548.                 inp = vld1q_f32(src_dd + 1 * IW);

  549.             if (height > 0)

  550.                 out0 = vmlaq_f32(out0, inp, kr1);

  551.             if (height > 1)

  552.                 out1 = vmlaq_f32(out1, inp, kr0);

  553. 

  554.             if (height > 0)

  555.                 inp = vld1q_f32(src_dd + 2 * IW);

  556.             if (height > 0)

  557.                 out0 = vmlaq_f32(out0, inp, kr2);

  558.             if (height > 1)

  559.                 out1 = vmlaq_f32(out1, inp, kr1);

  560.             if (height > 2)

  561.                 out2 = vmlaq_f32(out2, inp, kr0);

  562. 

  563.             if (height > 0)

  564.                 inp = vld1q_f32(src_dd + 3 * IW);

  565.             if (height > 0)

  566.                 out0 = vmlaq_f32(out0, inp, kr3);

  567.             if (height > 1)

  568.                 out1 = vmlaq_f32(out1, inp, kr2);

  569.             if (height > 2)

  570.                 out2 = vmlaq_f32(out2, inp, kr1);

  571.             if (height > 3)

  572.                 out3 = vmlaq_f32(out3, inp, kr0);

  573. 

  574.             if (height > 0)

  575.                 inp = vld1q_f32(src_dd + 4 * IW);

  576.             if (height > 0)

  577.                 out0 = vmlaq_f32(out0, inp, kr4);

  578.             if (height > 1)

  579.                 out1 = vmlaq_f32(out1, inp, kr3);

  580.             if (height > 2)

  581.                 out2 = vmlaq_f32(out2, inp, kr2);

  582.             if (height > 3)

  583.                 out3 = vmlaq_f32(out3, inp, kr1);

  584. 

  585.             if (height > 0)

  586.                 inp = vld1q_f32(src_dd + 5 * IW);

  587.             if (height > 0)

  588.                 out0 = vmlaq_f32(out0, inp, kr5);

  589.             if (height > 1)

  590.                 out1 = vmlaq_f32(out1, inp, kr4);

  591.             if (height > 2)

  592.                 out2 = vmlaq_f32(out2, inp, kr3);

  593.             if (height > 3)

  594.                 out3 = vmlaq_f32(out3, inp, kr2);

  595. 

  596.             if (height > 0)

  597.                 inp = vld1q_f32(src_dd + 6 * IW);

  598.             if (height > 0)

  599.                 out0 = vmlaq_f32(out0, inp, kr6);

  600.             if (height > 1)

  601.                 out1 = vmlaq_f32(out1, inp, kr5);

  602.             if (height > 2)

  603.                 out2 = vmlaq_f32(out2, inp, kr4);

  604.             if (height > 3)

  605.                 out3 = vmlaq_f32(out3, inp, kr3);

  606. 

  607.             if (height > 1)

  608.                 inp = vld1q_f32(src_dd + 7 * IW);

  609.             if (height > 1)

  610.                 out1 = vmlaq_f32(out1, inp, kr6);

  611.             if (height > 2)

  612.                 out2 = vmlaq_f32(out2, inp, kr5);

  613.             if (height > 3)

  614.                 out3 = vmlaq_f32(out3, inp, kr4);

  615. 

  616.             if (height > 2)

  617.                 inp = vld1q_f32(src_dd + 8 * IW);

  618.             if (height > 2)

  619.                 out2 = vmlaq_f32(out2, inp, kr6);

  620.             if (height > 3)

  621.                 out3 = vmlaq_f32(out3, inp, kr5);

  622. 

  623.             if (height > 3)

  624.                 inp = vld1q_f32(src_dd + 9 * IW);

  625.             if (height > 3)

  626.                 out3 = vmlaq_f32(out3, inp, kr6);

  627.         }

  628.         STORE_OUT;

  629.     }

  630. };

  631. #undef cb_load

  632. #undef cb_load

  633. #undef LOAD_OUT

  634. #undef STORE_OUT

  635. 

  636. template <int FH, int height, int width>

  637. void do_pixel(const float* src, const float* filter, float* dst, const int IH,

  638.               const int IW, const int OH, const int OW, const int FW,

  639.               const int oh, const int ow) {

  640.     do_pixel_proxy<FH, height, width>::exec(src, filter, dst, IH, IW, OH, OW,

  641.                                             FW, oh, ow);

  642. }

  643. 

  644. template <int FH>

  645. void do_conv_tpl_enable_prefetch(const float* src, const float* filter,

  646.                                  float* dst, const int IH, const int IW,

  647.                                  const int OH, const int OW, const int FW) {

  648.     const int hbeg = 0, hend = OH;

  649.     const int wbeg = 0, wend = OW;

  650.     int i, j;

  651.     for (i = hbeg; i + 4 <= hend; i += 4) {

  652.         for (j = wbeg; j + 4 <= wend; j += 4) {

  653.             // do prefetch

  654.             const int prefetch_index_input =

  655.                     (j + 16) < wend

  656.                             ? i * IW + j + 16

  657.                             : (i + 4) * IW + (((j + 16 - wend) >> 2) << 2);

  658.             const int prefetch_index_output =

  659.                     (j + 16) < wend

  660.                             ? i * OW + j + 16

  661.                             : (i + 4) * OW + (((j + 16 - wend) >> 2) << 2);

  662.             const float* src_prefetch = src + prefetch_index_input;

  663.             const float* dst_prefetch = dst + prefetch_index_output;

  664.             for (int iw_id = 0; iw_id < FH + 3; ++iw_id) {

  665.                 __builtin_prefetch(src_prefetch + iw_id * IW, 0, 3);

  666.             }

  667.             __builtin_prefetch(dst_prefetch + 0 * OW, 1, 3);

  668.             __builtin_prefetch(dst_prefetch + 1 * OW, 1, 3);

  669.             __builtin_prefetch(dst_prefetch + 2 * OW, 1, 3);

  670.             __builtin_prefetch(dst_prefetch + 3 * OW, 1, 3);

  671.             do_pixel<FH, 4, 4>(src, filter, dst, IH, IW, OH, OW, FW, i, j);

  672.         }

  673. #define DISPATCH(width)                                                     \

  674.     do {                                                                    \

  675.         const int prefetch_index_input = (i + 4) * IW + 12;                 \

  676.         const int prefetch_index_output = (i + 4) * OW + 12;                \

  677.         const float* src_prefetch = src + prefetch_index_input;             \

  678.         const float* dst_prefetch = dst + prefetch_index_output;            \

  679.         for (int iw_id = 0; iw_id < FH + 3; ++iw_id) {                      \

  680.             __builtin_prefetch(src_prefetch + iw_id * IW, 0, 3);            \

  681.         }                                                                   \

  682.         __builtin_prefetch(dst_prefetch + 0 * OW, 1, 3);                    \

  683.         __builtin_prefetch(dst_prefetch + 1 * OW, 1, 3);                    \

  684.         __builtin_prefetch(dst_prefetch + 2 * OW, 1, 3);                    \

  685.         __builtin_prefetch(dst_prefetch + 3 * OW, 1, 3);                    \

  686.         do_pixel<FH, 4, width>(src, filter, dst, IH, IW, OH, OW, FW, i, j); \

  687.     } while (0)

  688.         switch (wend - j) {

  689.             case 1:

  690.                 DISPATCH(1);

  691.                 break;

  692.             case 2:

  693.                 DISPATCH(2);

  694.                 break;

  695.             case 3:

  696.                 DISPATCH(3);

  697.                 break;

  698.         }

  699. #undef DISPATCH

  700.     }

  701. 

  702. #define DISPATCH2(height, width)                                             \

  703.     do {                                                                     \

  704.         const int prefetch_index_input = IH * IW + 12;                       \

  705.         const float* src_prefetch = src + prefetch_index_input;              \

  706.         for (int iw_id = 0; iw_id < FH + 3; ++iw_id) {                       \

  707.             __builtin_prefetch(src_prefetch + iw_id * IW, 0, 3);             \

  708.         }                                                                    \

  709.         do_pixel<FH, height, width>(src, filter, dst, IH, IW, OH, OW, FW, i, \

  710.                                     j);                                      \

  711.     } while (0)

  712. 

  713. #define DISPATCH1(height)                                                    \

  714.     do {                                                                     \

  715.         for (j = wbeg; j + 4 <= wend; j += 4) {                              \

  716.             const int prefetch_index_input =                                 \

  717.                     (j + 16) < wend                                          \

  718.                             ? i * IW + j + 16                                \

  719.                             : (i + 4) * IW + (((j + 16 - wend) >> 2) << 2);  \

  720.             const int prefetch_index_output =                                \

  721.                     (j + 16) < wend                                          \

  722.                             ? i * OW + j + 16                                \

  723.                             : (i + 4) * OW + (((j + 16 - wend) >> 2) << 2);  \

  724.             const float* src_prefetch = src + prefetch_index_input;          \

  725.             const float* dst_prefetch = dst + prefetch_index_output;         \

  726.             for (int iw_id = 0; iw_id < FH + 3; ++iw_id) {                   \

  727.                 __builtin_prefetch(src_prefetch + iw_id * IW, 0, 3);         \

  728.             }                                                                \

  729.             __builtin_prefetch(dst_prefetch + 0 * OW, 1, 3);                 \

  730.             __builtin_prefetch(dst_prefetch + 1 * OW, 1, 3);                 \

  731.             __builtin_prefetch(dst_prefetch + 2 * OW, 1, 3);                 \

  732.             __builtin_prefetch(dst_prefetch + 3 * OW, 1, 3);                 \

  733.             do_pixel<FH, height, 4>(src, filter, dst, IH, IW, OH, OW, FW, i, \

  734.                                     j);                                      \

  735.         }                                                                    \

  736.         switch (wend - j) {                                                  \

  737.             case 1:                                                          \

  738.                 DISPATCH2(height, 1);                                        \

  739.                 break;                                                       \

  740.             case 2:                                                          \

  741.                 DISPATCH2(height, 2);                                        \

  742.                 break;                                                       \

  743.             case 3:                                                          \

  744.                 DISPATCH2(height, 3);                                        \

  745.                 break;                                                       \

  746.         }                                                                    \

  747.     } while (0)

  748.     switch (hend - i) {

  749.         case 1:

  750.             DISPATCH1(1);

  751.             break;

  752.         case 2:

  753.             DISPATCH1(2);

  754.             break;

  755.         case 3:

  756.             DISPATCH1(3);

  757.             break;

  758.     }

  759. #undef DISPATCH1

  760. #undef DISPATCH2

  761. }

  762. template <int FH>

  763. void do_conv_tpl_disable_prefetch(const float* src, const float* filter,

  764.                                   float* dst, const int IH, const int IW,

  765.                                   const int OH, const int OW, const int FW) {

  766.     const int hbeg = 0, hend = OH;

  767.     const int wbeg = 0, wend = OW;

  768.     int i, j;

  769.     for (i = hbeg; i + 4 <= hend; i += 4) {

  770.         for (j = wbeg; j + 4 <= wend; j += 4) {

  771.             do_pixel<FH, 4, 4>(src, filter, dst, IH, IW, OH, OW, FW, i, j);

  772.         }

  773. #define DISPATCH(width)                                                     \

  774.     do {                                                                    \

  775.         do_pixel<FH, 4, width>(src, filter, dst, IH, IW, OH, OW, FW, i, j); \

  776.     } while (0)

  777.         switch (wend - j) {

  778.             case 1:

  779.                 DISPATCH(1);

  780.                 break;

  781.             case 2:

  782.                 DISPATCH(2);

  783.                 break;

  784.             case 3:

  785.                 DISPATCH(3);

  786.                 break;

  787.         }

  788. #undef DISPATCH

  789.     }

  790. #define DISPATCH2(height, width)                                             \

  791.     do {                                                                     \

  792.         do_pixel<FH, height, width>(src, filter, dst, IH, IW, OH, OW, FW, i, \

  793.                                     j);                                      \

  794.     } while (0)

  795. #define DISPATCH1(height)                                                    \

  796.     do {                                                                     \

  797.         for (j = wbeg; j + 4 <= wend; j += 4) {                              \

  798.             do_pixel<FH, height, 4>(src, filter, dst, IH, IW, OH, OW, FW, i, \

  799.                                     j);                                      \

  800.         }                                                                    \

  801.         switch (wend - j) {                                                  \

  802.             case 1:                                                          \

  803.                 DISPATCH2(height, 1);                                        \

  804.                 break;                                                       \

  805.             case 2:                                                          \

  806.                 DISPATCH2(height, 2);                                        \

  807.                 break;                                                       \

  808.             case 3:                                                          \

  809.                 DISPATCH2(height, 3);                                        \

  810.                 break;                                                       \

  811.         }                                                                    \

  812.     } while (0)

  813.     switch (hend - i) {

  814.         case 1:

  815.             DISPATCH1(1);

  816.             break;

  817.         case 2:

  818.             DISPATCH1(2);

  819.             break;

  820.         case 3:

  821.             DISPATCH1(3);

  822.             break;

  823.     }

  824. #undef DISPATCH1

  825. #undef DISPATCH2

  826. }

  827. }  // anonymous namespace

  828. 

  829. void conv_bias::kern_direct(const float* src, const float* filter, float* dst,

  830.                               const int IH, const int IW, const int OH,

  831.                               const int OW, const int FH, const int FW) {

  832.     megdnn_assert_internal(FH <= 7);

  833.     if (IH > 100 && IW > 100) {

  834. #define GAO(FH)                                                              \

  835.     do {                                                                     \

  836.         return do_conv_tpl_enable_prefetch<FH>(src, filter, dst, IH, IW, OH, \

  837.                                                OW, FW);                      \

  838.     } while (0)

  839.         switch (FH) {

  840.             case 1:

  841.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(0)) { GAO(1); }

  842.                 MIDOUT_END();

  843.                 break;

  844.             case 2:

  845.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(1)) { GAO(2); }

  846.                 MIDOUT_END();

  847.                 break;

  848.             case 3:

  849.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(2)) { GAO(3); }

  850.                 MIDOUT_END();

  851.                 break;

  852.             case 4:

  853.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(3)) { GAO(4); }

  854.                 MIDOUT_END();

  855.                 break;

  856.             case 5:

  857.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(4)) { GAO(5); }

  858.                 MIDOUT_END();

  859.                 break;

  860.             case 6:

  861.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(5)) { GAO(6); }

  862.                 MIDOUT_END();

  863.                 break;

  864.             case 7:

  865.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(6)) { GAO(7); }

  866.                 MIDOUT_END();

  867.                 break;

  868.         }

  869. #undef GAO

  870.     } else {

  871. #define GAO(FH)                                                               \

  872.     do {                                                                      \

  873.         return do_conv_tpl_disable_prefetch<FH>(src, filter, dst, IH, IW, OH, \

  874.                                                 OW, FW);                      \

  875.     } while (0)

  876.         switch (FH) {

  877.             case 1:

  878.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(0)) { GAO(1); }

  879.                 MIDOUT_END();

  880.                 break;

  881.             case 2:

  882.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(1)) { GAO(2); }

  883.                 MIDOUT_END();

  884.                 break;

  885.             case 3:

  886.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(2)) { GAO(3); }

  887.                 MIDOUT_END();

  888.                 break;

  889.             case 4:

  890.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(3)) { GAO(4); }

  891.                 MIDOUT_END();

  892.                 break;

  893.             case 5:

  894.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(4)) { GAO(5); }

  895.                 MIDOUT_END();

  896.                 break;

  897.             case 6:

  898.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(5)) { GAO(6); }

  899.                 MIDOUT_END();

  900.                 break;

  901.             case 7:

  902.                 MIDOUT_BEGIN(megdnn_arm_conv_f32, midout_iv(6)) { GAO(7); }

  903.                 MIDOUT_END();

  904.                 break;

  905.         }

  906. #undef GAO

  907.     }

  908.     megdnn_assert_internal(0);

  909. }

  910. 

  911. // vim: syntax=cpp.doxygen