nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d8e9fc1443
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd8e9fc1443'
${ noResults }
ncnn/src/layer/arm/relu_arm.cpp

966 lines
36 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. 

  15. #include "relu_arm.h"

  16. 

  17. #if __ARM_NEON

  18. #include <arm_neon.h>

  19. #endif // __ARM_NEON

  20. 

  21. namespace ncnn {

  22. 

  23. ReLU_arm::ReLU_arm()

  24. {

  25. #if __ARM_NEON

  26.     support_packing = true;

  27. #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  28.     support_fp16_storage = true;

  29. #endif

  30. #endif // __ARM_NEON

  31. 

  32.     support_bf16_storage = true;

  33. }

  34. 

  35. int ReLU_arm::forward_inplace(Mat& bottom_top_blob, const Option& opt) const

  36. {

  37.     if (bottom_top_blob.elemsize == 1u)

  38.         return forward_inplace_int8_neon(bottom_top_blob, opt);

  39. 

  40.     int elembits = bottom_top_blob.elembits();

  41. 

  42. #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  43.     if (opt.use_fp16_storage && elembits == 16)

  44.         return forward_inplace_fp16s(bottom_top_blob, opt);

  45. #endif

  46. 

  47.     if (opt.use_bf16_storage && elembits == 16)

  48.         return forward_inplace_bf16s(bottom_top_blob, opt);

  49. 

  50.     int w = bottom_top_blob.w;

  51.     int h = bottom_top_blob.h;

  52.     int channels = bottom_top_blob.c;

  53.     int size = w * h;

  54.     int elempack = bottom_top_blob.elempack;

  55. 

  56. #if __ARM_NEON

  57.     if (elempack == 4)

  58.     {

  59.         if (slope == 0.f)

  60.         {

  61.             #pragma omp parallel for num_threads(opt.num_threads)

  62.             for (int q = 0; q < channels; q++)

  63.             {

  64.                 float* ptr = bottom_top_blob.channel(q);

  65. 

  66. #if __aarch64__

  67.                 asm volatile(

  68.                     "eor    v16.16b, v16.16b, v16.16b \n"

  69. 

  70.                     "lsr    w4, %w2, #3             \n" // w4 = nn = size >> 3

  71.                     "cmp    w4, #0                  \n"

  72.                     "beq    1f                      \n"

  73. 

  74.                     "0:                             \n"

  75.                     "prfm   pldl1keep, [%0, #512]   \n"

  76.                     "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n"

  77.                     "prfm   pldl1keep, [%0, #512]   \n"

  78.                     "ld1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%0] \n"

  79.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  80.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  81.                     "fmax   v2.4s, v2.4s, v16.4s    \n"

  82.                     "fmax   v3.4s, v3.4s, v16.4s    \n"

  83.                     "sub    %0, %0, #64             \n"

  84.                     "fmax   v4.4s, v4.4s, v16.4s    \n"

  85.                     "fmax   v5.4s, v5.4s, v16.4s    \n"

  86.                     "fmax   v6.4s, v6.4s, v16.4s    \n"

  87.                     "fmax   v7.4s, v7.4s, v16.4s    \n"

  88.                     "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n"

  89.                     "subs   w4, w4, #1              \n"

  90.                     "st1    {v4.4s, v5.4s, v6.4s, v7.4s}, [%0], #64 \n"

  91.                     "bne    0b                      \n"

  92. 

  93.                     "1:                             \n"

  94. 

  95.                     "and    w4, %w2, #7             \n" // w4 = remain = size & 7

  96. 

  97.                     "cmp    w4, #4                  \n" // w4 >= 4

  98.                     "blt    2f                      \n"

  99.                     "prfm   pldl1keep, [%0, #512]   \n"

  100.                     "ld1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0] \n"

  101.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  102.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  103.                     "fmax   v2.4s, v2.4s, v16.4s    \n"

  104.                     "fmax   v3.4s, v3.4s, v16.4s    \n"

  105.                     "sub    w4, w4, #4              \n"

  106.                     "st1    {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n"

  107.                     "2:                             \n"

  108. 

  109.                     "cmp    w4, #2                  \n" // w4 >= 2

  110.                     "blt    3f                      \n"

  111.                     "prfm   pldl1keep, [%0, #256]   \n"

  112.                     "ld1    {v0.4s, v1.4s}, [%0]    \n"

  113.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  114.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  115.                     "sub    w4, w4, #2              \n"

  116.                     "st1    {v0.4s, v1.4s}, [%0], #32 \n"

  117.                     "3:                             \n"

  118. 

  119.                     "cmp    w4, #0                  \n" // w4 > 0

  120.                     "beq    4f                      \n"

  121.                     "prfm   pldl1keep, [%0, #128]   \n"

  122.                     "ld1    {v0.4s}, [%0]           \n"

  123.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  124.                     "st1    {v0.4s}, [%0], #16      \n"

  125.                     "4:                             \n"

  126. 

  127.                     : "=r"(ptr) // %0

  128.                     : "0"(ptr),

  129.                     "r"(size) // %2

  130.                     : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");

  131. #else  // __aarch64__

  132.                 asm volatile(

  133.                     "veor       q12, q12            \n"

  134. 

  135.                     "lsr        r4, %2, #3          \n" // r4 = nn = size >> 3

  136.                     "cmp        r4, #0              \n"

  137.                     "beq        1f                  \n"

  138. 

  139.                     "0:                             \n"

  140.                     "pld        [%0, #512]          \n"

  141.                     "vldm       %0!, {d0-d7}        \n"

  142.                     "pld        [%0, #512]          \n"

  143.                     "vldm       %0, {d16-d23}       \n"

  144.                     "vmax.f32   q0, q0, q12         \n"

  145.                     "vmax.f32   q1, q1, q12         \n"

  146.                     "vmax.f32   q2, q2, q12         \n"

  147.                     "vmax.f32   q3, q3, q12         \n"

  148.                     "sub        %0, %0, #64         \n"

  149.                     "vmax.f32   q8, q8, q12         \n"

  150.                     "vmax.f32   q9, q9, q12         \n"

  151.                     "vmax.f32   q10, q10, q12       \n"

  152.                     "vmax.f32   q11, q11, q12       \n"

  153.                     "vstm       %0!, {d0-d7}        \n"

  154.                     "subs       r4, r4, #1          \n"

  155.                     "vstm       %0!, {d16-d23}      \n"

  156.                     "bne        0b                  \n"

  157. 

  158.                     "1:                             \n"

  159. 

  160.                     "and        r4, %2, #7          \n" // r4 = remain = size & 7

  161. 

  162.                     "cmp        r4, #4              \n" // r4 >= 4

  163.                     "blt        2f                  \n"

  164.                     "pld        [%0, #512]          \n"

  165.                     "vldm       %0, {d0-d7}         \n"

  166.                     "vmax.f32   q0, q0, q12         \n"

  167.                     "vmax.f32   q1, q1, q12         \n"

  168.                     "vmax.f32   q2, q2, q12         \n"

  169.                     "vmax.f32   q3, q3, q12         \n"

  170.                     "sub        r4, r4, #4          \n"

  171.                     "vstm       %0!, {d0-d7}        \n"

  172.                     "2:                             \n"

  173. 

  174.                     "cmp        r4, #2              \n" // r4 >= 2

  175.                     "blt        3f                  \n"

  176.                     "pld        [%0, #256]          \n"

  177.                     "vld1.f32   {d0-d3}, [%0 :128]  \n"

  178.                     "vmax.f32   q0, q0, q12         \n"

  179.                     "vmax.f32   q1, q1, q12         \n"

  180.                     "sub        r4, r4, #2          \n"

  181.                     "vst1.f32   {d0-d3}, [%0 :128]! \n"

  182.                     "3:                             \n"

  183. 

  184.                     "cmp        r4, #0              \n" // r4 > 0

  185.                     "beq        4f                  \n"

  186.                     "pld        [%0, #128]          \n"

  187.                     "vld1.f32   {d0-d1}, [%0 :128]  \n"

  188.                     "vmax.f32   q0, q0, q12         \n"

  189.                     "vst1.f32   {d0-d1}, [%0 :128]! \n"

  190.                     "4:                             \n"

  191. 

  192.                     : "=r"(ptr) // %0

  193.                     : "0"(ptr),

  194.                     "r"(size) // %2

  195.                     : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12");

  196. #endif // __aarch64__

  197.             }

  198.         }

  199.         else

  200.         {

  201.             #pragma omp parallel for num_threads(opt.num_threads)

  202.             for (int q = 0; q < channels; q++)

  203.             {

  204.                 float* ptr = bottom_top_blob.channel(q);

  205. 

  206.                 float32x4_t _zero = vdupq_n_f32(0.f);

  207.                 float32x4_t _slope = vdupq_n_f32(slope);

  208.                 for (int i = 0; i < size; i++)

  209.                 {

  210.                     float32x4_t _p = vld1q_f32(ptr);

  211.                     uint32x4_t _lemask = vcleq_f32(_p, _zero);

  212.                     float32x4_t _ps = vmulq_f32(_p, _slope);

  213.                     _p = vbslq_f32(_lemask, _ps, _p);

  214.                     vst1q_f32(ptr, _p);

  215. 

  216.                     ptr += 4;

  217.                 }

  218.             }

  219.         }

  220. 

  221.         return 0;

  222.     }

  223. #endif // __ARM_NEON

  224. 

  225.     if (slope == 0.f)

  226.     {

  227.         #pragma omp parallel for num_threads(opt.num_threads)

  228.         for (int q = 0; q < channels; q++)

  229.         {

  230.             float* ptr = bottom_top_blob.channel(q);

  231. 

  232. #if __ARM_NEON

  233.             int nn = size >> 2;

  234.             int remain = size - (nn << 2);

  235. #else

  236.             int remain = size;

  237. #endif // __ARM_NEON

  238. 

  239. #if __ARM_NEON

  240. #if __aarch64__

  241.             float32x4_t _zero = vdupq_n_f32(0.f);

  242.             for (; nn > 0; nn--)

  243.             {

  244.                 float32x4_t _p = vld1q_f32(ptr);

  245.                 _p = vmaxq_f32(_p, _zero);

  246.                 vst1q_f32(ptr, _p);

  247. 

  248.                 ptr += 4;

  249.             }

  250. #else

  251.             if (nn > 0)

  252.             {

  253.                 asm volatile(

  254.                     "veor       q1, q0, q0          \n"

  255.                     "0:                             \n"

  256.                     "pld        [%1, #128]          \n"

  257.                     "vld1.f32   {d0-d1}, [%1 :128]  \n"

  258.                     "vmax.f32   q0, q0, q1          \n"

  259.                     "subs       %0, #1              \n"

  260.                     "vst1.f32   {d0-d1}, [%1 :128]! \n"

  261.                     "bne        0b                  \n"

  262.                     : "=r"(nn), // %0

  263.                     "=r"(ptr) // %1

  264.                     : "0"(nn),

  265.                     "1"(ptr)

  266.                     : "cc", "memory", "q0", "q1");

  267.             }

  268. #endif // __aarch64__

  269. #endif // __ARM_NEON

  270.             for (; remain > 0; remain--)

  271.             {

  272.                 *ptr = std::max(*ptr, 0.f);

  273. 

  274.                 ptr++;

  275.             }

  276.         }

  277.     }

  278.     else

  279.     {

  280.         #pragma omp parallel for num_threads(opt.num_threads)

  281.         for (int q = 0; q < channels; q++)

  282.         {

  283.             float* ptr = bottom_top_blob.channel(q);

  284. 

  285. #if __ARM_NEON

  286.             int nn = size >> 2;

  287.             int remain = size - (nn << 2);

  288. #else

  289.             int remain = size;

  290. #endif // __ARM_NEON

  291. 

  292. #if __ARM_NEON

  293. #if __aarch64__

  294.             float32x4_t _zero = vdupq_n_f32(0.f);

  295.             float32x4_t _slope = vdupq_n_f32(slope);

  296.             for (; nn > 0; nn--)

  297.             {

  298.                 float32x4_t _p = vld1q_f32(ptr);

  299.                 uint32x4_t _lemask = vcleq_f32(_p, _zero);

  300.                 float32x4_t _ps = vmulq_f32(_p, _slope);

  301.                 _p = vbslq_f32(_lemask, _ps, _p);

  302.                 vst1q_f32(ptr, _p);

  303. 

  304.                 ptr += 4;

  305.             }

  306. #else

  307.             if (nn > 0)

  308.             {

  309.                 asm volatile(

  310.                     "veor       q1, q0, q0          \n"

  311.                     "vdup.f32   q2, %4              \n"

  312.                     "0:                             \n"

  313.                     "pld        [%1, #128]          \n"

  314.                     "vld1.f32   {d0-d1}, [%1 :128]  \n"

  315.                     "vcle.f32   q3, q0, q1          \n"

  316.                     "vmul.f32   q4, q0, q2          \n"

  317.                     "vbit.32    q0, q4, q3          \n"

  318.                     "subs       %0, #1              \n"

  319.                     "vst1.f32   {d0-d1}, [%1 :128]! \n"

  320.                     "bne        0b                  \n"

  321.                     : "=r"(nn), // %0

  322.                     "=r"(ptr) // %1

  323.                     : "0"(nn),

  324.                     "1"(ptr),

  325.                     "r"(slope) // %4

  326.                     : "cc", "memory", "q0", "q1", "q2", "q3", "q4");

  327.             }

  328. #endif // __aarch64__

  329. #endif // __ARM_NEON

  330.             for (; remain > 0; remain--)

  331.             {

  332.                 if (*ptr < 0)

  333.                     *ptr *= slope;

  334. 

  335.                 ptr++;

  336.             }

  337.         }

  338.     }

  339. 

  340.     return 0;

  341. }

  342. 

  343. #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  344. int ReLU_arm::forward_inplace_fp16s(Mat& bottom_top_blob, const Option& opt) const

  345. {

  346.     int w = bottom_top_blob.w;

  347.     int h = bottom_top_blob.h;

  348.     int channels = bottom_top_blob.c;

  349.     int size = w * h;

  350.     int elempack = bottom_top_blob.elempack;

  351. 

  352.     if (elempack == 8)

  353.     {

  354.         if (slope == 0.f)

  355.         {

  356.             #pragma omp parallel for num_threads(opt.num_threads)

  357.             for (int q = 0; q < channels; q++)

  358.             {

  359.                 __fp16* ptr = bottom_top_blob.channel(q);

  360. 

  361.                 asm volatile(

  362.                     "eor    v16.16b, v16.16b, v16.16b \n"

  363. 

  364.                     "lsr    w4, %w2, #2             \n" // w4 = nn = size >> 2

  365.                     "cmp    w4, #0                  \n"

  366.                     "beq    1f                      \n"

  367. 

  368.                     "0:                             \n"

  369.                     "prfm   pldl1keep, [%0, #512]   \n"

  370.                     "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n"

  371.                     "fmax   v0.8h, v0.8h, v16.8h    \n"

  372.                     "fmax   v1.8h, v1.8h, v16.8h    \n"

  373.                     "fmax   v2.8h, v2.8h, v16.8h    \n"

  374.                     "fmax   v3.8h, v3.8h, v16.8h    \n"

  375.                     "subs   w4, w4, #1              \n"

  376.                     "st1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"

  377.                     "bne    0b                      \n"

  378.                     "1:                             \n"

  379. 

  380.                     "and    w4, %w2, #3             \n" // w4 = remain = size & 3

  381. 

  382.                     "cmp    w4, #2                  \n" // w4 >= 2

  383.                     "blt    2f                      \n"

  384.                     "prfm   pldl1keep, [%0, #256]   \n"

  385.                     "ld1    {v0.8h, v1.8h}, [%0]    \n"

  386.                     "fmax   v0.8h, v0.8h, v16.8h    \n"

  387.                     "fmax   v1.8h, v1.8h, v16.8h    \n"

  388.                     "sub    w4, w4, #2              \n"

  389.                     "st1    {v0.8h, v1.8h}, [%0], #32 \n"

  390.                     "2:                             \n"

  391. 

  392.                     "cmp    w4, #0                  \n" // w4 > 0

  393.                     "beq    3f                      \n"

  394.                     "prfm   pldl1keep, [%0, #128]   \n"

  395.                     "ld1    {v0.8h}, [%0]           \n"

  396.                     "fmax   v0.8h, v0.8h, v16.8h    \n"

  397.                     "st1    {v0.8h}, [%0], #16      \n"

  398.                     "3:                             \n"

  399. 

  400.                     : "=r"(ptr) // %0

  401.                     : "0"(ptr),

  402.                     "r"(size) // %2

  403.                     : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");

  404.             }

  405.         }

  406.         else

  407.         {

  408.             #pragma omp parallel for num_threads(opt.num_threads)

  409.             for (int q = 0; q < channels; q++)

  410.             {

  411.                 __fp16* ptr = bottom_top_blob.channel(q);

  412. 

  413.                 float16x8_t _zero = vdupq_n_f16((__fp16)0.f);

  414.                 float16x8_t _slope = vdupq_n_f16((__fp16)slope);

  415. 

  416.                 for (int i = 0; i < size; i++)

  417.                 {

  418.                     float16x8_t _p = vld1q_f16(ptr);

  419.                     uint16x8_t _lemask = vcleq_f16(_p, _zero);

  420.                     float16x8_t _ps = vmulq_f16(_p, _slope);

  421.                     _p = vbslq_f16(_lemask, _ps, _p);

  422.                     vst1q_f16(ptr, _p);

  423. 

  424.                     ptr += 8;

  425.                 }

  426.             }

  427.         }

  428. 

  429.         return 0;

  430.     }

  431. 

  432.     if (elempack == 4)

  433.     {

  434.         if (slope == 0.f)

  435.         {

  436.             #pragma omp parallel for num_threads(opt.num_threads)

  437.             for (int q = 0; q < channels; q++)

  438.             {

  439.                 __fp16* ptr = bottom_top_blob.channel(q);

  440. 

  441.                 float16x8_t _zero = vdupq_n_f16((__fp16)0.f);

  442. 

  443.                 int i = 0;

  444.                 for (; i + 1 < size; i += 2)

  445.                 {

  446.                     float16x8_t _p = vld1q_f16(ptr);

  447.                     _p = vmaxq_f16(_p, _zero);

  448.                     vst1q_f16(ptr, _p);

  449. 

  450.                     ptr += 8;

  451.                 }

  452.                 for (; i < size; i++)

  453.                 {

  454.                     float16x4_t _p = vld1_f16(ptr);

  455.                     _p = vmax_f16(_p, vget_low_f16(_zero));

  456.                     vst1_f16(ptr, _p);

  457. 

  458.                     ptr += 4;

  459.                 }

  460.             }

  461.         }

  462.         else

  463.         {

  464.             #pragma omp parallel for num_threads(opt.num_threads)

  465.             for (int q = 0; q < channels; q++)

  466.             {

  467.                 __fp16* ptr = bottom_top_blob.channel(q);

  468. 

  469.                 float16x8_t _zero = vdupq_n_f16((__fp16)0.f);

  470.                 float16x8_t _slope = vdupq_n_f16((__fp16)slope);

  471. 

  472.                 int i = 0;

  473.                 for (; i + 1 < size; i += 2)

  474.                 {

  475.                     float16x8_t _p = vld1q_f16(ptr);

  476.                     uint16x8_t _lemask = vcleq_f16(_p, _zero);

  477.                     float16x8_t _ps = vmulq_f16(_p, _slope);

  478.                     _p = vbslq_f16(_lemask, _ps, _p);

  479.                     vst1q_f16(ptr, _p);

  480. 

  481.                     ptr += 8;

  482.                 }

  483.                 for (; i < size; i++)

  484.                 {

  485.                     float16x4_t _p = vld1_f16(ptr);

  486.                     uint16x4_t _lemask = vcle_f16(_p, vget_low_f16(_zero));

  487.                     float16x4_t _ps = vmul_f16(_p, vget_low_f16(_slope));

  488.                     _p = vbsl_f16(_lemask, _ps, _p);

  489.                     vst1_f16(ptr, _p);

  490. 

  491.                     ptr += 4;

  492.                 }

  493.             }

  494.         }

  495. 

  496.         return 0;

  497.     }

  498. 

  499.     if (slope == 0.f)

  500.     {

  501.         #pragma omp parallel for num_threads(opt.num_threads)

  502.         for (int q = 0; q < channels; q++)

  503.         {

  504.             __fp16* ptr = bottom_top_blob.channel(q);

  505. 

  506.             float16x8_t _zero = vdupq_n_f16((__fp16)0.f);

  507. 

  508.             int i = 0;

  509.             for (; i + 7 < size; i += 8)

  510.             {

  511.                 float16x8_t _p = vld1q_f16(ptr);

  512.                 _p = vmaxq_f16(_p, _zero);

  513.                 vst1q_f16(ptr, _p);

  514. 

  515.                 ptr += 8;

  516.             }

  517.             for (; i + 3 < size; i += 4)

  518.             {

  519.                 float16x4_t _p = vld1_f16(ptr);

  520.                 _p = vmax_f16(_p, vget_low_f16(_zero));

  521.                 vst1_f16(ptr, _p);

  522. 

  523.                 ptr += 4;

  524.             }

  525.             for (; i < size; i++)

  526.             {

  527.                 __fp16 v = ptr[0];

  528.                 if (v < (__fp16)0.f)

  529.                     ptr[0] = (__fp16)0.f;

  530. 

  531.                 ptr += 1;

  532.             }

  533.         }

  534.     }

  535.     else

  536.     {

  537.         #pragma omp parallel for num_threads(opt.num_threads)

  538.         for (int q = 0; q < channels; q++)

  539.         {

  540.             __fp16* ptr = bottom_top_blob.channel(q);

  541. 

  542.             float16x8_t _zero = vdupq_n_f16((__fp16)0.f);

  543.             float16x8_t _slope = vdupq_n_f16((__fp16)slope);

  544. 

  545.             int i = 0;

  546.             for (; i + 7 < size; i += 8)

  547.             {

  548.                 float16x8_t _p = vld1q_f16(ptr);

  549.                 uint16x8_t _lemask = vcleq_f16(_p, _zero);

  550.                 float16x8_t _ps = vmulq_f16(_p, _slope);

  551.                 _p = vbslq_f16(_lemask, _ps, _p);

  552.                 vst1q_f16(ptr, _p);

  553. 

  554.                 ptr += 8;

  555.             }

  556.             for (; i + 3 < size; i += 4)

  557.             {

  558.                 float16x4_t _p = vld1_f16(ptr);

  559.                 uint16x4_t _lemask = vcle_f16(_p, vget_low_f16(_zero));

  560.                 float16x4_t _ps = vmul_f16(_p, vget_low_f16(_slope));

  561.                 _p = vbsl_f16(_lemask, _ps, _p);

  562.                 vst1_f16(ptr, _p);

  563. 

  564.                 ptr += 4;

  565.             }

  566.             for (; i < size; i++)

  567.             {

  568.                 __fp16 v = ptr[0];

  569.                 if (v < (__fp16)0.f)

  570.                     ptr[0] = v * (__fp16)slope;

  571. 

  572.                 ptr += 1;

  573.             }

  574.         }

  575.     }

  576. 

  577.     return 0;

  578. }

  579. #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  580. 

  581. int ReLU_arm::forward_inplace_bf16s(Mat& bottom_top_blob, const Option& opt) const

  582. {

  583.     int w = bottom_top_blob.w;

  584.     int h = bottom_top_blob.h;

  585.     int channels = bottom_top_blob.c;

  586.     int size = w * h;

  587.     int elempack = bottom_top_blob.elempack;

  588. 

  589. #if __ARM_NEON

  590.     if (elempack == 4)

  591.     {

  592.         if (slope == 0.f)

  593.         {

  594.             #pragma omp parallel for num_threads(opt.num_threads)

  595.             for (int q = 0; q < channels; q++)

  596.             {

  597.                 unsigned short* ptr = bottom_top_blob.channel(q);

  598. 

  599. #if __aarch64__

  600.                 asm volatile(

  601.                     "eor    v16.16b, v16.16b, v16.16b \n"

  602. 

  603.                     "lsr    w4, %w2, #3             \n" // w4 = nn = size >> 3

  604.                     "cmp    w4, #0                  \n"

  605.                     "beq    1f                      \n"

  606. 

  607.                     "0:                             \n"

  608.                     "prfm   pldl1keep, [%0, #512]   \n"

  609.                     "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n"

  610.                     "shll   v0.4s, v4.4h, #16       \n"

  611.                     "shll2  v1.4s, v4.8h, #16       \n"

  612.                     "shll   v2.4s, v5.4h, #16       \n"

  613.                     "shll2  v3.4s, v5.8h, #16       \n"

  614.                     "shll   v4.4s, v6.4h, #16       \n"

  615.                     "shll2  v5.4s, v6.8h, #16       \n"

  616.                     "shll   v6.4s, v7.4h, #16       \n"

  617.                     "shll2  v7.4s, v7.8h, #16       \n"

  618.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  619.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  620.                     "fmax   v2.4s, v2.4s, v16.4s    \n"

  621.                     "fmax   v3.4s, v3.4s, v16.4s    \n"

  622.                     "fmax   v4.4s, v4.4s, v16.4s    \n"

  623.                     "fmax   v5.4s, v5.4s, v16.4s    \n"

  624.                     "fmax   v6.4s, v6.4s, v16.4s    \n"

  625.                     "fmax   v7.4s, v7.4s, v16.4s    \n"

  626.                     "shrn   v0.4h, v0.4s, #16       \n"

  627.                     "shrn2  v0.8h, v1.4s, #16       \n"

  628.                     "shrn   v1.4h, v2.4s, #16       \n"

  629.                     "shrn2  v1.8h, v3.4s, #16       \n"

  630.                     "shrn   v2.4h, v4.4s, #16       \n"

  631.                     "shrn2  v2.8h, v5.4s, #16       \n"

  632.                     "shrn   v3.4h, v6.4s, #16       \n"

  633.                     "shrn2  v3.8h, v7.4s, #16       \n"

  634.                     "subs   w4, w4, #1              \n"

  635.                     "st1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"

  636.                     "bne    0b                      \n"

  637. 

  638.                     "1:                             \n"

  639. 

  640.                     "and    w4, %w2, #7             \n" // w4 = remain = size & 7

  641. 

  642.                     "cmp    w4, #4                  \n" // w4 >= 4

  643.                     "blt    2f                      \n"

  644.                     "prfm   pldl1keep, [%0, #256]   \n"

  645.                     "ld1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%0] \n"

  646.                     "shll   v0.4s, v0.4h, #16       \n"

  647.                     "shll   v1.4s, v1.4h, #16       \n"

  648.                     "shll   v2.4s, v2.4h, #16       \n"

  649.                     "shll   v3.4s, v3.4h, #16       \n"

  650.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  651.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  652.                     "fmax   v2.4s, v2.4s, v16.4s    \n"

  653.                     "fmax   v3.4s, v3.4s, v16.4s    \n"

  654.                     "shrn   v0.4h, v0.4s, #16       \n"

  655.                     "shrn   v1.4h, v1.4s, #16       \n"

  656.                     "shrn   v2.4h, v2.4s, #16       \n"

  657.                     "shrn   v3.4h, v3.4s, #16       \n"

  658.                     "sub    w4, w4, #4              \n"

  659.                     "st1    {v0.4h, v1.4h, v2.4h, v3.4h}, [%0], #32 \n"

  660.                     "2:                             \n"

  661. 

  662.                     "cmp    w4, #2                  \n" // w4 >= 2

  663.                     "blt    3f                      \n"

  664.                     "prfm   pldl1keep, [%0, #128]   \n"

  665.                     "ld1    {v0.4h, v1.4h}, [%0]    \n"

  666.                     "shll   v0.4s, v0.4h, #16       \n"

  667.                     "shll   v1.4s, v1.4h, #16       \n"

  668.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  669.                     "fmax   v1.4s, v1.4s, v16.4s    \n"

  670.                     "shrn   v0.4h, v0.4s, #16       \n"

  671.                     "shrn   v1.4h, v1.4s, #16       \n"

  672.                     "sub    w4, w4, #2              \n"

  673.                     "st1    {v0.4h, v1.4h}, [%0], #16 \n"

  674.                     "3:                             \n"

  675. 

  676.                     "cmp    w4, #0                  \n" // w4 > 0

  677.                     "beq    4f                      \n"

  678.                     "prfm   pldl1keep, [%0, #64]    \n"

  679.                     "ld1    {v0.4h}, [%0]           \n"

  680.                     "shll   v0.4s, v0.4h, #16       \n"

  681.                     "fmax   v0.4s, v0.4s, v16.4s    \n"

  682.                     "shrn   v0.4h, v0.4s, #16       \n"

  683.                     "st1    {v0.4h}, [%0], #8       \n"

  684.                     "4:                             \n"

  685. 

  686.                     : "=r"(ptr) // %0

  687.                     : "0"(ptr),

  688.                     "r"(size) // %2

  689.                     : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");

  690. #else  // __aarch64__

  691.                 asm volatile(

  692.                     "veor       q12, q12            \n"

  693. 

  694.                     "lsr        r4, %2, #3          \n" // r4 = nn = size >> 3

  695.                     "cmp        r4, #0              \n"

  696.                     "beq        1f                  \n"

  697. 

  698.                     "0:                             \n"

  699.                     "pld        [%0, #512]          \n"

  700.                     "vldm       %0, {d16-d23}       \n"

  701.                     "vshll.u16  q0, d16, #16        \n"

  702.                     "vshll.u16  q1, d17, #16        \n"

  703.                     "vshll.u16  q2, d18, #16        \n"

  704.                     "vshll.u16  q3, d19, #16        \n"

  705.                     "vshll.u16  q8, d20, #16        \n"

  706.                     "vshll.u16  q9, d21, #16        \n"

  707.                     "vshll.u16  q10, d22, #16       \n"

  708.                     "vshll.u16  q11, d23, #16       \n"

  709.                     "vmax.f32   q0, q0, q12         \n"

  710.                     "vmax.f32   q1, q1, q12         \n"

  711.                     "vmax.f32   q2, q2, q12         \n"

  712.                     "vmax.f32   q3, q3, q12         \n"

  713.                     "vmax.f32   q8, q8, q12         \n"

  714.                     "vmax.f32   q9, q9, q12         \n"

  715.                     "vmax.f32   q10, q10, q12       \n"

  716.                     "vmax.f32   q11, q11, q12       \n"

  717.                     "vshrn.u32  d0, q0, #16         \n"

  718.                     "vshrn.u32  d1, q1, #16         \n"

  719.                     "vshrn.u32  d2, q2, #16         \n"

  720.                     "vshrn.u32  d3, q3, #16         \n"

  721.                     "vshrn.u32  d4, q8, #16         \n"

  722.                     "vshrn.u32  d5, q9, #16         \n"

  723.                     "vshrn.u32  d6, q10, #16        \n"

  724.                     "vshrn.u32  d7, q11, #16        \n"

  725.                     "subs       r4, r4, #1          \n"

  726.                     "vstm       %0!, {d0-d7}        \n"

  727.                     "bne        0b                  \n"

  728. 

  729.                     "1:                             \n"

  730. 

  731.                     "and        r4, %2, #7          \n" // r4 = remain = size & 7

  732. 

  733.                     "cmp        r4, #4              \n" // r4 >= 4

  734.                     "blt        2f                  \n"

  735.                     "pld        [%0, #256]          \n"

  736.                     "vld1.u16   {d4-d7}, [%0 :64]   \n"

  737.                     "vshll.u16  q0, d4, #16         \n"

  738.                     "vshll.u16  q1, d5, #16         \n"

  739.                     "vshll.u16  q2, d6, #16         \n"

  740.                     "vshll.u16  q3, d7, #16         \n"

  741.                     "vmax.f32   q0, q0, q12         \n"

  742.                     "vmax.f32   q1, q1, q12         \n"

  743.                     "vmax.f32   q2, q2, q12         \n"

  744.                     "vmax.f32   q3, q3, q12         \n"

  745.                     "vshrn.u32  d0, q0, #16         \n"

  746.                     "vshrn.u32  d1, q1, #16         \n"

  747.                     "vshrn.u32  d2, q2, #16         \n"

  748.                     "vshrn.u32  d3, q3, #16         \n"

  749.                     "sub        r4, r4, #4          \n"

  750.                     "vst1.u16   {d0-d3}, [%0 :64]!  \n"

  751.                     "2:                             \n"

  752. 

  753.                     "cmp        r4, #2              \n" // r4 >= 2

  754.                     "blt        3f                  \n"

  755.                     "pld        [%0, #128]          \n"

  756.                     "vld1.u16   {d2-d3}, [%0 :64]   \n"

  757.                     "vshll.u16  q0, d2, #16         \n"

  758.                     "vshll.u16  q1, d3, #16         \n"

  759.                     "vmax.f32   q0, q0, q12         \n"

  760.                     "vmax.f32   q1, q1, q12         \n"

  761.                     "vshrn.u32  d0, q0, #16         \n"

  762.                     "vshrn.u32  d1, q1, #16         \n"

  763.                     "sub        r4, r4, #2          \n"

  764.                     "vst1.u16   {d0-d1}, [%0 :64]!  \n"

  765.                     "3:                             \n"

  766. 

  767.                     "cmp        r4, #0              \n" // r4 > 0

  768.                     "beq        4f                  \n"

  769.                     "pld        [%0, #64]           \n"

  770.                     "vld1.u16   {d0}, [%0 :64]      \n"

  771.                     "vshll.u16  q0, d0, #16         \n"

  772.                     "vmax.f32   q0, q0, q12         \n"

  773.                     "vshrn.u32  d0, q0, #16         \n"

  774.                     "vst1.u16   {d0}, [%0 :64]!     \n"

  775.                     "4:                             \n"

  776. 

  777.                     : "=r"(ptr) // %0

  778.                     : "0"(ptr),

  779.                     "r"(size) // %2

  780.                     : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12");

  781. #endif // __aarch64__

  782. 

  783.                 //                 float32x4_t _zero = vdupq_n_f32(0.f);

  784.                 //                 for (int i=0; i<size; i++)

  785.                 //                 {

  786.                 //                     float32x4_t _p = vcvt_f32_bf16(vld1_u16(ptr));

  787.                 //                     _p = vmaxq_f32(_p, _zero);

  788.                 //                     vst1_u16(ptr, vcvt_bf16_f32(_p));

  789.                 //

  790.                 //                     ptr += 4;

  791.                 //                 }

  792.             }

  793.         }

  794.         else

  795.         {

  796.             #pragma omp parallel for num_threads(opt.num_threads)

  797.             for (int q = 0; q < channels; q++)

  798.             {

  799.                 unsigned short* ptr = bottom_top_blob.channel(q);

  800. 

  801.                 float32x4_t _zero = vdupq_n_f32(0.f);

  802.                 float32x4_t _slope = vdupq_n_f32(slope);

  803.                 for (int i = 0; i < size; i++)

  804.                 {

  805.                     float32x4_t _p = vcvt_f32_bf16(vld1_u16(ptr));

  806.                     uint32x4_t _lemask = vcleq_f32(_p, _zero);

  807.                     float32x4_t _ps = vmulq_f32(_p, _slope);

  808.                     _p = vbslq_f32(_lemask, _ps, _p);

  809.                     vst1_u16(ptr, vcvt_bf16_f32(_p));

  810. 

  811.                     ptr += 4;

  812.                 }

  813.             }

  814.         }

  815. 

  816.         return 0;

  817.     }

  818. #endif // __ARM_NEON

  819. 

  820.     if (slope == 0.f)

  821.     {

  822.         #pragma omp parallel for num_threads(opt.num_threads)

  823.         for (int q = 0; q < channels; q++)

  824.         {

  825.             unsigned short* ptr = bottom_top_blob.channel(q);

  826. 

  827.             int i = 0;

  828. #if __ARM_NEON

  829.             float32x4_t _zero = vdupq_n_f32(0.f);

  830.             for (; i + 3 < size; i += 4)

  831.             {

  832.                 float32x4_t _p = vcvt_f32_bf16(vld1_u16(ptr));

  833.                 _p = vmaxq_f32(_p, _zero);

  834.                 vst1_u16(ptr, vcvt_bf16_f32(_p));

  835. 

  836.                 ptr += 4;

  837.             }

  838. #endif // __ARM_NEON

  839.             for (; i < size; i++)

  840.             {

  841.                 float v = bfloat16_to_float32(ptr[0]);

  842.                 if (v < 0.f)

  843.                     ptr[0] = float32_to_bfloat16(0.f);

  844. 

  845.                 ptr += 1;

  846.             }

  847.         }

  848.     }

  849.     else

  850.     {

  851.         #pragma omp parallel for num_threads(opt.num_threads)

  852.         for (int q = 0; q < channels; q++)

  853.         {

  854.             unsigned short* ptr = bottom_top_blob.channel(q);

  855. 

  856.             int i = 0;

  857. #if __ARM_NEON

  858.             float32x4_t _zero = vdupq_n_f32(0.f);

  859.             float32x4_t _slope = vdupq_n_f32(slope);

  860.             for (; i + 3 < size; i += 4)

  861.             {

  862.                 float32x4_t _p = vcvt_f32_bf16(vld1_u16(ptr));

  863.                 uint32x4_t _lemask = vcleq_f32(_p, _zero);

  864.                 float32x4_t _ps = vmulq_f32(_p, _slope);

  865.                 _p = vbslq_f32(_lemask, _ps, _p);

  866.                 vst1_u16(ptr, vcvt_bf16_f32(_p));

  867. 

  868.                 ptr += 4;

  869.             }

  870. #endif // __ARM_NEON

  871.             for (; i < size; i++)

  872.             {

  873.                 float v = bfloat16_to_float32(ptr[0]);

  874.                 if (v < 0.f)

  875.                     ptr[0] = float32_to_bfloat16(v * slope);

  876. 

  877.                 ptr += 1;

  878.             }

  879.         }

  880.     }

  881. 

  882.     return 0;

  883. }

  884. 

  885. int ReLU_arm::forward_inplace_int8_neon(Mat& bottom_top_blob, const Option& opt) const

  886. {

  887.     int w = bottom_top_blob.w;

  888.     int h = bottom_top_blob.h;

  889.     int channels = bottom_top_blob.c;

  890.     int size = w * h;

  891. 

  892.     if (slope == 0.f)

  893.     {

  894.         #pragma omp parallel for num_threads(opt.num_threads)

  895.         for (int q = 0; q < channels; q++)

  896.         {

  897.             signed char* ptr = bottom_top_blob.channel(q);

  898. 

  899. #if __ARM_NEON

  900.             int nn = size >> 4;

  901.             int remain = size - (nn << 4);

  902. #else

  903.             int remain = size;

  904. #endif // __ARM_NEON

  905. 

  906. #if __ARM_NEON

  907. #if __aarch64__

  908.             int8x16_t _zero = vdupq_n_s8(0);

  909.             for (; nn > 0; nn--)

  910.             {

  911.                 int8x16_t _p = vld1q_s8(ptr);

  912.                 _p = vmaxq_s8(_p, _zero);

  913.                 vst1q_s8(ptr, _p);

  914. 

  915.                 ptr += 16;

  916.             }

  917. #else

  918.             if (nn > 0)

  919.             {

  920.                 asm volatile(

  921.                     "veor       q1, q0, q0          \n"

  922.                     "0:                             \n"

  923.                     "pld        [%1, #128]          \n"

  924.                     "vld1.s8    {d0-d1}, [%1 :128]  \n"

  925.                     "vmax.s8    q0, q0, q1          \n"

  926.                     "subs       %0, #1              \n"

  927.                     "vst1.s8    {d0-d1}, [%1 :128]! \n"

  928.                     "bne        0b                  \n"

  929.                     : "=r"(nn), // %0

  930.                     "=r"(ptr) // %1

  931.                     : "0"(nn),

  932.                     "1"(ptr)

  933.                     : "cc", "memory", "q0", "q1");

  934.             }

  935. #endif // __aarch64__

  936. #endif // __ARM_NEON

  937.             for (; remain > 0; remain--)

  938.             {

  939.                 if (*ptr < 0)

  940.                     *ptr = 0;

  941. 

  942.                 ptr++;

  943.             }

  944.         }

  945.     }

  946.     else

  947.     {

  948.         // TODO

  949.         // #pragma omp parallel for num_threads(opt.num_threads)

  950.         // for (int q=0; q<channels; q++)

  951.         // {

  952.         //     float* ptr = bottom_top_blob.channel(q);

  953. 

  954.         //     for (int i=0; i<size; i++)

  955.         //     {

  956.         //         if (ptr[i] < 0)

  957.         //             ptr[i] *= slope;

  958.         //     }

  959.         // }

  960.     }

  961. 

  962.     return 0;

  963. }

  964. 

  965. } // namespace ncnn