diff --git a/src/layer/arm/innerproduct_arm.cpp b/src/layer/arm/innerproduct_arm.cpp index 88d5136db..37df9db0e 100644 --- a/src/layer/arm/innerproduct_arm.cpp +++ b/src/layer/arm/innerproduct_arm.cpp @@ -27,6 +27,11 @@ namespace ncnn { +#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ +#include "innerproduct_fp16s.h" +#include "innerproduct_gemm_fp16s.h" +#endif + InnerProduct_arm::InnerProduct_arm() { #if __ARM_NEON @@ -76,6 +81,13 @@ int InnerProduct_arm::create_pipeline(const Option& opt) } #endif +#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ + if (cpu_support_arm_vfpv4() && opt.use_fp16_storage) + { + return create_pipeline_fp16s(opt); + } +#endif + const int num_input = weight_data_size / num_output; int out_elempack = 1; @@ -161,6 +173,13 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio return forward_bf16s(bottom_blob, top_blob, opt); #endif +#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ + if (cpu_support_arm_vfpv4() && opt.use_fp16_storage) + { + return forward_fp16s(bottom_blob, top_blob, opt); + } +#endif + const int num_input = weight_data_size / num_output; if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1) @@ -829,6 +848,86 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio return 0; } +#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ +int InnerProduct_arm::create_pipeline_fp16s(const Option& opt) +{ + const int num_input = weight_data_size / num_output; + + innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt); + +#if NCNN_ARM82 + if (ncnn::cpu_support_arm_asimdhp() && opt.use_fp16_arithmetic) + { + ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); + } +#endif + + if (opt.lightmode) + { + weight_data.release(); + } + + return 0; +} + +int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + const int num_input = weight_data_size / num_output; + + if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1) + { + // gemm + int h = bottom_blob.h; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt); + + return 0; + } + + // flatten + Mat bottom_blob_flattened = bottom_blob; + if (bottom_blob.dims != 1) + { + Option opt_flatten = opt; + opt_flatten.blob_allocator = opt.workspace_allocator; + + flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); + } + + size_t elemsize = bottom_blob_flattened.elemsize; + int elempack = bottom_blob_flattened.elempack; + + int out_elempack = 1; + if (opt.use_packing_layout) + { + out_elempack = num_output % 4 == 0 ? 4 : 1; + } + size_t out_elemsize = elemsize / elempack * out_elempack; + + top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + if (out_elempack == 4) + { + innerproduct_fp16s_pack4_neon(bottom_blob_flattened, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt); + } + + if (out_elempack == 1) + { + innerproduct_fp16s_neon(bottom_blob_flattened, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt); + } + + return 0; +} +#endif // (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ + #if NCNN_BF16 int InnerProduct_arm::create_pipeline_bf16s(const Option& opt) { diff --git a/src/layer/arm/innerproduct_arm.h b/src/layer/arm/innerproduct_arm.h index b0e26b4ba..00c7bc804 100644 --- a/src/layer/arm/innerproduct_arm.h +++ b/src/layer/arm/innerproduct_arm.h @@ -32,9 +32,11 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; protected: -#if NCNN_ARM82 +#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__ int create_pipeline_fp16s(const Option& opt); int forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +#endif +#if NCNN_ARM82 int forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; #endif #if NCNN_BF16 diff --git a/src/layer/arm/innerproduct_arm_asimdhp.cpp b/src/layer/arm/innerproduct_arm_asimdhp.cpp index f57b3b69c..762a0935d 100644 --- a/src/layer/arm/innerproduct_arm_asimdhp.cpp +++ b/src/layer/arm/innerproduct_arm_asimdhp.cpp @@ -14,7 +14,7 @@ #include "innerproduct_arm.h" -#include "layer_type.h" +#include "cpu.h" #if __ARM_NEON #include @@ -25,346 +25,30 @@ namespace ncnn { -#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -int InnerProduct_arm::create_pipeline_fp16s(const Option& opt) -{ - const int num_input = weight_data_size / num_output; - - int out_elempack = 1; +#include "innerproduct_fp16s.h" +#include "innerproduct_gemm_fp16s.h" - if (opt.use_packing_layout) - { - out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - } - - // src = inch-outch - // dst = pb-inch-outch/pb - { - Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - - weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - __fp16* g0 = weight_data_tm.row<__fp16>(q / out_elempack); - - for (int p = 0; p < num_input; p++) - { - for (int j = 0; j < out_elempack; j++) - { - *g0++ = (__fp16)(weight_data_r2.row(q + j)[p]); - } - } - } - } - - ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); - - if (opt.lightmode) - { - weight_data.release(); - } - - return 0; +void innerproduct_fp16s_pack4_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ + innerproduct_fp16s_pack4_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); } -int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +void innerproduct_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) { - const int num_input = weight_data_size / num_output; - - if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1) - { - // gemm - int h = bottom_blob.h; - size_t elemsize = bottom_blob.elemsize; - int elempack = bottom_blob.elempack; - - top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - int num_output_elempack = 1; - if (opt.use_packing_layout) - { - num_output_elempack = num_output % 4 == 0 ? 4 : 1; - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int j = 0; j < h; j++) - { - if (elempack == 4 && num_output_elempack == 4) - { - __fp16* outptr = top_blob.row<__fp16>(j); - - for (int p = 0; p < num_output / num_output_elempack; p++) - { - const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; - const __fp16* m = bottom_blob.row(j); - - float32x4_t _sum0 = vdupq_n_f32(0.f); - float32x4_t _sum1 = vdupq_n_f32(0.f); - float32x4_t _sum2 = vdupq_n_f32(0.f); - float32x4_t _sum3 = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum0 = vdupq_n_f32(bias_data[p * 4 + 0]); - _sum1 = vdupq_n_f32(bias_data[p * 4 + 1]); - _sum2 = vdupq_n_f32(bias_data[p * 4 + 2]); - _sum3 = vdupq_n_f32(bias_data[p * 4 + 3]); - } - - for (int i = 0; i < num_input; i++) - { - float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); - float32x4_t _k = vcvt_f32_f16(vld1_f16(kptr)); - _sum0 = vfmaq_laneq_f32(_sum0, _val, _k, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _val, _k, 1); - _sum2 = vfmaq_laneq_f32(_sum2, _val, _k, 2); - _sum3 = vfmaq_laneq_f32(_sum3, _val, _k, 3); - - m += 4; - kptr += 4; - } - - _sum0 = activation_ps(_sum0, activation_type, activation_params); - _sum1 = activation_ps(_sum1, activation_type, activation_params); - _sum2 = activation_ps(_sum2, activation_type, activation_params); - _sum3 = activation_ps(_sum3, activation_type, activation_params); - - vst1_f16(outptr, vcvt_f16_f32(_sum0)); - vst1_f16(outptr + 4, vcvt_f16_f32(_sum1)); - vst1_f16(outptr + 8, vcvt_f16_f32(_sum2)); - vst1_f16(outptr + 12, vcvt_f16_f32(_sum3)); - outptr += 16; - } - } - - if (elempack == 1 && num_output_elempack == 4) - { - __fp16* outptr = top_blob.row<__fp16>(j); - - for (int p = 0; p < num_output / num_output_elempack; p++) - { - const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; - const __fp16* m = bottom_blob.row(j); - - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32((const float*)bias_data + p * 4); - } - - for (int i = 0; i < num_input; i++) - { - float32x4_t _val = vdupq_n_f32((float)m[0]); - float32x4_t _k = vcvt_f32_f16(vld1_f16(kptr)); - _sum = vfmaq_f32(_sum, _val, _k); - - m += 1; - kptr += 4; - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr, vcvt_f16_f32(_sum)); - outptr += 4; - } - } - - if (elempack == 4 && num_output_elempack == 1) - { - __fp16* outptr = top_blob.row<__fp16>(j); - - for (int p = 0; p < num_output; p++) - { - const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; - const __fp16* m = bottom_blob.row(j); - - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vdupq_n_f32(bias_data[p]); - } - - for (int i = 0; i < num_input; i++) - { - float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); - float32x4_t _k = vdupq_n_f32((float)kptr[0]); - _sum = vfmaq_f32(_sum, _val, _k); - - m += 4; - kptr += 1; - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr, vcvt_f16_f32(_sum)); - outptr += 4; - } - } - - if (elempack == 1 && num_output_elempack == 1) - { - __fp16* outptr = top_blob.row<__fp16>(j); - - for (int p = 0; p < num_output; p++) - { - const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; - const __fp16* m = bottom_blob.row(j); - - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - for (int i = 0; i < num_input; i++) - { - sum += (float)*m * (float)*kptr; - - m += 1; - kptr += 1; - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[0] = (__fp16)sum; - outptr += 1; - } - } - } - - return 0; - } - - // flatten - Mat bottom_blob_flattened = bottom_blob; - if (bottom_blob.dims != 1) - { - Option opt_flatten = opt; - opt_flatten.blob_allocator = opt.workspace_allocator; - - flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); - } - - size_t elemsize = bottom_blob_flattened.elemsize; - int elempack = bottom_blob_flattened.elempack; - - int out_elempack = opt.use_packing_layout && num_output % 4 == 0 ? 4 : 1; - size_t out_elemsize = elemsize / elempack * out_elempack; - - top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - if (out_elempack == 4) - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32(((const float*)bias_data) + p * 4); - } - - const __fp16* kptr = weight_data_tm.row(p); - - const __fp16* sptr = bottom_blob_flattened; - - int i = 0; - for (; i + 3 < num_input; i += 4) - { - float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr)); - - float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr)); - float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4)); - float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8)); - float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12)); - - _sum = vfmaq_laneq_f32(_sum, _w0, _val, 0); - _sum = vfmaq_laneq_f32(_sum, _w1, _val, 1); - _sum = vfmaq_laneq_f32(_sum, _w2, _val, 2); - _sum = vfmaq_laneq_f32(_sum, _w3, _val, 3); - - sptr += 4; - kptr += 16; - } - for (; i < num_input; i++) - { - float32x4_t _val = vdupq_n_f32((float)sptr[0]); - - float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); - - _sum = vfmaq_f32(_sum, _val, _w); - - sptr += 1; - kptr += 4; - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - __fp16* outptr = (__fp16*)top_blob; - vst1_f16(outptr + p * 4, vcvt_f16_f32(_sum)); - } - } - - if (out_elempack == 1) - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - float sum = 0.f; - - if (bias_term) - sum = bias_data[p]; - - const __fp16* kptr = weight_data_tm.row<__fp16>(p); - - const __fp16* sptr = bottom_blob_flattened; - - float32x4_t _sum = vdupq_n_f32(0.f); - int i = 0; - for (; i + 3 < num_input; i += 4) - { - float32x4_t _m = vcvt_f32_f16(vld1_f16(sptr)); - float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); - - _sum = vfmaq_f32(_sum, _m, _w); - - sptr += 4; - kptr += 4; - } - for (; i < num_input; i++) - { - float v = (float)(*sptr); - float k = (float)(*kptr); - - sum += v * k; - - sptr++; - kptr++; - } - - sum += vaddvq_f32(_sum); - - sum = activation_ss(sum, activation_type, activation_params); + innerproduct_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); +} - __fp16* outptr = (__fp16*)top_blob; - outptr[p] = (__fp16)sum; - } - } +void innerproduct_gemm_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ + innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); +} - return 0; +void innerproduct_transform_kernel_fp16s_neon_asimdhp(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt) +{ + innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt); } +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { const int num_input = weight_data_size / num_output; diff --git a/src/layer/arm/innerproduct_arm_vfpv4.cpp b/src/layer/arm/innerproduct_arm_vfpv4.cpp new file mode 100644 index 000000000..a6c7b8c8c --- /dev/null +++ b/src/layer/arm/innerproduct_arm_vfpv4.cpp @@ -0,0 +1,51 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "innerproduct_arm.h" + +#include "cpu.h" + +#if __ARM_NEON +#include +#endif // __ARM_NEON + +#include "arm_activation.h" +#include "arm_usability.h" + +namespace ncnn { + +#include "innerproduct_fp16s.h" +#include "innerproduct_gemm_fp16s.h" + +void innerproduct_fp16s_pack4_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ + innerproduct_fp16s_pack4_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); +} + +void innerproduct_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ + innerproduct_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); +} + +void innerproduct_gemm_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ + innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); +} + +void innerproduct_transform_kernel_fp16s_neon_vfpv4(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt) +{ + innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt); +} + +} // namespace ncnn diff --git a/src/layer/arm/innerproduct_fp16s.h b/src/layer/arm/innerproduct_fp16s.h new file mode 100644 index 000000000..49983980e --- /dev/null +++ b/src/layer/arm/innerproduct_fp16s.h @@ -0,0 +1,534 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC +void innerproduct_fp16s_pack4_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +void innerproduct_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +void innerproduct_transform_kernel_fp16s_neon_asimdhp(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt); +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) +void innerproduct_fp16s_pack4_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +void innerproduct_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +void innerproduct_transform_kernel_fp16s_neon_vfpv4(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt); +#endif + +static void innerproduct_fp16s_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (ncnn::cpu_support_arm_asimdhp()) + { + innerproduct_fp16s_pack4_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) + if (ncnn::cpu_support_arm_vfpv4()) + { + innerproduct_fp16s_pack4_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if (__ARM_FP & 2) + const int num_input = bottom_blob.w * bottom_blob.elempack; + const int num_output = top_blob.w; + + const float* bias_data_ptr = bias_data; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output; p++) + { + float32x4_t _sum0 = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum0 = vld1q_f32(bias_data_ptr + p * 4); + } + + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* sptr = bottom_blob; + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* sptr = bottom_blob; + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + int i = 0; + for (; i + 7 < num_input; i += 8) + { +#if __aarch64__ +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v1.8h}, [%0], #16 \n" + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [%1], #64 \n" + "fcvtl v0.4s, v1.4h \n" + "fcvtl2 v1.4s, v1.8h \n" + "fcvtl v2.4s, v6.4h \n" + "fcvtl2 v3.4s, v6.8h \n" + "fcvtl v4.4s, v7.4h \n" + "fcvtl2 v5.4s, v7.8h \n" + "fcvtl v6.4s, v8.4h \n" + "fcvtl2 v7.4s, v8.8h \n" + "fcvtl v8.4s, v9.4h \n" + "fcvtl2 v9.4s, v9.8h \n" + "fmla %2.4s, v2.4s, v0.s[0] \n" + "fmla %3.4s, v3.4s, v0.s[1] \n" + "fmla %4.4s, v4.4s, v0.s[2] \n" + "fmla %5.4s, v5.4s, v0.s[3] \n" + "fmla %2.4s, v6.4s, v1.s[0] \n" + "fmla %3.4s, v7.4s, v1.s[1] \n" + "fmla %4.4s, v8.4s, v1.s[2] \n" + "fmla %5.4s, v9.4s, v1.s[3] \n" + : "=r"(sptr), // %0 + "=r"(kptr), // %1 + "=w"(_sum0), // %2 + "=w"(_sum1), // %3 + "=w"(_sum2), // %4 + "=w"(_sum3) // %5 + : "0"(sptr), + "1"(kptr), + "2"(_sum0), + "3"(_sum1), + "4"(_sum2), + "5"(_sum3) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9"); +#else // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.4s, v1.4s}, [%0], #32 \n" + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [%1], #64 \n" + "fcvtl v2.4s, v6.4h \n" + "fcvtl2 v3.4s, v6.8h \n" + "fcvtl v4.4s, v7.4h \n" + "fcvtl2 v5.4s, v7.8h \n" + "fcvtl v6.4s, v8.4h \n" + "fcvtl2 v7.4s, v8.8h \n" + "fcvtl v8.4s, v9.4h \n" + "fcvtl2 v9.4s, v9.8h \n" + "fmla %2.4s, v2.4s, v0.s[0] \n" + "fmla %3.4s, v3.4s, v0.s[1] \n" + "fmla %4.4s, v4.4s, v0.s[2] \n" + "fmla %5.4s, v5.4s, v0.s[3] \n" + "fmla %2.4s, v6.4s, v1.s[0] \n" + "fmla %3.4s, v7.4s, v1.s[1] \n" + "fmla %4.4s, v8.4s, v1.s[2] \n" + "fmla %5.4s, v9.4s, v1.s[3] \n" + : "=r"(sptr), // %0 + "=r"(kptr), // %1 + "=w"(_sum0), // %2 + "=w"(_sum1), // %3 + "=w"(_sum2), // %4 + "=w"(_sum3) // %5 + : "0"(sptr), + "1"(kptr), + "2"(_sum0), + "3"(_sum1), + "4"(_sum2), + "5"(_sum3) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9"); +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#else // __aarch64__ + asm volatile( + "pld [%0, #256] \n" + "vld1.f32 {d0-d3}, [%0 :128]! \n" + "pld [%1, #512] \n" + "vldm %1!, {d12-d19} \n" + "vcvt.f32.f16 q2, d12 \n" + "vcvt.f32.f16 q3, d13 \n" + "vcvt.f32.f16 q4, d14 \n" + "vcvt.f32.f16 q5, d15 \n" + "vcvt.f32.f16 q6, d16 \n" + "vcvt.f32.f16 q7, d17 \n" + "vcvt.f32.f16 q8, d18 \n" + "vcvt.f32.f16 q9, d19 \n" + "vmla.f32 %q2, q2, d0[0] \n" + "vmla.f32 %q3, q3, d0[1] \n" + "vmla.f32 %q4, q4, d1[0] \n" + "vmla.f32 %q5, q5, d1[1] \n" + "vmla.f32 %q2, q6, d2[0] \n" + "vmla.f32 %q3, q7, d2[1] \n" + "vmla.f32 %q4, q8, d3[0] \n" + "vmla.f32 %q5, q9, d3[1] \n" + : "=r"(sptr), // %0 + "=r"(kptr), // %1 + "=w"(_sum0), // %2 + "=w"(_sum1), // %3 + "=w"(_sum2), // %4 + "=w"(_sum3) // %5 + : "0"(sptr), + "1"(kptr), + "2"(_sum0), + "3"(_sum1), + "4"(_sum2), + "5"(_sum3) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9"); +#endif // __aarch64__ + } + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr)); + float16x8_t _w01 = vld1q_f16(kptr); + float16x8_t _w23 = vld1q_f16(kptr + 8); + float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01)); + float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01)); + float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23)); + float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23)); +#else + float32x4_t _val = vld1q_f32(sptr); + uint16x8_t _w01 = vld1q_u16(kptr); + uint16x8_t _w23 = vld1q_u16(kptr + 8); + float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w01))); + float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w01))); + float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w23))); + float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w23))); +#endif + +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _w0, _val, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _w1, _val, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _w2, _val, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _w3, _val, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _w0, vget_low_f32(_val), 0); + _sum1 = vmlaq_lane_f32(_sum1, _w1, vget_low_f32(_val), 1); + _sum2 = vmlaq_lane_f32(_sum2, _w2, vget_high_f32(_val), 0); + _sum3 = vmlaq_lane_f32(_sum3, _w3, vget_high_f32(_val), 1); +#endif + + sptr += 4; + kptr += 16; + } + for (; i < num_input; i++) + { + float32x4_t _val = vdupq_n_f32((float)sptr[0]); +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + _sum0 = vfmaq_f32(_sum0, _val, _w); + + sptr += 1; + kptr += 4; + } + + _sum0 = vaddq_f32(_sum0, _sum1); + _sum2 = vaddq_f32(_sum2, _sum3); + _sum0 = vaddq_f32(_sum0, _sum2); + + _sum0 = activation_ps(_sum0, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = (__fp16*)top_blob; + vst1_f16(outptr + p * 4, vcvt_f16_f32(_sum0)); +#else + float* outptr = top_blob; + vst1q_f32(outptr + p * 4, _sum0); +#endif + } +#else // (__ARM_FP & 2) + (void)bottom_blob; + (void)top_blob; + (void)weight_data_fp16; + (void)bias_data; + (void)activation_type; + (void)activation_params; + (void)opt; +#endif // (__ARM_FP & 2) +} + +static void innerproduct_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (ncnn::cpu_support_arm_asimdhp()) + { + innerproduct_fp16s_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) + if (ncnn::cpu_support_arm_vfpv4()) + { + innerproduct_fp16s_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if (__ARM_FP & 2) + const int num_input = bottom_blob.w * bottom_blob.elempack; + const int num_output = top_blob.w; + + const float* bias_data_ptr = bias_data; + + int nn_num_output = num_output >> 2; + int remain_num_output_start = nn_num_output << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_num_output; pp++) + { + int p = pp * 4; + + float sums[4] = {0.0f}; + if (bias_data_ptr) + { + sums[0] = bias_data_ptr[p]; + sums[1] = bias_data_ptr[p + 1]; + sums[2] = bias_data_ptr[p + 2]; + sums[3] = bias_data_ptr[p + 3]; + } + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* sptr = bottom_blob; + const __fp16* kptr0 = weight_data_fp16.row(p); + const __fp16* kptr1 = weight_data_fp16.row(p + 1); + const __fp16* kptr2 = weight_data_fp16.row(p + 2); + const __fp16* kptr3 = weight_data_fp16.row(p + 3); +#else + const float* sptr = bottom_blob; + const unsigned short* kptr0 = weight_data_fp16.row(p); + const unsigned short* kptr1 = weight_data_fp16.row(p + 1); + const unsigned short* kptr2 = weight_data_fp16.row(p + 2); + const unsigned short* kptr3 = weight_data_fp16.row(p + 3); +#endif + + int i = 0; + + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr)); + float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr0)); + float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr1)); + float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr2)); + float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr3)); +#else + float32x4_t _val = vld1q_f32(sptr); + float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr0))); + float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr1))); + float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr2))); + float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr3))); +#endif + + _sum0 = vfmaq_f32(_sum0, _val, _w0); + _sum1 = vfmaq_f32(_sum1, _val, _w1); + _sum2 = vfmaq_f32(_sum2, _val, _w2); + _sum3 = vfmaq_f32(_sum3, _val, _w3); + + sptr += 4; + kptr0 += 4; + kptr1 += 4; + kptr2 += 4; + kptr3 += 4; + } + +#if __aarch64__ + sums[0] += vaddvq_f32(_sum0); + sums[1] += vaddvq_f32(_sum1); + sums[2] += vaddvq_f32(_sum2); + sums[3] += vaddvq_f32(_sum3); +#else + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); + float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1)); + float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2)); + float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3)); + float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); + float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); + sums[0] += vget_lane_f32(_sum01ss, 0); + sums[1] += vget_lane_f32(_sum01ss, 1); + sums[2] += vget_lane_f32(_sum23ss, 0); + sums[3] += vget_lane_f32(_sum23ss, 1); +#endif + + for (; i < num_input; i++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + sums[0] += (float)(*sptr) * (float)(*kptr0); + sums[1] += (float)(*sptr) * (float)(*kptr1); + sums[2] += (float)(*sptr) * (float)(*kptr2); + sums[3] += (float)(*sptr) * (float)(*kptr3); +#else + sums[0] += *sptr * float16_to_float32(*kptr0); + sums[1] += *sptr * float16_to_float32(*kptr1); + sums[2] += *sptr * float16_to_float32(*kptr2); + sums[3] += *sptr * float16_to_float32(*kptr3); +#endif + + sptr++; + kptr0++; + kptr1++; + kptr2++; + kptr3++; + } + + float32x4_t _sum = vld1q_f32(sums); + + _sum = activation_ps(_sum, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = (__fp16*)top_blob; + vst1_f16(outptr + p, vcvt_f16_f32(_sum)); +#else + float* outptr = top_blob; + vst1q_f32(outptr + p, _sum); +#endif + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_num_output_start; p < num_output; p++) + { + float sum = 0.f; + + if (bias_data_ptr) + sum = bias_data_ptr[p]; + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* sptr = bottom_blob; + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* sptr = bottom_blob; + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + int i = 0; + + float32x4_t _sum = vdupq_n_f32(0.f); + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _val = vld1q_f32(sptr); + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + _sum = vfmaq_f32(_sum, _val, _w); + + sptr += 4; + kptr += 4; + } + for (; i < num_input; i++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + sum += (float)(*sptr) * (float)(*kptr); +#else + sum += *sptr * float16_to_float32(*kptr); +#endif + sptr++; + kptr++; + } + +#if __aarch64__ + sum += vaddvq_f32(_sum); +#else + float32x2_t _sumss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum)); + _sumss = vpadd_f32(_sumss, _sumss); + sum += vget_lane_f32(_sumss, 0); +#endif // __aarch64__ + + sum = activation_ss(sum, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = (__fp16*)top_blob; + outptr[p] = (__fp16)sum; +#else + float* outptr = top_blob; + outptr[p] = sum; +#endif + } +#else // (__ARM_FP & 2) + (void)bottom_blob; + (void)top_blob; + (void)weight_data_fp16; + (void)bias_data; + (void)activation_type; + (void)activation_params; + (void)opt; +#endif // (__ARM_FP & 2) +} + +static void innerproduct_transform_kernel_fp16s_neon(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt) +{ +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (ncnn::cpu_support_arm_asimdhp()) + { + innerproduct_transform_kernel_fp16s_neon_asimdhp(weight_data, weight_data_tm, num_input, num_output, opt); + return; + } +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) + if (ncnn::cpu_support_arm_vfpv4()) + { + innerproduct_transform_kernel_fp16s_neon_vfpv4(weight_data, weight_data_tm, num_input, num_output, opt); + return; + } +#endif + +#if (__ARM_FP & 2) + int out_elempack = 1; + if (opt.use_packing_layout) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; +#else + out_elempack = num_output % 4 == 0 ? 4 : 1; +#endif + } + + Mat weight_data_fp16; + ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); + + // src = inch-outch + // dst = pb-inch-outch/pb + { + Mat weight_data_r2 = weight_data_fp16.reshape(num_input, num_output); + + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + unsigned short* g0 = weight_data_tm.row(q / out_elempack); + + for (int p = 0; p < num_input; p++) + { + for (int j = 0; j < out_elempack; j++) + { + *g0++ = weight_data_r2.row(q + j)[p]; + } + } + } + } +#else // (__ARM_FP & 2) + (void)weight_data; + (void)weight_data_tm; + (void)num_input; + (void)num_output; + (void)opt; +#endif // (__ARM_FP & 2) +} diff --git a/src/layer/arm/innerproduct_gemm_fp16s.h b/src/layer/arm/innerproduct_gemm_fp16s.h new file mode 100644 index 000000000..c176a6f18 --- /dev/null +++ b/src/layer/arm/innerproduct_gemm_fp16s.h @@ -0,0 +1,398 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC +void innerproduct_gemm_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) +void innerproduct_gemm_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt); +#endif + +static void innerproduct_gemm_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt) +{ +#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (ncnn::cpu_support_arm_asimdhp()) + { + innerproduct_gemm_fp16s_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2) + if (ncnn::cpu_support_arm_vfpv4()) + { + innerproduct_gemm_fp16s_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt); + return; + } +#endif + +#if (__ARM_FP & 2) + const int num_input = bottom_blob.w; + const int elempack = bottom_blob.elempack; + const int num_output = top_blob.w; + const int h = bottom_blob.h; + + const float* bias_data_ptr = bias_data; + + int num_output_elempack = 1; + if (opt.use_packing_layout) + { + num_output_elempack = num_output % 4 == 0 ? 4 : 1; + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < h; j++) + { + if (elempack == 4 && num_output_elempack == 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = top_blob.row<__fp16>(j); +#else + float* outptr = top_blob.row(j); +#endif + + for (int p = 0; p < num_output / num_output_elempack; p++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* m = bottom_blob.row(j); + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* m = bottom_blob.row(j); + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum0 = vdupq_n_f32(bias_data_ptr[p * 4 + 0]); + _sum1 = vdupq_n_f32(bias_data_ptr[p * 4 + 1]); + _sum2 = vdupq_n_f32(bias_data_ptr[p * 4 + 2]); + _sum3 = vdupq_n_f32(bias_data_ptr[p * 4 + 3]); + } + + int i = 0; + for (; i < num_input; i++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _val = vld1q_f32(m); + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _val, _w, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _val, _w, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _val, _w, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _val, _w, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _val, vget_low_f32(_w), 0); + _sum1 = vmlaq_lane_f32(_sum1, _val, vget_low_f32(_w), 1); + _sum2 = vmlaq_lane_f32(_sum2, _val, vget_high_f32(_w), 0); + _sum3 = vmlaq_lane_f32(_sum3, _val, vget_high_f32(_w), 1); +#endif + + m += 4; + kptr += 4; + } + + _sum0 = activation_ps(_sum0, activation_type, activation_params); + _sum1 = activation_ps(_sum1, activation_type, activation_params); + _sum2 = activation_ps(_sum2, activation_type, activation_params); + _sum3 = activation_ps(_sum3, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + vst1_f16(outptr, vcvt_f16_f32(_sum0)); + vst1_f16(outptr + 4, vcvt_f16_f32(_sum1)); + vst1_f16(outptr + 8, vcvt_f16_f32(_sum2)); + vst1_f16(outptr + 12, vcvt_f16_f32(_sum3)); +#else + vst1q_f32(outptr, _sum0); + vst1q_f32(outptr + 4, _sum1); + vst1q_f32(outptr + 8, _sum2); + vst1q_f32(outptr + 12, _sum3); +#endif + outptr += 16; + } + } + + if (elempack == 1 && num_output_elempack == 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = top_blob.row<__fp16>(j); +#else + float* outptr = top_blob.row(j); +#endif + + for (int p = 0; p < num_output / num_output_elempack; p++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* m = bottom_blob.row(j); + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* m = bottom_blob.row(j); + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + float32x4_t _sum0 = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum0 = vld1q_f32(bias_data_ptr + p * 4); + } + + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + + int i = 0; + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); + float16x8_t _w01 = vld1q_f16(kptr); + float16x8_t _w23 = vld1q_f16(kptr + 8); + float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01)); + float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01)); + float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23)); + float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23)); +#else + float32x4_t _val = vld1q_f32(m); + uint16x8_t _w01 = vld1q_u16(kptr); + uint16x8_t _w23 = vld1q_u16(kptr + 8); + float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w01))); + float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w01))); + float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w23))); + float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w23))); +#endif + +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _w0, _val, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _w1, _val, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _w2, _val, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _w3, _val, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _w0, vget_low_f32(_val), 0); + _sum1 = vmlaq_lane_f32(_sum1, _w1, vget_low_f32(_val), 1); + _sum2 = vmlaq_lane_f32(_sum2, _w2, vget_high_f32(_val), 0); + _sum3 = vmlaq_lane_f32(_sum3, _w3, vget_high_f32(_val), 1); +#endif + + m += 4; + kptr += 16; + } + for (; i < num_input; i++) + { + float32x4_t _val = vdupq_n_f32((float)m[0]); +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + _sum0 = vfmaq_f32(_sum0, _val, _w); + + m += 1; + kptr += 4; + } + + _sum0 = vaddq_f32(_sum0, _sum1); + _sum2 = vaddq_f32(_sum2, _sum3); + _sum0 = vaddq_f32(_sum0, _sum2); + + _sum0 = activation_ps(_sum0, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + vst1_f16(outptr, vcvt_f16_f32(_sum0)); +#else + vst1q_f32(outptr, _sum0); +#endif + outptr += 4; + } + } + + if (elempack == 4 && num_output_elempack == 1) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = top_blob.row<__fp16>(j); +#else + float* outptr = top_blob.row(j); +#endif + + for (int p = 0; p < num_output; p++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* m = bottom_blob.row(j); + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* m = bottom_blob.row(j); + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum0 = vdupq_n_f32(bias_data_ptr[p]); + } + + int i = 0; + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val0 = vcvt_f32_f16(vld1_f16(m)); + float32x4_t _val1 = vcvt_f32_f16(vld1_f16(m + 4)); + float32x4_t _val2 = vcvt_f32_f16(vld1_f16(m + 8)); + float32x4_t _val3 = vcvt_f32_f16(vld1_f16(m + 12)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _val0 = vld1q_f32(m); + float32x4_t _val1 = vld1q_f32(m + 4); + float32x4_t _val2 = vld1q_f32(m + 8); + float32x4_t _val3 = vld1q_f32(m + 12); + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _val0, _w, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _val1, _w, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _val2, _w, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _val3, _w, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _val0, vget_low_f32(_w), 0); + _sum1 = vmlaq_lane_f32(_sum1, _val1, vget_low_f32(_w), 1); + _sum2 = vmlaq_lane_f32(_sum2, _val2, vget_high_f32(_w), 0); + _sum3 = vmlaq_lane_f32(_sum3, _val3, vget_high_f32(_w), 1); +#endif + + m += 16; + kptr += 4; + } + for (; i < num_input; i++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); + float32x4_t _k = vdupq_n_f32((float)(kptr[0])); +#else + float32x4_t _val = vld1q_f32(m); + float32x4_t _k = vdupq_n_f32(float16_to_float32(kptr[0])); +#endif + _sum0 = vfmaq_f32(_sum0, _val, _k); + + m += 4; + kptr += 1; + } + + _sum0 = vaddq_f32(_sum0, _sum1); + _sum2 = vaddq_f32(_sum2, _sum3); + _sum0 = vaddq_f32(_sum0, _sum2); + + _sum0 = activation_ps(_sum0, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + vst1_f16(outptr, vcvt_f16_f32(_sum0)); +#else + vst1q_f32(outptr, _sum0); +#endif + outptr += 4; + } + } + + if (elempack == 1 && num_output_elempack == 1) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + __fp16* outptr = top_blob.row<__fp16>(j); +#else + float* outptr = top_blob.row(j); +#endif + + for (int p = 0; p < num_output; p++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + const __fp16* m = bottom_blob.row(j); + const __fp16* kptr = weight_data_fp16.row(p); +#else + const float* m = bottom_blob.row(j); + const unsigned short* kptr = weight_data_fp16.row(p); +#endif + + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + int i = 0; + float32x4_t _sum = vdupq_n_f32(0.f); + for (; i + 3 < num_input; i += 4) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + float32x4_t _val = vcvt_f32_f16(vld1_f16(m)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); +#else + float32x4_t _val = vld1q_f32(m); + float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr))); +#endif + + _sum = vfmaq_f32(_sum, _val, _w); + + m += 4; + kptr += 4; + } +#if __aarch64__ + sum += vaddvq_f32(_sum); +#else + float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum)); + _ss = vpadd_f32(_ss, _ss); + sum += vget_lane_f32(_ss, 0); +#endif + for (; i < num_input; i++) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + sum += (float)(*m++) * (float)(*kptr++); +#else + sum += *m++ * float16_to_float32(*kptr++); +#endif + } + + sum = activation_ss(sum, activation_type, activation_params); + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + outptr[0] = (__fp16)sum; +#else + outptr[0] = sum; +#endif + outptr += 1; + } + } + } +#else // (__ARM_FP & 2) + (void)bottom_blob; + (void)top_blob; + (void)weight_data_fp16; + (void)bias_data; + (void)activation_type; + (void)activation_params; + (void)opt; +#endif // (__ARM_FP & 2) +} diff --git a/tests/testutil.h b/tests/testutil.h index b99493e55..f64379f4b 100644 --- a/tests/testutil.h +++ b/tests/testutil.h @@ -1240,7 +1240,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec opts[4].use_fp16_packed = true; opts[4].use_fp16_storage = true; opts[4].use_fp16_arithmetic = true; - opts[4].use_bf16_storage = true; + opts[4].use_bf16_storage = false; opts[4].use_shader_pack8 = true; opts[4].use_image_storage = true; @@ -1388,7 +1388,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec opts[4].use_fp16_packed = true; opts[4].use_fp16_storage = true; opts[4].use_fp16_arithmetic = true; - opts[4].use_bf16_storage = true; + opts[4].use_bf16_storage = false; opts[4].use_shader_pack8 = true; opts[4].use_image_storage = true;