diff --git a/src/layer/arm/cast_arm.cpp b/src/layer/arm/cast_arm.cpp index fc6fa17d7..8f3d90368 100644 --- a/src/layer/arm/cast_arm.cpp +++ b/src/layer/arm/cast_arm.cpp @@ -26,6 +26,9 @@ Cast_arm::Cast_arm() { #if __ARM_NEON support_packing = true; +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + support_fp16_storage = true; +#endif #endif // __ARM_NEON support_bf16_storage = true; diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index 38b447ada..7feda6303 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -1043,7 +1043,6 @@ int Convolution_arm::create_pipeline_fp16s(const Option& opt) int elempack = (support_packing && opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1; int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; -#if __ARM_NEON // pack4 if (elempack == 4 && out_elempack == 4) { @@ -1195,7 +1194,6 @@ int Convolution_arm::create_pipeline_fp16s(const Option& opt) } } } -#endif // __ARM_NEON // pack1 if (elempack == 1 && out_elempack == 1) @@ -1267,7 +1265,6 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const } } -#if __ARM_NEON if (elempack == 4 && out_elempack == 4) { { @@ -1424,7 +1421,6 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const } } } -#endif // __ARM_NEON if (elempack == 1 && out_elempack == 1) { @@ -1464,32 +1460,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const kptr += maxk; } - if (activation_type == 1) - { - sum = std::max(sum, 0.f); - } - else if (activation_type == 2) - { - float slope = activation_params[0]; - sum = sum > 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; - } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } - else if (activation_type == 5) - { - sum = static_cast(sum * tanh(log(exp(sum) + 1.f))); - } + sum = activation_ss(sum, activation_type, activation_params); outptr[j] = (__fp16)sum; } @@ -1560,7 +1531,6 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } } -#if __ARM_NEON if (elempack == 4 && out_elempack == 4) { { @@ -1717,7 +1687,6 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } } } -#endif // __ARM_NEON if (elempack == 1 && out_elempack == 1) { @@ -1757,32 +1726,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const kptr += maxk; } - if (activation_type == 1) - { - sum = std::max(sum, 0.f); - } - else if (activation_type == 2) - { - float slope = activation_params[0]; - sum = sum > 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; - } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } - else if (activation_type == 5) - { - sum = static_cast(sum * tanh(log(exp(sum) + 1.f))); - } + sum = activation_ss(sum, activation_type, activation_params); outptr[j] = sum; } diff --git a/src/layer/arm/convolutiondepthwise_arm.cpp b/src/layer/arm/convolutiondepthwise_arm.cpp index 1d80d3eab..59107d3f5 100644 --- a/src/layer/arm/convolutiondepthwise_arm.cpp +++ b/src/layer/arm/convolutiondepthwise_arm.cpp @@ -40,6 +40,9 @@ ConvolutionDepthWise_arm::ConvolutionDepthWise_arm() { #if __ARM_NEON support_packing = true; +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + support_fp16_storage = true; +#endif #endif // __ARM_NEON support_bf16_storage = true; @@ -116,43 +119,79 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) return 0; } } - else + + int elempack = (support_packing && opt.use_packing_layout && channels % 4 == 0) ? 4 : 1; + +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (opt.use_fp16_storage) { - int elempack = (support_packing && opt.use_packing_layout && channels % 4 == 0) ? 4 : 1; + if (elempack == 4) + { + Mat weight_data_r2 = weight_data.reshape(maxk, group); + convert_packing(weight_data_r2, weight_data_pack4, 4); + + ncnn::cast_float32_to_float16(weight_data_pack4, weight_data_pack4_fp16, opt); + } + + if (elempack == 1) + { + ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); + } + + ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); + + return 0; + } +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (opt.use_bf16_storage) + { #if __ARM_NEON - // pack4 if (elempack == 4) { Mat weight_data_r2 = weight_data.reshape(maxk, group); convert_packing(weight_data_r2, weight_data_pack4, 4); ncnn::cast_float32_to_bfloat16(weight_data_pack4, weight_data_pack4_bf16, opt); - - return 0; } #endif // __ARM_NEON if (elempack == 1) { ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt); + } - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) - { - return 0; - } - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) - { - return 0; - } - if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) - { - return 0; - } - if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) - { - return 0; - } + return 0; + } + +#if __ARM_NEON + // pack4 + if (elempack == 4) + { + Mat weight_data_r2 = weight_data.reshape(maxk, group); + convert_packing(weight_data_r2, weight_data_pack4, 4); + + return 0; + } +#endif // __ARM_NEON + + if (elempack == 1) + { + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + return 0; + } + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + return 0; + } + if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + return 0; + } + if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + return 0; } } } @@ -269,6 +308,16 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con return forward_int8_arm(bottom_blob, top_blob, opt); } +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (opt.use_fp16_storage) + { + if (opt.use_fp16_arithmetic) + return forward_fp16sa(bottom_blob, top_blob, opt); + else + return forward_fp16s(bottom_blob, top_blob, opt); + } +#endif + if (opt.use_bf16_storage) return forward_bf16s(bottom_blob, top_blob, opt); @@ -512,6 +561,451 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con return 0; } +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +int ConvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + Mat bottom_blob_bordered; + make_padding(bottom_blob, bottom_blob_bordered, opt); + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; + size_t out_elemsize = elemsize / elempack * out_elempack; + + top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // depth-wise + if (channels * elempack == group && group == num_output) + { + if (elempack == 4) + { + { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int g = 0; g < channels; g++) + { + __fp16* outptr = top_blob.channel(g); + const __fp16* kptr = (const __fp16*)weight_data_pack4_fp16 + maxk * g * 4; + const Mat m = bottom_blob_bordered.channel(g); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_term) + { + _sum = vld1q_f32(((const float*)bias_data) + g * 4); + } + + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr + k * 4)); + _sum = vfmaq_f32(_sum, _val, _w); + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum)); + } + + outptr += outw * 4; + } + } + } + + return 0; + } + + if (elempack == 1) + { + { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int g = 0; g < group; g++) + { + __fp16* outptr = top_blob.channel(g); + const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const Mat m = bottom_blob_bordered.channel(g); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_term) + sum = bias_data[g]; + + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float val = (float)sptr[space_ofs[k]]; + float w = (float)kptr[k]; + sum += val * w; + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = (__fp16)sum; + } + + outptr += outw; + } + } + } + } + + return 0; + } + + // group convolution + const int channels_g = channels * elempack / group; + const int num_output_g = num_output / group; + + int g_elempack = (support_packing && opt.use_packing_layout && channels_g % 4 == 0) ? 4 : 1; + int out_g_elempack = (support_packing && opt.use_packing_layout && num_output_g % 4 == 0) ? 4 : 1; + + // unpacking + Mat bottom_blob_bordered_unpacked = bottom_blob_bordered; + if (elempack == 4 && g_elempack == 1) + { + Option opt_p = opt; + opt_p.blob_allocator = opt.workspace_allocator; + convert_packing(bottom_blob_bordered, bottom_blob_bordered_unpacked, 1, opt_p); + } + + Mat top_blob_unpacked = top_blob; + if (out_g_elempack == 1 && out_elempack == 4) + { + top_blob_unpacked.create(outw, outh, num_output, out_elemsize / out_elempack, 1, opt.workspace_allocator); + if (top_blob_unpacked.empty()) + return -100; + } + + for (int g = 0; g < group; g++) + { + const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack); + Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack); + + const ncnn::Layer* op = group_ops[g]; + + Option opt_g = opt; + opt_g.blob_allocator = top_blob_unpacked.allocator; + + // forward + op->forward(bottom_blob_bordered_g, top_blob_g, opt_g); + } + + // packing + if (out_g_elempack == 1 && out_elempack == 4) + { + convert_packing(top_blob_unpacked, top_blob, 4, opt); + } + else + { + top_blob = top_blob_unpacked; + } + + return 0; +} + +int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + Mat bottom_blob_bordered; + make_padding(bottom_blob, bottom_blob_bordered, opt); + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; + size_t out_elemsize = elemsize / elempack * out_elempack; + + top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // depth-wise + if (channels * elempack == group && group == num_output) + { + if (elempack == 4) + { + { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int g = 0; g < channels; g++) + { + __fp16* outptr = top_blob.channel(g); + const __fp16* kptr = (const __fp16*)weight_data_pack4_fp16 + maxk * g * 4; + const Mat m = bottom_blob_bordered.channel(g); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x4_t _sum = vdup_n_f16((__fp16)0.f); + + if (bias_term) + { + _sum = vld1_f16(((const __fp16*)bias_data_fp16) + g * 4); + } + + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); + float16x4_t _w = vld1_f16(kptr + k * 4); + _sum = vfma_f16(_sum, _val, _w); + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } + } + + return 0; + } + + if (elempack == 1) + { + { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int g = 0; g < group; g++) + { + __fp16* outptr = top_blob.channel(g); + const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const Mat m = bottom_blob_bordered.channel(g); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_term) + sum = bias_data[g]; + + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + __fp16 val = sptr[space_ofs[k]]; + __fp16 w = kptr[k]; + sum += val * w; + } + + if (activation_type == 1) + { + sum = std::max(sum, 0.f); + } + else if (activation_type == 2) + { + float slope = activation_params[0]; + sum = sum > 0.f ? sum : sum * slope; + } + else if (activation_type == 3) + { + float min = activation_params[0]; + float max = activation_params[1]; + if (sum < min) + sum = min; + if (sum > max) + sum = max; + } + else if (activation_type == 4) + { + sum = static_cast(1.f / (1.f + exp(-sum))); + } + else if (activation_type == 5) + { + sum = static_cast(sum * tanh(log(exp(sum) + 1.f))); + } + + outptr[j] = (__fp16)sum; + } + + outptr += outw; + } + } + } + } + + return 0; + } + + // group convolution + const int channels_g = channels * elempack / group; + const int num_output_g = num_output / group; + + int g_elempack = (support_packing && opt.use_packing_layout && channels_g % 4 == 0) ? 4 : 1; + int out_g_elempack = (support_packing && opt.use_packing_layout && num_output_g % 4 == 0) ? 4 : 1; + + // unpacking + Mat bottom_blob_bordered_unpacked = bottom_blob_bordered; + if (elempack == 4 && g_elempack == 1) + { + Option opt_p = opt; + opt_p.blob_allocator = opt.workspace_allocator; + convert_packing(bottom_blob_bordered, bottom_blob_bordered_unpacked, 1, opt_p); + } + + Mat top_blob_unpacked = top_blob; + if (out_g_elempack == 1 && out_elempack == 4) + { + top_blob_unpacked.create(outw, outh, num_output, out_elemsize / out_elempack, 1, opt.workspace_allocator); + if (top_blob_unpacked.empty()) + return -100; + } + + for (int g = 0; g < group; g++) + { + const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack); + Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack); + + const ncnn::Layer* op = group_ops[g]; + + Option opt_g = opt; + opt_g.blob_allocator = top_blob_unpacked.allocator; + + // forward + op->forward(bottom_blob_bordered_g, top_blob_g, opt_g); + } + + // packing + if (out_g_elempack == 1 && out_elempack == 4) + { + convert_packing(top_blob_unpacked, top_blob, 4, opt); + } + else + { + top_blob = top_blob_unpacked; + } + + return 0; +} +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { int w = bottom_blob.w; diff --git a/src/layer/arm/convolutiondepthwise_arm.h b/src/layer/arm/convolutiondepthwise_arm.h index 2ca6bc053..3138a3be4 100644 --- a/src/layer/arm/convolutiondepthwise_arm.h +++ b/src/layer/arm/convolutiondepthwise_arm.h @@ -30,6 +30,10 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; protected: +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + int forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + int forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +#endif int forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; int forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; @@ -40,6 +44,11 @@ public: // packing Mat weight_data_pack4; + // fp16 + Mat weight_data_fp16; + Mat weight_data_pack4_fp16; + Mat bias_data_fp16; + // bf16 Mat weight_data_bf16; Mat weight_data_pack4_bf16;