From bc3822acc37ecc23845f83668f2cc237c5da8153 Mon Sep 17 00:00:00 2001 From: nihuini Date: Thu, 30 Jul 2020 16:49:06 +0800 Subject: [PATCH] convolution flatten arm fp16sa pack8 --- src/layer/arm/convolution_arm.cpp | 471 ++++++++++++++++++++--------- src/layer/arm/convolution_arm.h | 3 - src/layer/arm/flatten_arm.cpp | 284 ++++++++++++----- src/layer/arm/innerproduct_arm.cpp | 2 +- src/layer/arm/neon_activation.h | 9 + 5 files changed, 542 insertions(+), 227 deletions(-) diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index 2f52a64f1..2485a0e73 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -1042,169 +1042,48 @@ int Convolution_arm::create_pipeline_fp16s(const Option& opt) const int maxk = kernel_w * kernel_h; const int num_input = weight_data_size / maxk / num_output; - int elempack = (support_packing && opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1; - int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; + int elempack = 1; + int out_elempack = 1; - // pack4 - if (elempack == 4 && out_elempack == 4) + if (opt.use_packing_layout) { - { - // src = kw-kh-inch-outch - // dst = 4b-4a-kw-kh-inch/4a-outch/4b - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4_fp16.create(maxk, num_input / 4, num_output / 4, (size_t)2 * 16, 16); - - for (int q = 0; q + 3 < num_output; q += 4) - { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack4_fp16.channel(q / 4); - - for (int p = 0; p + 3 < num_input; p += 4) - { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - const float* k10 = k1.row(p); - const float* k11 = k1.row(p + 1); - const float* k12 = k1.row(p + 2); - const float* k13 = k1.row(p + 3); - - const float* k20 = k2.row(p); - const float* k21 = k2.row(p + 1); - const float* k22 = k2.row(p + 2); - const float* k23 = k2.row(p + 3); - - const float* k30 = k3.row(p); - const float* k31 = k3.row(p + 1); - const float* k32 = k3.row(p + 2); - const float* k33 = k3.row(p + 3); - - __fp16* g00 = g0.row<__fp16>(p / 4); - - for (int k = 0; k < maxk; k++) - { - g00[0] = k00[k]; - g00[1] = k10[k]; - g00[2] = k20[k]; - g00[3] = k30[k]; - - g00[4] = k01[k]; - g00[5] = k11[k]; - g00[6] = k21[k]; - g00[7] = k31[k]; - - g00[8] = k02[k]; - g00[9] = k12[k]; - g00[10] = k22[k]; - g00[11] = k32[k]; - - g00[12] = k03[k]; - g00[13] = k13[k]; - g00[14] = k23[k]; - g00[15] = k33[k]; - - g00 += 16; - } - } - } - } + elempack = opt.use_fp16_arithmetic && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1; + out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; } - // pack1to4 - if (elempack == 1 && out_elempack == 4) + // src = kw-kh-inch-outch + // dst = pb-pa-kw-kh-inch/pa-outch/pb { - // src = kw-kh-inch-outch - // dst = 4b-kw-kh-inch-outch/4b - { - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack1to4_fp16.create(maxk, num_input, num_output / 4, (size_t)2 * 4, 4); - - for (int q = 0; q + 3 < num_output; q += 4) - { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack1to4_fp16.channel(q / 4); - - for (int p = 0; p < num_input; p++) - { - const float* k00 = k0.row(p); - const float* k10 = k1.row(p); - const float* k20 = k2.row(p); - const float* k30 = k3.row(p); - - __fp16* g00 = g0.row<__fp16>(p); - - for (int k = 0; k < maxk; k++) - { - g00[0] = k00[k]; - g00[1] = k10[k]; - g00[2] = k20[k]; - g00[3] = k30[k]; + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - g00 += 4; - } - } - } - } - } + weight_data_fp16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack); - // pack4to1 - if (elempack == 4 && out_elempack == 1) - { + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - // src = kw-kh-inch-outch - // dst = 4a-kw-kh-inch/4a-outch - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4to1_fp16.create(maxk, num_input / 4, num_output, (size_t)2 * 4, 4); + Mat g0 = weight_data_fp16.channel(q / out_elempack); - for (int q = 0; q < num_output; q++) + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { - const Mat k0 = weight_data_r2.channel(q); - Mat g0 = weight_data_pack4to1_fp16.channel(q); + __fp16* g00 = g0.row<__fp16>(p / elempack); - for (int p = 0; p + 3 < num_input; p += 4) + for (int k = 0; k < maxk; k++) { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - __fp16* g00 = g0.row<__fp16>(p / 4); - - for (int k = 0; k < maxk; k++) + for (int i = 0; i < elempack; i++) { - g00[0] = k00[k]; - g00[1] = k01[k]; - g00[2] = k02[k]; - g00[3] = k03[k]; + for (int j = 0; j < out_elempack; j++) + { + const float* k00 = weight_data_r2.channel(q + j).row(p + i); - g00 += 4; + g00[0] = (__fp16)k00[k]; + + g00++; + } } } } } } - // pack1 - if (elempack == 1 && out_elempack == 1) - { - { - ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); - } - } - ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); return 0; @@ -1287,7 +1166,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const _sum = vld1q_f32((const float*)bias_data + p * 4); } - const __fp16* kptr = weight_data_pack4_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1344,7 +1223,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const _sum = vld1q_f32((const float*)bias_data + p * 4); } - const __fp16* kptr = weight_data_pack1to4_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1393,7 +1272,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const sum = bias_data[p]; } - const __fp16* kptr = weight_data_pack4to1_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1499,7 +1378,11 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; - int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; + int out_elempack = 1; + if (opt.use_packing_layout) + { + out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; + } size_t out_elemsize = elemsize / elempack * out_elempack; top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); @@ -1533,6 +1416,294 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } } + if (elempack == 8 && out_elempack == 8) + { + { + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output / out_elempack; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_term) + { + _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob_bordered.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + + float16x8_t _w0 = vld1q_f16(kptr); + float16x8_t _w1 = vld1q_f16(kptr + 8); + float16x8_t _w2 = vld1q_f16(kptr + 16); + float16x8_t _w3 = vld1q_f16(kptr + 24); + float16x8_t _w4 = vld1q_f16(kptr + 32); + float16x8_t _w5 = vld1q_f16(kptr + 40); + float16x8_t _w6 = vld1q_f16(kptr + 48); + float16x8_t _w7 = vld1q_f16(kptr + 56); + + _sum = vfmaq_laneq_f16(_sum, _w0, _val, 0); + _sum = vfmaq_laneq_f16(_sum, _w1, _val, 1); + _sum = vfmaq_laneq_f16(_sum, _w2, _val, 2); + _sum = vfmaq_laneq_f16(_sum, _w3, _val, 3); + _sum = vfmaq_laneq_f16(_sum, _w4, _val, 4); + _sum = vfmaq_laneq_f16(_sum, _w5, _val, 5); + _sum = vfmaq_laneq_f16(_sum, _w6, _val, 6); + _sum = vfmaq_laneq_f16(_sum, _w7, _val, 7); + + kptr += 64; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } + } + } + + if (elempack == 1 && out_elempack == 8) + { + { + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output / out_elempack; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_term) + { + _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob_bordered.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vdupq_n_f16(sptr[space_ofs[k]]); + float16x8_t _w = vld1q_f16(kptr); + _sum = vfmaq_f16(_sum, _val, _w); + + kptr += 8; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } + } + } + + if (elempack == 4 && out_elempack == 8) + { + { + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output / out_elempack; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_term) + { + _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob_bordered.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); + + float16x8_t _w0 = vld1q_f16(kptr); + float16x8_t _w1 = vld1q_f16(kptr + 8); + float16x8_t _w2 = vld1q_f16(kptr + 16); + float16x8_t _w3 = vld1q_f16(kptr + 24); + + _sum = vfmaq_lane_f16(_sum, _w0, _val, 0); + _sum = vfmaq_lane_f16(_sum, _w1, _val, 1); + _sum = vfmaq_lane_f16(_sum, _w2, _val, 2); + _sum = vfmaq_lane_f16(_sum, _w3, _val, 3); + + kptr += 32; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } + } + } + + if (elempack == 8 && out_elempack == 1) + { + { + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_term) + { + sum = bias_data[p]; + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob_bordered.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + float16x8_t _w = vld1q_f16(kptr); + float16x8_t _s8 = vmulq_f16(_val, _w); + + float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8)); + sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot + + kptr += 8; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = sum; + } + + outptr += outw; + } + } + } + } + + if (elempack == 8 && out_elempack == 4) + { + { + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output / out_elempack; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x4_t _sum = vdup_n_f16((__fp16)0.f); + + if (bias_term) + { + _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob_bordered.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + + float16x4_t _w0 = vld1_f16(kptr); + float16x4_t _w1 = vld1_f16(kptr + 4); + float16x4_t _w2 = vld1_f16(kptr + 8); + float16x4_t _w3 = vld1_f16(kptr + 12); + float16x4_t _w4 = vld1_f16(kptr + 16); + float16x4_t _w5 = vld1_f16(kptr + 20); + float16x4_t _w6 = vld1_f16(kptr + 24); + float16x4_t _w7 = vld1_f16(kptr + 28); + + _sum = vfma_laneq_f16(_sum, _w0, _val, 0); + _sum = vfma_laneq_f16(_sum, _w1, _val, 1); + _sum = vfma_laneq_f16(_sum, _w2, _val, 2); + _sum = vfma_laneq_f16(_sum, _w3, _val, 3); + _sum = vfma_laneq_f16(_sum, _w4, _val, 4); + _sum = vfma_laneq_f16(_sum, _w5, _val, 5); + _sum = vfma_laneq_f16(_sum, _w6, _val, 6); + _sum = vfma_laneq_f16(_sum, _w7, _val, 7); + + kptr += 32; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } + } + } + if (elempack == 4 && out_elempack == 4) { { @@ -1553,7 +1724,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); } - const __fp16* kptr = weight_data_pack4_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1610,7 +1781,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); } - const __fp16* kptr = weight_data_pack1to4_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1659,7 +1830,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const sum = bias_data[p]; } - const __fp16* kptr = weight_data_pack4to1_fp16.channel(p); + const __fp16* kptr = weight_data_fp16.channel(p); // channels for (int q = 0; q < channels; q++) diff --git a/src/layer/arm/convolution_arm.h b/src/layer/arm/convolution_arm.h index 273c20f85..0a3e7b8ea 100644 --- a/src/layer/arm/convolution_arm.h +++ b/src/layer/arm/convolution_arm.h @@ -59,9 +59,6 @@ public: Mat weight_data_pack4to1; // fp16 - Mat weight_data_pack4_fp16; - Mat weight_data_pack1to4_fp16; - Mat weight_data_pack4to1_fp16; Mat weight_data_fp16; Mat bias_data_fp16; diff --git a/src/layer/arm/flatten_arm.cpp b/src/layer/arm/flatten_arm.cpp index ef34b5c90..714a09b5b 100644 --- a/src/layer/arm/flatten_arm.cpp +++ b/src/layer/arm/flatten_arm.cpp @@ -52,38 +52,46 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op return 0; } -#if __ARM_NEON + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + int size = w * h; + + int total = size * channels * elempack; + + int out_elempack = 1; if (opt.use_packing_layout) { - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; - int elempack = bottom_blob.elempack; - int size = w * h; - - int total = size * channels * elempack; + out_elempack = total % 4 == 0 ? 4 : 1; + } + size_t out_elemsize = elemsize / elempack * out_elempack; - int out_elempack = total % 4 == 0 ? 4 : 1; - size_t out_elemsize = elemsize / elempack * out_elempack; + if (out_elempack == 1) + { + return Flatten::forward(bottom_blob, top_blob, opt); + } - if (dims == 2 && elempack == 1) - { - top_blob = bottom_blob; - top_blob.dims = 1; - top_blob.w = total / out_elempack; - top_blob.h = 1; - top_blob.cstep = top_blob.w; - top_blob.elemsize = out_elemsize; - top_blob.elempack = out_elempack; - return 0; - } + if (dims == 2 && elempack == 1) // out_elempack == 4 + { + top_blob = bottom_blob; + top_blob.dims = 1; + top_blob.w = total / out_elempack; + top_blob.h = 1; + top_blob.cstep = top_blob.w; + top_blob.elemsize = out_elemsize; + top_blob.elempack = out_elempack; + return 0; + } - top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); - if (top_blob.empty()) - return -100; + top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; - if (dims == 2 && elempack == 4) + if (dims == 2) + { + if (elempack == 4) // out_elempack == 4 { #pragma omp parallel for num_threads(opt.num_threads) for (int i = 0; i < h; i++) @@ -95,6 +103,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op float* outptr3 = (float*)top_blob + w * (i * 4 + 3); int j = 0; +#if __ARM_NEON for (; j + 3 < w; j += 4) { float32x4x4_t _v4 = vld4q_f32(ptr); @@ -109,6 +118,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op outptr2 += 4; outptr3 += 4; } +#endif for (; j < w; j++) { *outptr0++ = ptr[0]; @@ -119,11 +129,12 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op ptr += 4; } } - - return 0; } + } - if (dims == 3 && elempack == 4) + if (dims == 3) + { + if (elempack == 4) // out_elempack == 4 { #pragma omp parallel for num_threads(opt.num_threads) for (int q = 0; q < channels; q++) @@ -135,6 +146,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op float* outptr3 = (float*)top_blob + size * (q * 4 + 3); int i = 0; +#if __ARM_NEON for (; i + 3 < size; i += 4) { float32x4x4_t _v4 = vld4q_f32(ptr); @@ -149,6 +161,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op outptr2 += 4; outptr3 += 4; } +#endif for (; i < size; i++) { *outptr0++ = ptr[0]; @@ -159,11 +172,9 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op ptr += 4; } } - - return 0; } - if (dims == 3 && elempack == 1 && out_elempack == 4) + if (elempack == 1) // out_elempack == 4 { #pragma omp parallel for num_threads(opt.num_threads) for (int q = 0; q < channels; q++) @@ -172,6 +183,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op float* outptr = (float*)top_blob + size * q; int i = 0; +#if __ARM_NEON for (; i + 3 < size; i += 4) { float32x4_t _v = vld1q_f32(ptr); @@ -179,19 +191,16 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op ptr += 4; outptr += 4; } +#endif for (; i < size; i++) { *outptr++ = *ptr++; } } - - return 0; } + } - } // opt.use_packing_layout -#endif // __ARM_NEON - - return Flatten::forward(bottom_blob, top_blob, opt); + return 0; } int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const @@ -204,38 +213,106 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons return 0; } -#if __ARM_NEON + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + int size = w * h; + + int total = size * channels * elempack; + + int out_elempack = 1; if (opt.use_packing_layout) { - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; - int elempack = bottom_blob.elempack; - int size = w * h; + out_elempack = opt.use_fp16_arithmetic && total % 8 == 0 ? 8 : total % 4 == 0 ? 4 : 1; + } + size_t out_elemsize = elemsize / elempack * out_elempack; - int total = size * channels * elempack; + if (out_elempack == 1) + { + return Flatten::forward(bottom_blob, top_blob, opt); + } - int out_elempack = total % 4 == 0 ? 4 : 1; - size_t out_elemsize = elemsize / elempack * out_elempack; + if (dims == 2 && elempack == 1) // out_elempack == 4 || out_elempack == 8 + { + top_blob = bottom_blob; + top_blob.dims = 1; + top_blob.w = total / out_elempack; + top_blob.h = 1; + top_blob.cstep = top_blob.w; + top_blob.elemsize = out_elemsize; + top_blob.elempack = out_elempack; + return 0; + } - if (dims == 2 && elempack == 1) + top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + if (dims == 2) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (elempack == 8) // out_elempack == 8 { - top_blob = bottom_blob; - top_blob.dims = 1; - top_blob.w = total / out_elempack; - top_blob.h = 1; - top_blob.cstep = top_blob.w; - top_blob.elemsize = out_elemsize; - top_blob.elempack = out_elempack; - return 0; - } + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < h; i++) + { + const __fp16* ptr = bottom_blob.row(i); + __fp16* outptr0 = (__fp16*)top_blob + w * i * 8; + __fp16* outptr1 = (__fp16*)top_blob + w * (i * 8 + 1); + __fp16* outptr2 = (__fp16*)top_blob + w * (i * 8 + 2); + __fp16* outptr3 = (__fp16*)top_blob + w * (i * 8 + 3); + __fp16* outptr4 = (__fp16*)top_blob + w * (i * 8 + 4); + __fp16* outptr5 = (__fp16*)top_blob + w * (i * 8 + 5); + __fp16* outptr6 = (__fp16*)top_blob + w * (i * 8 + 6); + __fp16* outptr7 = (__fp16*)top_blob + w * (i * 8 + 7); + + int j = 0; + for (; j + 3 < w; j += 4) + { + float16x8x4_t _v4 = vld4q_f16(ptr); + float16x8_t _v_01 = vuzp1q_f16(_v4.val[0], _v4.val[1]); + float16x8_t _v_23 = vuzp1q_f16(_v4.val[2], _v4.val[3]); + float16x8_t _v_45 = vuzp2q_f16(_v4.val[0], _v4.val[1]); + float16x8_t _v_67 = vuzp2q_f16(_v4.val[2], _v4.val[3]); + vst1_f16(outptr0, vget_low_f16(_v_01)); + vst1_f16(outptr1, vget_high_f16(_v_01)); + vst1_f16(outptr2, vget_low_f16(_v_23)); + vst1_f16(outptr3, vget_high_f16(_v_23)); + vst1_f16(outptr4, vget_low_f16(_v_45)); + vst1_f16(outptr5, vget_high_f16(_v_45)); + vst1_f16(outptr6, vget_low_f16(_v_67)); + vst1_f16(outptr7, vget_high_f16(_v_67)); + + ptr += 32; + outptr0 += 4; + outptr1 += 4; + outptr2 += 4; + outptr3 += 4; + outptr4 += 4; + outptr5 += 4; + outptr6 += 4; + outptr7 += 4; + } + for (; j < w; j++) + { + *outptr0++ = ptr[0]; + *outptr1++ = ptr[1]; + *outptr2++ = ptr[2]; + *outptr3++ = ptr[3]; + *outptr4++ = ptr[4]; + *outptr5++ = ptr[5]; + *outptr6++ = ptr[6]; + *outptr7++ = ptr[7]; - top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); - if (top_blob.empty()) - return -100; + ptr += 8; + } + } + } +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - if (dims == 2 && elempack == 4) + if (elempack == 4) // out_elempack == 4 || out_elempack == 8 { #pragma omp parallel for num_threads(opt.num_threads) for (int i = 0; i < h; i++) @@ -247,6 +324,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons unsigned short* outptr3 = (unsigned short*)top_blob + w * (i * 4 + 3); int j = 0; +#if __ARM_NEON for (; j + 3 < w; j += 4) { uint16x4x4_t _v4 = vld4_u16(ptr); @@ -261,6 +339,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons outptr2 += 4; outptr3 += 4; } +#endif for (; j < w; j++) { *outptr0++ = ptr[0]; @@ -271,11 +350,72 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons ptr += 4; } } + } + } + + if (dims == 3) + { +#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + if (elempack == 8) // out_elempack == 8 + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + const __fp16* ptr = bottom_blob.channel(q); + __fp16* outptr0 = (__fp16*)top_blob + size * q * 8; + __fp16* outptr1 = (__fp16*)top_blob + size * (q * 8 + 1); + __fp16* outptr2 = (__fp16*)top_blob + size * (q * 8 + 2); + __fp16* outptr3 = (__fp16*)top_blob + size * (q * 8 + 3); + __fp16* outptr4 = (__fp16*)top_blob + size * (q * 8 + 4); + __fp16* outptr5 = (__fp16*)top_blob + size * (q * 8 + 5); + __fp16* outptr6 = (__fp16*)top_blob + size * (q * 8 + 6); + __fp16* outptr7 = (__fp16*)top_blob + size * (q * 8 + 7); + + int i = 0; + for (; i + 3 < size; i += 4) + { + float16x8x4_t _v4 = vld4q_f16(ptr); + float16x8_t _v_01 = vuzp1q_f16(_v4.val[0], _v4.val[1]); + float16x8_t _v_23 = vuzp1q_f16(_v4.val[2], _v4.val[3]); + float16x8_t _v_45 = vuzp2q_f16(_v4.val[0], _v4.val[1]); + float16x8_t _v_67 = vuzp2q_f16(_v4.val[2], _v4.val[3]); + vst1_f16(outptr0, vget_low_f16(_v_01)); + vst1_f16(outptr1, vget_high_f16(_v_01)); + vst1_f16(outptr2, vget_low_f16(_v_23)); + vst1_f16(outptr3, vget_high_f16(_v_23)); + vst1_f16(outptr4, vget_low_f16(_v_45)); + vst1_f16(outptr5, vget_high_f16(_v_45)); + vst1_f16(outptr6, vget_low_f16(_v_67)); + vst1_f16(outptr7, vget_high_f16(_v_67)); + + ptr += 32; + outptr0 += 4; + outptr1 += 4; + outptr2 += 4; + outptr3 += 4; + outptr4 += 4; + outptr5 += 4; + outptr6 += 4; + outptr7 += 4; + } + for (; i < size; i++) + { + *outptr0++ = ptr[0]; + *outptr1++ = ptr[1]; + *outptr2++ = ptr[2]; + *outptr3++ = ptr[3]; + *outptr4++ = ptr[4]; + *outptr5++ = ptr[5]; + *outptr6++ = ptr[6]; + *outptr7++ = ptr[7]; - return 0; + ptr += 8; + } + } } +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - if (dims == 3 && elempack == 4) + if (elempack == 4) // out_elempack == 4 || out_elempack == 8 { #pragma omp parallel for num_threads(opt.num_threads) for (int q = 0; q < channels; q++) @@ -287,6 +427,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons unsigned short* outptr3 = (unsigned short*)top_blob + size * (q * 4 + 3); int i = 0; +#if __ARM_NEON for (; i + 3 < size; i += 4) { uint16x4x4_t _v4 = vld4_u16(ptr); @@ -301,6 +442,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons outptr2 += 4; outptr3 += 4; } +#endif for (; i < size; i++) { *outptr0++ = ptr[0]; @@ -311,11 +453,9 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons ptr += 4; } } - - return 0; } - if (dims == 3 && elempack == 1 && out_elempack == 4) + if (elempack == 1) // out_elempack == 4 || out_elempack == 8 { #pragma omp parallel for num_threads(opt.num_threads) for (int q = 0; q < channels; q++) @@ -324,6 +464,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons unsigned short* outptr = (unsigned short*)top_blob + size * q; int i = 0; +#if __ARM_NEON for (; i + 3 < size; i += 4) { uint16x4_t _v = vld1_u16(ptr); @@ -331,19 +472,16 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons ptr += 4; outptr += 4; } +#endif for (; i < size; i++) { *outptr++ = *ptr++; } } - - return 0; } + } - } // opt.use_packing_layout -#endif // __ARM_NEON - - return Flatten::forward(bottom_blob, top_blob, opt); + return 0; } } // namespace ncnn diff --git a/src/layer/arm/innerproduct_arm.cpp b/src/layer/arm/innerproduct_arm.cpp index f5fab7e34..872b792d4 100644 --- a/src/layer/arm/innerproduct_arm.cpp +++ b/src/layer/arm/innerproduct_arm.cpp @@ -657,7 +657,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons int elempack = bottom_blob.elempack; int size = w * h; - if (elempack == 4) + if (elempack == 4 || elempack == 8) { // flatten Mat bottom_blob_flattened = bottom_blob; diff --git a/src/layer/arm/neon_activation.h b/src/layer/arm/neon_activation.h index 6d6ff377f..87f43eb8d 100644 --- a/src/layer/arm/neon_activation.h +++ b/src/layer/arm/neon_activation.h @@ -106,5 +106,14 @@ static inline float16x4_t activation_ps(float16x4_t _v, int activation_type, con _v32 = activation_ps(_v32, activation_type, activation_params); return vcvt_f16_f32(_v32); } + +static inline float16x8_t activation_ps(float16x8_t _v, int activation_type, const ncnn::Mat& activation_params) +{ + float32x4_t _v32_low = vcvt_f32_f16(vget_low_f16(_v)); + float32x4_t _v32_high = vcvt_f32_f16(vget_high_f16(_v)); + _v32_low = activation_ps(_v32_low, activation_type, activation_params); + _v32_high = activation_ps(_v32_high, activation_type, activation_params); + return vcombine_f16(vcvt_f16_f32(_v32_low), vcvt_f16_f32(_v32_high)); +} #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC #endif // __ARM_NEON