diff --git a/src/layer/arm/convolution_3x3_pack1to8_fp16s.h b/src/layer/arm/convolution_3x3_pack1to8_fp16s.h index c57a7c78c..519344f68 100644 --- a/src/layer/arm/convolution_3x3_pack1to8_fp16s.h +++ b/src/layer/arm/convolution_3x3_pack1to8_fp16s.h @@ -12,6 +12,385 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. +static void conv3x3s1_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int inch = bottom_blob.c; + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const __fp16* bias = _bias; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + Mat out0 = top_blob.channel(p); + + float16x8_t _bias0 = bias ? vld1q_f16(bias + p * 8) : vdupq_n_f16((__fp16)0.f); + out0.fill(_bias0); + + const __fp16* k0 = kernel.channel(p); + + int q = 0; + for (; q < inch; q++) + { + __fp16* outptr0 = out0; + + const Mat img0 = bottom_blob.channel(q); + + const __fp16* r0 = img0.row(0); + const __fp16* r1 = img0.row(1); + const __fp16* r2 = img0.row(2); + + float16x8_t _k00 = vld1q_f16(k0); + float16x8_t _k01 = vld1q_f16(k0 + 8); + float16x8_t _k02 = vld1q_f16(k0 + 16); + float16x8_t _k10 = vld1q_f16(k0 + 24); + float16x8_t _k11 = vld1q_f16(k0 + 32); + float16x8_t _k12 = vld1q_f16(k0 + 40); + float16x8_t _k20 = vld1q_f16(k0 + 48); + float16x8_t _k21 = vld1q_f16(k0 + 56); + float16x8_t _k22 = vld1q_f16(k0 + 64); + + int i = 0; + for (; i < outh; i++) + { + int j = 0; + for (; j + 7 < outw; j += 8) + { + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%0], #64 \n" // sum0 sum1 sum2 sum3 + + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum4 sum5 sum6 sum7 + + "sub %0, %0, #64 \n" + + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v0.8h}, [%1], #16 \n" // r0 + "ld1 {v1.4h}, [%1] \n" + + "fmla v24.8h, %8.8h, v0.h[0] \n" + "fmla v25.8h, %8.8h, v0.h[1] \n" + "fmla v26.8h, %8.8h, v0.h[2] \n" + "fmla v27.8h, %8.8h, v0.h[3] \n" + "fmla v28.8h, %8.8h, v0.h[4] \n" + "fmla v29.8h, %8.8h, v0.h[5] \n" + "fmla v30.8h, %8.8h, v0.h[6] \n" + "fmla v31.8h, %8.8h, v0.h[7] \n" + + "fmla v24.8h, %9.8h, v0.h[1] \n" + "fmla v25.8h, %9.8h, v0.h[2] \n" + "fmla v26.8h, %9.8h, v0.h[3] \n" + "fmla v27.8h, %9.8h, v0.h[4] \n" + "fmla v28.8h, %9.8h, v0.h[5] \n" + "fmla v29.8h, %9.8h, v0.h[6] \n" + "fmla v30.8h, %9.8h, v0.h[7] \n" + "fmla v31.8h, %9.8h, v1.h[0] \n" + + "fmla v24.8h, %10.8h, v0.h[2] \n" + "fmla v25.8h, %10.8h, v0.h[3] \n" + "fmla v26.8h, %10.8h, v0.h[4] \n" + "fmla v27.8h, %10.8h, v0.h[5] \n" + "fmla v28.8h, %10.8h, v0.h[6] \n" + "fmla v29.8h, %10.8h, v0.h[7] \n" + "fmla v30.8h, %10.8h, v1.h[0] \n" + "fmla v31.8h, %10.8h, v1.h[1] \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v2.8h}, [%2], #16 \n" // r1 + "ld1 {v3.4h}, [%2] \n" + + "fmla v24.8h, %11.8h, v2.h[0] \n" + "fmla v25.8h, %11.8h, v2.h[1] \n" + "fmla v26.8h, %11.8h, v2.h[2] \n" + "fmla v27.8h, %11.8h, v2.h[3] \n" + "fmla v28.8h, %11.8h, v2.h[4] \n" + "fmla v29.8h, %11.8h, v2.h[5] \n" + "fmla v30.8h, %11.8h, v2.h[6] \n" + "fmla v31.8h, %11.8h, v2.h[7] \n" + + "fmla v24.8h, %12.8h, v2.h[1] \n" + "fmla v25.8h, %12.8h, v2.h[2] \n" + "fmla v26.8h, %12.8h, v2.h[3] \n" + "fmla v27.8h, %12.8h, v2.h[4] \n" + "fmla v28.8h, %12.8h, v2.h[5] \n" + "fmla v29.8h, %12.8h, v2.h[6] \n" + "fmla v30.8h, %12.8h, v2.h[7] \n" + "fmla v31.8h, %12.8h, v3.h[0] \n" + + "fmla v24.8h, %13.8h, v2.h[2] \n" + "fmla v25.8h, %13.8h, v2.h[3] \n" + "fmla v26.8h, %13.8h, v2.h[4] \n" + "fmla v27.8h, %13.8h, v2.h[5] \n" + "fmla v28.8h, %13.8h, v2.h[6] \n" + "fmla v29.8h, %13.8h, v2.h[7] \n" + "fmla v30.8h, %13.8h, v3.h[0] \n" + "fmla v31.8h, %13.8h, v3.h[1] \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v4.8h}, [%3], #16 \n" // r2 + "ld1 {v5.4h}, [%3] \n" + + "fmla v24.8h, %14.8h, v4.h[0] \n" + "fmla v25.8h, %14.8h, v4.h[1] \n" + "fmla v26.8h, %14.8h, v4.h[2] \n" + "fmla v27.8h, %14.8h, v4.h[3] \n" + "fmla v28.8h, %14.8h, v4.h[4] \n" + "fmla v29.8h, %14.8h, v4.h[5] \n" + "fmla v30.8h, %14.8h, v4.h[6] \n" + "fmla v31.8h, %14.8h, v4.h[7] \n" + + "fmla v24.8h, %15.8h, v4.h[1] \n" + "fmla v25.8h, %15.8h, v4.h[2] \n" + "fmla v26.8h, %15.8h, v4.h[3] \n" + "fmla v27.8h, %15.8h, v4.h[4] \n" + "fmla v28.8h, %15.8h, v4.h[5] \n" + "fmla v29.8h, %15.8h, v4.h[6] \n" + "fmla v30.8h, %15.8h, v4.h[7] \n" + "fmla v31.8h, %15.8h, v5.h[0] \n" + + "fmla v24.8h, %16.8h, v4.h[2] \n" + "fmla v25.8h, %16.8h, v4.h[3] \n" + "fmla v26.8h, %16.8h, v4.h[4] \n" + "fmla v27.8h, %16.8h, v4.h[5] \n" + "fmla v28.8h, %16.8h, v4.h[6] \n" + "fmla v29.8h, %16.8h, v4.h[7] \n" + "fmla v30.8h, %16.8h, v5.h[0] \n" + "fmla v31.8h, %16.8h, v5.h[1] \n" + + "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%0], #64 \n" + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; j + 3 < outw; j += 4) + { + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum0 sum1 sum2 sum3 + + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v0.8h}, [%1] \n" // r0 + + "fmla v28.8h, %8.8h, v0.h[0] \n" + "fmla v29.8h, %8.8h, v0.h[1] \n" + "fmla v30.8h, %8.8h, v0.h[2] \n" + "fmla v31.8h, %8.8h, v0.h[3] \n" + + "fmla v28.8h, %9.8h, v0.h[1] \n" + "fmla v29.8h, %9.8h, v0.h[2] \n" + "fmla v30.8h, %9.8h, v0.h[3] \n" + "fmla v31.8h, %9.8h, v0.h[4] \n" + + "fmla v28.8h, %10.8h, v0.h[2] \n" + "fmla v29.8h, %10.8h, v0.h[3] \n" + "fmla v30.8h, %10.8h, v0.h[4] \n" + "fmla v31.8h, %10.8h, v0.h[5] \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v1.8h}, [%2] \n" // r1 + + "fmla v28.8h, %11.8h, v1.h[0] \n" + "fmla v29.8h, %11.8h, v1.h[1] \n" + "fmla v30.8h, %11.8h, v1.h[2] \n" + "fmla v31.8h, %11.8h, v1.h[3] \n" + + "fmla v28.8h, %12.8h, v1.h[1] \n" + "fmla v29.8h, %12.8h, v1.h[2] \n" + "fmla v30.8h, %12.8h, v1.h[3] \n" + "fmla v31.8h, %12.8h, v1.h[4] \n" + + "fmla v28.8h, %13.8h, v1.h[2] \n" + "fmla v29.8h, %13.8h, v1.h[3] \n" + "fmla v30.8h, %13.8h, v1.h[4] \n" + "fmla v31.8h, %13.8h, v1.h[5] \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v2.8h}, [%3] \n" // r2 + + "fmla v28.8h, %14.8h, v2.h[0] \n" + "fmla v29.8h, %14.8h, v2.h[1] \n" + "fmla v30.8h, %14.8h, v2.h[2] \n" + "fmla v31.8h, %14.8h, v2.h[3] \n" + + "fmla v28.8h, %15.8h, v2.h[1] \n" + "fmla v29.8h, %15.8h, v2.h[2] \n" + "fmla v30.8h, %15.8h, v2.h[3] \n" + "fmla v31.8h, %15.8h, v2.h[4] \n" + + "fmla v28.8h, %16.8h, v2.h[2] \n" + "fmla v29.8h, %16.8h, v2.h[3] \n" + "fmla v30.8h, %16.8h, v2.h[4] \n" + "fmla v31.8h, %16.8h, v2.h[5] \n" + + "add %1, %1, #8 \n" + "add %2, %2, #8 \n" + "add %3, %3, #8 \n" + + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v28", "v29", "v30", "v31"); + } + for (; j + 1 < outw; j += 2) + { + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v30.8h, v31.8h}, [%0] \n" // sum0 sum1 + + "prfm pldl1keep, [%1, #64] \n" + "ld1 {v0.4h}, [%1] \n" // r0 + + "fmla v30.8h, %8.8h, v0.h[0] \n" + "fmla v31.8h, %8.8h, v0.h[1] \n" + "fmla v30.8h, %9.8h, v0.h[1] \n" + "fmla v31.8h, %9.8h, v0.h[2] \n" + "fmla v30.8h, %10.8h, v0.h[2] \n" + "fmla v31.8h, %10.8h, v0.h[3] \n" + + "prfm pldl1keep, [%2, #64] \n" + "ld1 {v1.4h}, [%2] \n" // r1 + + "fmla v30.8h, %11.8h, v1.h[0] \n" + "fmla v31.8h, %11.8h, v1.h[1] \n" + "fmla v30.8h, %12.8h, v1.h[1] \n" + "fmla v31.8h, %12.8h, v1.h[2] \n" + "fmla v30.8h, %13.8h, v1.h[2] \n" + "fmla v31.8h, %13.8h, v1.h[3] \n" + + "prfm pldl1keep, [%3, #64] \n" + "ld1 {v2.4h}, [%3] \n" // r2 + + "fmla v30.8h, %14.8h, v2.h[0] \n" + "fmla v31.8h, %14.8h, v2.h[1] \n" + "fmla v30.8h, %15.8h, v2.h[1] \n" + "fmla v31.8h, %15.8h, v2.h[2] \n" + "fmla v30.8h, %16.8h, v2.h[2] \n" + "fmla v31.8h, %16.8h, v2.h[3] \n" + + "add %1, %1, #4 \n" + "add %2, %2, #4 \n" + "add %3, %3, #4 \n" + + "st1 {v30.8h, v31.8h}, [%0], #32 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v30", "v31"); + } + for (; j < outw; j++) + { + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v30.8h}, [%0] \n" // sum0 + + "prfm pldl1keep, [%1, #64] \n" + "ld1 {v0.4h}, [%1] \n" // r0 + + "fmla v30.8h, %8.8h, v0.h[0] \n" + "fmla v30.8h, %9.8h, v0.h[1] \n" + "fmla v30.8h, %10.8h, v0.h[2] \n" + + "prfm pldl1keep, [%2, #64] \n" + "ld1 {v1.4h}, [%2] \n" // r1 + + "fmla v30.8h, %11.8h, v1.h[0] \n" + "fmla v30.8h, %12.8h, v1.h[1] \n" + "fmla v30.8h, %13.8h, v1.h[2] \n" + + "prfm pldl1keep, [%3, #64] \n" + "ld1 {v2.4h}, [%3] \n" // r2 + + "fmla v30.8h, %14.8h, v2.h[0] \n" + "fmla v30.8h, %15.8h, v2.h[1] \n" + "fmla v30.8h, %16.8h, v2.h[2] \n" + + "add %1, %1, #2 \n" + "add %2, %2, #2 \n" + "add %3, %3, #2 \n" + + "st1 {v30.8h}, [%0], #16 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v30"); + } + + r0 += 2; + r1 += 2; + r2 += 2; + } + + k0 += 9 * 8; + } + } +} + static void conv3x3s2_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) { int w = bottom_blob.w; @@ -56,143 +435,204 @@ static void conv3x3s2_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob float16x8_t _k22 = vld1q_f16(k0 + 64); int i = 0; - for (; i < outh; i++) { int j = 0; - for (; j + 3 < outw; j += 4) { - float16x8_t _sum0 = vld1q_f16(outptr0); - float16x8_t _sum1 = vld1q_f16(outptr0 + 8); - float16x8_t _sum2 = vld1q_f16(outptr0 + 16); - float16x8_t _sum3 = vld1q_f16(outptr0 + 24); - - float16x8_t _r0 = vld1q_f16(r0); - float16x8_t _r1 = vld1q_f16(r1); - float16x8_t _r2 = vld1q_f16(r2); - - r0 += 8; - r1 += 8; - r2 += 8; - - float16x4_t _r0n = vld1_dup_f16(r0); - float16x4_t _r1n = vld1_dup_f16(r1); - float16x4_t _r2n = vld1_dup_f16(r2); - - _sum0 = vfmaq_laneq_f16(_sum0, _k00, _r0, 0); - _sum0 = vfmaq_laneq_f16(_sum0, _k01, _r0, 1); - _sum0 = vfmaq_laneq_f16(_sum0, _k02, _r0, 2); - _sum0 = vfmaq_laneq_f16(_sum0, _k10, _r1, 0); - _sum0 = vfmaq_laneq_f16(_sum0, _k11, _r1, 1); - _sum0 = vfmaq_laneq_f16(_sum0, _k12, _r1, 2); - _sum0 = vfmaq_laneq_f16(_sum0, _k20, _r2, 0); - _sum0 = vfmaq_laneq_f16(_sum0, _k21, _r2, 1); - _sum0 = vfmaq_laneq_f16(_sum0, _k22, _r2, 2); - - _sum1 = vfmaq_laneq_f16(_sum1, _k00, _r0, 2); - _sum1 = vfmaq_laneq_f16(_sum1, _k01, _r0, 3); - _sum1 = vfmaq_laneq_f16(_sum1, _k02, _r0, 4); - _sum1 = vfmaq_laneq_f16(_sum1, _k10, _r1, 2); - _sum1 = vfmaq_laneq_f16(_sum1, _k11, _r1, 3); - _sum1 = vfmaq_laneq_f16(_sum1, _k12, _r1, 4); - _sum1 = vfmaq_laneq_f16(_sum1, _k20, _r2, 2); - _sum1 = vfmaq_laneq_f16(_sum1, _k21, _r2, 3); - _sum1 = vfmaq_laneq_f16(_sum1, _k22, _r2, 4); - - _sum2 = vfmaq_laneq_f16(_sum2, _k00, _r0, 4); - _sum2 = vfmaq_laneq_f16(_sum2, _k01, _r0, 5); - _sum2 = vfmaq_laneq_f16(_sum2, _k02, _r0, 6); - _sum2 = vfmaq_laneq_f16(_sum2, _k10, _r1, 4); - _sum2 = vfmaq_laneq_f16(_sum2, _k11, _r1, 5); - _sum2 = vfmaq_laneq_f16(_sum2, _k12, _r1, 6); - _sum2 = vfmaq_laneq_f16(_sum2, _k20, _r2, 4); - _sum2 = vfmaq_laneq_f16(_sum2, _k21, _r2, 5); - _sum2 = vfmaq_laneq_f16(_sum2, _k22, _r2, 6); - - _sum3 = vfmaq_laneq_f16(_sum3, _k00, _r0, 6); - _sum3 = vfmaq_laneq_f16(_sum3, _k01, _r0, 7); - _sum3 = vfmaq_lane_f16(_sum3, _k02, _r0n, 0); - _sum3 = vfmaq_laneq_f16(_sum3, _k10, _r1, 6); - _sum3 = vfmaq_laneq_f16(_sum3, _k11, _r1, 7); - _sum3 = vfmaq_lane_f16(_sum3, _k12, _r1n, 0); - _sum3 = vfmaq_laneq_f16(_sum3, _k20, _r2, 6); - _sum3 = vfmaq_laneq_f16(_sum3, _k21, _r2, 7); - _sum3 = vfmaq_lane_f16(_sum3, _k22, _r2n, 0); - - vst1q_f16(outptr0, _sum0); - vst1q_f16(outptr0 + 8, _sum1); - vst1q_f16(outptr0 + 16, _sum2); - vst1q_f16(outptr0 + 24, _sum3); - - outptr0 += 32; + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum0 sum1 sum2 sum3 + + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v0.8h}, [%1], #16 \n" // r0 + "ld1 {v1.h}[0], [%1] \n" + + "fmla v28.8h, %8.8h, v0.h[0] \n" + "fmla v29.8h, %8.8h, v0.h[2] \n" + "fmla v30.8h, %8.8h, v0.h[4] \n" + "fmla v31.8h, %8.8h, v0.h[6] \n" + + "fmla v28.8h, %9.8h, v0.h[1] \n" + "fmla v29.8h, %9.8h, v0.h[3] \n" + "fmla v30.8h, %9.8h, v0.h[5] \n" + "fmla v31.8h, %9.8h, v0.h[7] \n" + + "fmla v28.8h, %10.8h, v0.h[2] \n" + "fmla v29.8h, %10.8h, v0.h[4] \n" + "fmla v30.8h, %10.8h, v0.h[6] \n" + "fmla v31.8h, %10.8h, v1.h[0] \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v2.8h}, [%2], #16 \n" // r1 + "ld1 {v3.h}[0], [%2] \n" + + "fmla v28.8h, %11.8h, v2.h[0] \n" + "fmla v29.8h, %11.8h, v2.h[2] \n" + "fmla v30.8h, %11.8h, v2.h[4] \n" + "fmla v31.8h, %11.8h, v2.h[6] \n" + + "fmla v28.8h, %12.8h, v2.h[1] \n" + "fmla v29.8h, %12.8h, v2.h[3] \n" + "fmla v30.8h, %12.8h, v2.h[5] \n" + "fmla v31.8h, %12.8h, v2.h[7] \n" + + "fmla v28.8h, %13.8h, v2.h[2] \n" + "fmla v29.8h, %13.8h, v2.h[4] \n" + "fmla v30.8h, %13.8h, v2.h[6] \n" + "fmla v31.8h, %13.8h, v3.h[0] \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v4.8h}, [%3], #16 \n" // r2 + "ld1 {v5.h}[0], [%3] \n" + + "fmla v28.8h, %14.8h, v4.h[0] \n" + "fmla v29.8h, %14.8h, v4.h[2] \n" + "fmla v30.8h, %14.8h, v4.h[4] \n" + "fmla v31.8h, %14.8h, v4.h[6] \n" + + "fmla v28.8h, %15.8h, v4.h[1] \n" + "fmla v29.8h, %15.8h, v4.h[3] \n" + "fmla v30.8h, %15.8h, v4.h[5] \n" + "fmla v31.8h, %15.8h, v4.h[7] \n" + + "fmla v28.8h, %16.8h, v4.h[2] \n" + "fmla v29.8h, %16.8h, v4.h[4] \n" + "fmla v30.8h, %16.8h, v4.h[6] \n" + "fmla v31.8h, %16.8h, v5.h[0] \n" + + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v28", "v29", "v30", "v31"); } for (; j + 1 < outw; j += 2) { - float16x8_t _sum0 = vld1q_f16(outptr0); - float16x8_t _sum1 = vld1q_f16(outptr0 + 8); - - float16x4_t _r0 = vld1_f16(r0); - float16x4_t _r1 = vld1_f16(r1); - float16x4_t _r2 = vld1_f16(r2); - - r0 += 4; - r1 += 4; - r2 += 4; - - float16x4_t _r0n = vld1_dup_f16(r0); - float16x4_t _r1n = vld1_dup_f16(r1); - float16x4_t _r2n = vld1_dup_f16(r2); - - _sum0 = vfmaq_lane_f16(_sum0, _k00, _r0, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k01, _r0, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k02, _r0, 2); - _sum0 = vfmaq_lane_f16(_sum0, _k10, _r1, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k11, _r1, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k12, _r1, 2); - _sum0 = vfmaq_lane_f16(_sum0, _k20, _r2, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k21, _r2, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k22, _r2, 2); - - _sum1 = vfmaq_lane_f16(_sum1, _k00, _r0, 2); - _sum1 = vfmaq_lane_f16(_sum1, _k01, _r0, 3); - _sum1 = vfmaq_lane_f16(_sum1, _k02, _r0n, 0); - _sum1 = vfmaq_lane_f16(_sum1, _k10, _r1, 2); - _sum1 = vfmaq_lane_f16(_sum1, _k11, _r1, 3); - _sum1 = vfmaq_lane_f16(_sum1, _k12, _r1n, 0); - _sum1 = vfmaq_lane_f16(_sum1, _k20, _r2, 2); - _sum1 = vfmaq_lane_f16(_sum1, _k21, _r2, 3); - _sum1 = vfmaq_lane_f16(_sum1, _k22, _r2n, 0); - - vst1q_f16(outptr0, _sum0); - vst1q_f16(outptr0 + 8, _sum1); - - outptr0 += 16; + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v30.8h, v31.8h}, [%0] \n" // sum0 sum1 + + "prfm pldl1keep, [%1, #64] \n" + "ld1 {v0.4h}, [%1], #8 \n" // r0 + "ld1 {v1.h}[0], [%1] \n" + + "fmla v30.8h, %8.8h, v0.h[0] \n" + "fmla v31.8h, %8.8h, v0.h[2] \n" + "fmla v30.8h, %9.8h, v0.h[1] \n" + "fmla v31.8h, %9.8h, v0.h[3] \n" + "fmla v30.8h, %10.8h, v0.h[2] \n" + "fmla v31.8h, %10.8h, v1.h[0] \n" + + "prfm pldl1keep, [%2, #64] \n" + "ld1 {v2.4h}, [%2], #8 \n" // r1 + "ld1 {v3.h}[0], [%2] \n" + + "fmla v30.8h, %11.8h, v2.h[0] \n" + "fmla v31.8h, %11.8h, v2.h[2] \n" + "fmla v30.8h, %12.8h, v2.h[1] \n" + "fmla v31.8h, %12.8h, v2.h[3] \n" + "fmla v30.8h, %13.8h, v2.h[2] \n" + "fmla v31.8h, %13.8h, v3.h[0] \n" + + "prfm pldl1keep, [%3, #64] \n" + "ld1 {v4.4h}, [%3], #8 \n" // r2 + "ld1 {v5.h}[0], [%3] \n" + + "fmla v30.8h, %14.8h, v4.h[0] \n" + "fmla v31.8h, %14.8h, v4.h[2] \n" + "fmla v30.8h, %15.8h, v4.h[1] \n" + "fmla v31.8h, %15.8h, v4.h[3] \n" + "fmla v30.8h, %16.8h, v4.h[2] \n" + "fmla v31.8h, %16.8h, v5.h[0] \n" + + "st1 {v30.8h, v31.8h}, [%0], #32 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31"); } for (; j < outw; j++) { - float16x8_t _sum0 = vld1q_f16(outptr0); - - float16x4_t _r0 = vld1_f16(r0); - float16x4_t _r1 = vld1_f16(r1); - float16x4_t _r2 = vld1_f16(r2); - - _sum0 = vfmaq_lane_f16(_sum0, _k00, _r0, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k01, _r0, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k02, _r0, 2); - _sum0 = vfmaq_lane_f16(_sum0, _k10, _r1, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k11, _r1, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k12, _r1, 2); - _sum0 = vfmaq_lane_f16(_sum0, _k20, _r2, 0); - _sum0 = vfmaq_lane_f16(_sum0, _k21, _r2, 1); - _sum0 = vfmaq_lane_f16(_sum0, _k22, _r2, 2); - - vst1q_f16(outptr0, _sum0); - - r0 += 2; - r1 += 2; - r2 += 2; - outptr0 += 8; + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v30.8h}, [%0] \n" // sum0 + + "prfm pldl1keep, [%1, #64] \n" + "ld1 {v0.4h}, [%1] \n" // r0 + + "fmla v30.8h, %8.8h, v0.h[0] \n" + "fmla v30.8h, %9.8h, v0.h[1] \n" + "fmla v30.8h, %10.8h, v0.h[2] \n" + + "prfm pldl1keep, [%2, #64] \n" + "ld1 {v1.4h}, [%2] \n" // r1 + + "fmla v30.8h, %11.8h, v1.h[0] \n" + "fmla v30.8h, %12.8h, v1.h[1] \n" + "fmla v30.8h, %13.8h, v1.h[2] \n" + + "prfm pldl1keep, [%3, #64] \n" + "ld1 {v2.4h}, [%3] \n" // r2 + + "fmla v30.8h, %14.8h, v2.h[0] \n" + "fmla v30.8h, %15.8h, v2.h[1] \n" + "fmla v30.8h, %16.8h, v2.h[2] \n" + + "add %1, %1, #4 \n" + "add %2, %2, #4 \n" + "add %3, %3, #4 \n" + + "st1 {v30.8h}, [%0], #16 \n" + + : "=r"(outptr0), // %0 + "=r"(r0), // %1 + "=r"(r1), // %2 + "=r"(r2) // %3 + : "0"(outptr0), + "1"(r0), + "2"(r1), + "3"(r2), + "w"(_k00), // %8 + "w"(_k01), // %9 + "w"(_k02), // %10 + "w"(_k10), // %11 + "w"(_k11), // %12 + "w"(_k12), // %13 + "w"(_k20), // %14 + "w"(_k21), // %15 + "w"(_k22) // %16 + : "cc", "memory", "v0", "v1", "v2", "v30"); } r0 += tailstep; diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index db3e1a62c..a2b91fc2c 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -1533,7 +1533,16 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const if (elempack == 1 && out_elempack == 8) { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + conv3x3s1_pack1to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv3x3s2_pack1to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); diff --git a/src/layer/arm/padding_pack8_fp16s.h b/src/layer/arm/padding_pack8_fp16s.h index c6d1d96ce..dd5902ea8 100644 --- a/src/layer/arm/padding_pack8_fp16s.h +++ b/src/layer/arm/padding_pack8_fp16s.h @@ -17,45 +17,145 @@ static void padding_constant_pack8_fp16s_neon(const Mat& src, Mat& dst, int top, const __fp16* ptr = src; __fp16* outptr = dst; - // fill top - for (int y = 0; y < top; y++) - { - for (int x = 0; x < dst.w; x++) - { - vst1q_f16(outptr, _v); - outptr += 8; - } - } - // fill center - for (int y = 0; y < src.h; y++) - { - for (int x = 0; x < left; x++) - { - vst1q_f16(outptr, _v); - outptr += 8; - } - for (int x = 0; x < src.w; x++) - { - float16x8_t _p = vld1q_f16(ptr); - vst1q_f16(outptr, _p); - ptr += 8; - outptr += 8; - } - for (int x = 0; x < right; x++) - { - vst1q_f16(outptr, _v); - outptr += 8; - } - } - // fill bottom - for (int y = 0; y < bottom; y++) - { - for (int x = 0; x < dst.w; x++) - { - vst1q_f16(outptr, _v); - outptr += 8; - } - } + int w = src.w; + int h = src.h; + + int top_size = top * dst.w; + int bottom_size = bottom * dst.w; + + asm volatile( + "mov v0.16b, %10.16b \n" + "mov v1.16b, %10.16b \n" + "mov v2.16b, %10.16b \n" + "mov v3.16b, %10.16b \n" + + // fill top + "lsr w4, %w8, #2 \n" // w4 = nn = top_size >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "subs w4, w4, #1 \n" + "bne 0b \n" + + "1: \n" + + // fill top remain + "and w4, %w8, #3 \n" // w4 = remain = top_size & 3 + + "cmp w4, #2 \n" // w4 >= 2 + "blt 2f \n" + "sub w4, w4, #2 \n" + "st1 {v0.8h, v1.8h}, [%0], #32 \n" + "2: \n" + + "cmp w4, #0 \n" // w4 > 0 + "beq 3f \n" + "st1 {v0.8h}, [%0], #16 \n" + "3: \n" + + // fill center h loop + "cmp %w5, #0 \n" + "beq 13f \n" + "4: \n" + + // fill left + "mov w4, %w6 \n" // w4 = left + "cmp w4, #0 \n" + "beq 6f \n" + + "5: \n" + "st1 {v0.8h}, [%0], #16 \n" + "subs w4, w4, #1 \n" + "bne 5b \n" + + "6: \n" + + // fill middle + "lsr w4, %w4, #2 \n" // w4 = nn = w >> 2 + "cmp w4, #0 \n" + "beq 8f \n" + + "7: \n" + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + "subs w4, w4, #1 \n" + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0], #64 \n" + "bne 7b \n" + + "8: \n" + + "and w4, %w4, #3 \n" // w4 = remain = w & 3 + + "cmp w4, #2 \n" // w4 >= 2 + "blt 9f \n" + "prfm pldl1keep, [%1, #256] \n" + "ld1 {v16.8h, v17.8h}, [%1], #32 \n" + "sub w4, w4, #2 \n" + "st1 {v16.8h, v17.8h}, [%0], #32 \n" + "9: \n" + + "cmp w4, #0 \n" // w4 > 0 + "beq 10f \n" + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v16.8h}, [%1], #16 \n" + "st1 {v16.8h}, [%0], #16 \n" + "10: \n" + + // fill right + "mov w4, %w7 \n" // w4 = right + "cmp w4, #0 \n" + "beq 12f \n" + + "11: \n" + "subs w4, w4, #1 \n" + "st1 {v0.8h}, [%0], #16 \n" + "bne 11b \n" + "12: \n" + + "subs %w5, %w5, #1 \n" + "bne 4b \n" + + "13: \n" + + // fill bottom + "lsr w4, %w9, #2 \n" // w4 = nn = bottom_size >> 2 + "cmp w4, #0 \n" + "beq 15f \n" + + "14: \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "subs w4, w4, #1 \n" + "bne 14b \n" + "15: \n" + + // fill bottom remain + "and w4, %w9, #3 \n" // w4 = remain = bottom_size & 3 + + "cmp w4, #2 \n" // w4 >= 2 + "blt 16f \n" + "sub w4, w4, #2 \n" + "st1 {v0.8h, v1.8h}, [%0], #32 \n" + "16: \n" + + "cmp w4, #0 \n" // w4 > 0 + "beq 17f \n" + "st1 {v0.8h}, [%0], #16 \n" + "17: \n" + + : "=r"(outptr), // %0 + "=r"(ptr) // %1 + : "0"(outptr), + "1"(ptr), + "r"(w), // %4 + "r"(h), // %5 + "r"(left), // %6 + "r"(right), // %7 + "r"(top_size), // %8 + "r"(bottom_size), // %9 + "w"(_v) // %10 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19"); } static void padding_replicate_pack8_fp16s_neon(const Mat& src, Mat& dst, int top, int bottom, int left, int right) diff --git a/src/layer/arm/relu_arm.cpp b/src/layer/arm/relu_arm.cpp index 5420ea33c..bd7828ea9 100644 --- a/src/layer/arm/relu_arm.cpp +++ b/src/layer/arm/relu_arm.cpp @@ -358,16 +358,49 @@ int ReLU_arm::forward_inplace_fp16s(Mat& bottom_top_blob, const Option& opt) con { __fp16* ptr = bottom_top_blob.channel(q); - float16x8_t _zero = vdupq_n_f16((__fp16)0.f); + asm volatile( + "eor v16.16b, v16.16b, v16.16b \n" - for (int i = 0; i < size; i++) - { - float16x8_t _p = vld1q_f16(ptr); - _p = vmaxq_f16(_p, _zero); - vst1q_f16(ptr, _p); + "lsr w4, %w2, #2 \n" // w4 = nn = size >> 2 + "cmp w4, #0 \n" + "beq 1f \n" - ptr += 8; - } + "0: \n" + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" + "fmax v0.8h, v0.8h, v16.8h \n" + "fmax v1.8h, v1.8h, v16.8h \n" + "fmax v2.8h, v2.8h, v16.8h \n" + "fmax v3.8h, v3.8h, v16.8h \n" + "subs w4, w4, #1 \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "bne 0b \n" + "1: \n" + + "and w4, %w2, #3 \n" // w4 = remain = size & 3 + + "cmp w4, #2 \n" // w4 >= 2 + "blt 2f \n" + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.8h, v1.8h}, [%0] \n" + "fmax v0.8h, v0.8h, v16.8h \n" + "fmax v1.8h, v1.8h, v16.8h \n" + "sub w4, w4, #2 \n" + "st1 {v0.8h, v1.8h}, [%0], #32 \n" + "2: \n" + + "cmp w4, #0 \n" // w4 > 0 + "beq 3f \n" + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.8h}, [%0] \n" + "fmax v0.8h, v0.8h, v16.8h \n" + "st1 {v0.8h}, [%0], #16 \n" + "3: \n" + + : "=r"(ptr) // %0 + : "0"(ptr), + "r"(size) // %2 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16"); } } else