From a862a3432d62585bf185ecdfccbb7167b0d09f0c Mon Sep 17 00:00:00 2001 From: nihui Date: Sat, 15 Sep 2018 10:04:20 +0800 Subject: [PATCH] aarch64 assembly for conv3x3s1, about 18% speed gain --- src/layer/arm/convolution_3x3.h | 582 +++++++++++++++++++++----------- 1 file changed, 385 insertions(+), 197 deletions(-) diff --git a/src/layer/arm/convolution_3x3.h b/src/layer/arm/convolution_3x3.h index bce662dc5..bd482f37f 100644 --- a/src/layer/arm/convolution_3x3.h +++ b/src/layer/arm/convolution_3x3.h @@ -85,86 +85,148 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke #if __ARM_NEON #if __aarch64__ - for (; nn>0; nn--) + if (nn > 0) { - float32x4_t _sum0 = vld1q_f32(outptr0); - float32x4_t _sum1 = vld1q_f32(outptr1); - float32x4_t _sum0n = vld1q_f32(outptr0n); - float32x4_t _sum1n = vld1q_f32(outptr1n); + asm volatile( + "prfm pldl1keep, [%5, #256] \n" + "ld1 {v8.4s, v9.4s}, [%5] \n"// r0 + "add %5, %5, #16 \n" - float32x4_t _r00 = vld1q_f32(r0); - float32x4_t _r00n = vld1q_f32(r0 + 4); - float32x4_t _r01 = vextq_f32(_r00, _r00n, 1); - float32x4_t _r02 = vextq_f32(_r00, _r00n, 2); + "prfm pldl1keep, [%8, #256] \n" + "ld1 {v14.4s, v15.4s}, [%8] \n"// r3 + "add %8, %8, #16 \n" - float32x4_t _r10 = vld1q_f32(r1); - float32x4_t _r10n = vld1q_f32(r1 + 4); - float32x4_t _r11 = vextq_f32(_r10, _r10n, 1); - float32x4_t _r12 = vextq_f32(_r10, _r10n, 2); + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v14.16b, v15.16b, #8 \n" - float32x4_t _r20 = vld1q_f32(r2); - float32x4_t _r20n = vld1q_f32(r2 + 4); - float32x4_t _r21 = vextq_f32(_r20, _r20n, 1); - float32x4_t _r22 = vextq_f32(_r20, _r20n, 2); + "0: \n" - float32x4_t _r30 = vld1q_f32(r3); - float32x4_t _r30n = vld1q_f32(r3 + 4); - float32x4_t _r31 = vextq_f32(_r30, _r30n, 1); - float32x4_t _r32 = vextq_f32(_r30, _r30n, 2); - - _sum0 = vfmaq_laneq_f32(_sum0, _r00, _k00, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r01, _k00, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r02, _k00, 2); - _sum0 = vfmaq_laneq_f32(_sum0, _r10, _k03, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r11, _k03, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r12, _k03, 2); - _sum0 = vfmaq_laneq_f32(_sum0, _r20, _k06, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r21, _k06, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r22, _k06, 2); - - _sum1 = vfmaq_laneq_f32(_sum1, _r00, _k10, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r01, _k10, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r02, _k10, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r10, _k13, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r11, _k13, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r12, _k13, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r20, _k16, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r21, _k16, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r22, _k16, 2); - - _sum0n = vfmaq_laneq_f32(_sum0n, _r10, _k00, 0); - _sum0n = vfmaq_laneq_f32(_sum0n, _r11, _k00, 1); - _sum0n = vfmaq_laneq_f32(_sum0n, _r12, _k00, 2); - _sum0n = vfmaq_laneq_f32(_sum0n, _r20, _k03, 0); - _sum0n = vfmaq_laneq_f32(_sum0n, _r21, _k03, 1); - _sum0n = vfmaq_laneq_f32(_sum0n, _r22, _k03, 2); - _sum0n = vfmaq_laneq_f32(_sum0n, _r30, _k06, 0); - _sum0n = vfmaq_laneq_f32(_sum0n, _r31, _k06, 1); - _sum0n = vfmaq_laneq_f32(_sum0n, _r32, _k06, 2); - - _sum1n = vfmaq_laneq_f32(_sum1n, _r10, _k10, 0); - _sum1n = vfmaq_laneq_f32(_sum1n, _r11, _k10, 1); - _sum1n = vfmaq_laneq_f32(_sum1n, _r12, _k10, 2); - _sum1n = vfmaq_laneq_f32(_sum1n, _r20, _k13, 0); - _sum1n = vfmaq_laneq_f32(_sum1n, _r21, _k13, 1); - _sum1n = vfmaq_laneq_f32(_sum1n, _r22, _k13, 2); - _sum1n = vfmaq_laneq_f32(_sum1n, _r30, _k16, 0); - _sum1n = vfmaq_laneq_f32(_sum1n, _r31, _k16, 1); - _sum1n = vfmaq_laneq_f32(_sum1n, _r32, _k16, 2); - - vst1q_f32(outptr0, _sum0); - vst1q_f32(outptr1, _sum1); - vst1q_f32(outptr0n, _sum0n); - vst1q_f32(outptr1n, _sum1n); + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v6.4s}, [%1] \n"// _sum0 - r0 += 4; - r1 += 4; - r2 += 4; - r3 += 4; - outptr0 += 4; - outptr1 += 4; - outptr0n += 4; - outptr1n += 4; + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v7.4s}, [%2] \n"// _sum1 + + "fmla v6.4s, v8.4s, %18.s[0] \n" + "fmla v7.4s, v8.4s, %21.s[0] \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v12.4s}, [%3] \n"// _sum0n + + "prfm pldl1keep, [%4, #128] \n" + "ld1 {v13.4s}, [%4] \n"// _sum1n + + "fmla v12.4s, v14.4s, %20.s[0] \n" + "fmla v13.4s, v14.4s, %23.s[0] \n" + + "ext v8.16b, v8.16b, v9.16b, #8 \n" + "ext v9.16b, v14.16b, v15.16b, #4 \n" + + "fmla v6.4s, v10.4s, %18.s[1] \n" + "fmla v7.4s, v10.4s, %21.s[1] \n" + "fmla v12.4s, v11.4s, %20.s[2] \n" + "fmla v13.4s, v11.4s, %23.s[2] \n" + + "prfm pldl1keep, [%6, #256] \n" + "ld1 {v14.4s, v15.4s}, [%6] \n"// r1 + "add %6, %6, #16 \n" + + "fmla v6.4s, v8.4s, %18.s[2] \n" + "fmla v7.4s, v8.4s, %21.s[2] \n" + "fmla v12.4s, v9.4s, %20.s[1] \n" + "fmla v13.4s, v9.4s, %23.s[1] \n" + + "ext v10.16b, v14.16b, v15.16b, #4 \n" + + "fmla v6.4s, v14.4s, %19.s[0] \n" + "fmla v7.4s, v14.4s, %22.s[0] \n" + "fmla v12.4s, v14.4s, %18.s[0] \n" + "fmla v13.4s, v14.4s, %21.s[0] \n" + + "ext v11.16b, v14.16b, v15.16b, #8 \n" + + "fmla v6.4s, v10.4s, %19.s[1] \n" + "fmla v7.4s, v10.4s, %22.s[1] \n" + "fmla v12.4s, v10.4s, %18.s[1] \n" + "fmla v13.4s, v10.4s, %21.s[1] \n" + + "prfm pldl1keep, [%7, #256] \n" + "ld1 {v8.4s, v9.4s}, [%7] \n"// r2 + "add %7, %7, #16 \n" + + "fmla v6.4s, v11.4s, %19.s[2] \n" + "fmla v7.4s, v11.4s, %22.s[2] \n" + "fmla v12.4s, v11.4s, %18.s[2] \n" + "fmla v13.4s, v11.4s, %21.s[2] \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + + "fmla v6.4s, v8.4s, %20.s[0] \n" + "fmla v7.4s, v8.4s, %23.s[0] \n" + "fmla v12.4s, v8.4s, %19.s[0] \n" + "fmla v13.4s, v8.4s, %22.s[0] \n" + + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v6.4s, v10.4s, %20.s[1] \n" + "fmla v7.4s, v10.4s, %23.s[1] \n" + "fmla v12.4s, v10.4s, %19.s[1] \n" + "fmla v13.4s, v10.4s, %22.s[1] \n" + + "prfm pldl1keep, [%5, #256] \n" + "ld1 {v8.4s, v9.4s}, [%5] \n"// r0 + "add %5, %5, #16 \n" + + "fmla v6.4s, v11.4s, %20.s[2] \n" + "fmla v7.4s, v11.4s, %23.s[2] \n" + "fmla v12.4s, v11.4s, %19.s[2] \n" + "fmla v13.4s, v11.4s, %22.s[2] \n" + + "prfm pldl1keep, [%8, #256] \n" + "ld1 {v14.4s, v15.4s}, [%8] \n"// r3 + "add %8, %8, #16 \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + + "st1 {v6.4s}, [%1], #16 \n" + "st1 {v7.4s}, [%2], #16 \n" + + "ext v11.16b, v14.16b, v15.16b, #8 \n" + + "st1 {v12.4s}, [%3], #16 \n" + "st1 {v13.4s}, [%4], #16 \n" + + "subs %w0, %w0, #1 \n" + "bne 0b \n" + + "sub %5, %5, #16 \n" + "sub %8, %8, #16 \n" + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(outptr0n), // %3 + "=r"(outptr1n), // %4 + "=r"(r0), // %5 + "=r"(r1), // %6 + "=r"(r2), // %7 + "=r"(r3) // %8 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(outptr0n), + "4"(outptr1n), + "5"(r0), + "6"(r1), + "7"(r2), + "8"(r3), + "w"(_k00), // %18 + "w"(_k03), // %19 + "w"(_k06), // %20 + "w"(_k10), // %21 + "w"(_k13), // %22 + "w"(_k16) // %23 + : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + ); } #else if (nn > 0) @@ -441,54 +503,94 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke #if __ARM_NEON #if __aarch64__ - for (; nn>0; nn--) + if (nn > 0) { - float32x4_t _sum0 = vld1q_f32(outptr0); - float32x4_t _sum1 = vld1q_f32(outptr1); + asm volatile( + "0: \n" - float32x4_t _r00 = vld1q_f32(r0); - float32x4_t _r00n = vld1q_f32(r0 + 4); - float32x4_t _r01 = vextq_f32(_r00, _r00n, 1); - float32x4_t _r02 = vextq_f32(_r00, _r00n, 2); + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4s, v9.4s}, [%3] \n"// r0 + "add %3, %3, #16 \n" - float32x4_t _r10 = vld1q_f32(r1); - float32x4_t _r10n = vld1q_f32(r1 + 4); - float32x4_t _r11 = vextq_f32(_r10, _r10n, 1); - float32x4_t _r12 = vextq_f32(_r10, _r10n, 2); + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v6.4s}, [%1] \n"// _sum0 - float32x4_t _r20 = vld1q_f32(r2); - float32x4_t _r20n = vld1q_f32(r2 + 4); - float32x4_t _r21 = vextq_f32(_r20, _r20n, 1); - float32x4_t _r22 = vextq_f32(_r20, _r20n, 2); - - _sum0 = vfmaq_laneq_f32(_sum0, _r00, _k00, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r01, _k00, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r02, _k00, 2); - _sum0 = vfmaq_laneq_f32(_sum0, _r10, _k03, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r11, _k03, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r12, _k03, 2); - _sum0 = vfmaq_laneq_f32(_sum0, _r20, _k06, 0); - _sum0 = vfmaq_laneq_f32(_sum0, _r21, _k06, 1); - _sum0 = vfmaq_laneq_f32(_sum0, _r22, _k06, 2); - - _sum1 = vfmaq_laneq_f32(_sum1, _r00, _k10, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r01, _k10, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r02, _k10, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r10, _k13, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r11, _k13, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r12, _k13, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r20, _k16, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r21, _k16, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r22, _k16, 2); - - vst1q_f32(outptr0, _sum0); - vst1q_f32(outptr1, _sum1); + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v7.4s}, [%2] \n"// _sum1 - r0 += 4; - r1 += 4; - r2 += 4; - outptr0 += 4; - outptr1 += 4; + "fmul v14.4s, v8.4s, %12.s[0] \n" + "fmul v15.4s, v8.4s, %15.s[0] \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v6.4s, v10.4s, %12.s[1] \n" + "fmla v7.4s, v10.4s, %15.s[1] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4s, v9.4s}, [%4] \n"// r1 + "add %4, %4, #16 \n" + + "fmla v14.4s, v11.4s, %12.s[2] \n" + "fmla v15.4s, v11.4s, %15.s[2] \n" + + "fmla v6.4s, v8.4s, %13.s[0] \n" + "fmla v7.4s, v8.4s, %16.s[0] \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v14.4s, v10.4s, %13.s[1] \n" + "fmla v15.4s, v10.4s, %16.s[1] \n" + + "prfm pldl1keep, [%5, #256] \n" + "ld1 {v8.4s, v9.4s}, [%5] \n"// r2 + "add %5, %5, #16 \n" + + "fmla v6.4s, v11.4s, %13.s[2] \n" + "fmla v7.4s, v11.4s, %16.s[2] \n" + + "fmla v14.4s, v8.4s, %14.s[0] \n" + "fmla v15.4s, v8.4s, %17.s[0] \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v6.4s, v10.4s, %14.s[1] \n" + "fmla v7.4s, v10.4s, %17.s[1] \n" + + "fmla v14.4s, v11.4s, %14.s[2] \n" + "fmla v15.4s, v11.4s, %17.s[2] \n" + + "fadd v6.4s, v6.4s, v14.4s \n" + "fadd v7.4s, v7.4s, v15.4s \n" + + "st1 {v6.4s}, [%1], #16 \n" + "st1 {v7.4s}, [%2], #16 \n" + + "subs %w0, %w0, #1 \n" + "bne 0b \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(r0), // %3 + "=r"(r1), // %4 + "=r"(r2) // %5 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(r0), + "4"(r1), + "5"(r2), + "w"(_k00), // %12 + "w"(_k03), // %13 + "w"(_k06), // %14 + "w"(_k10), // %15 + "w"(_k13), // %16 + "w"(_k16) // %17 + : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + ); } #else if (nn > 0) @@ -701,63 +803,112 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke #if __ARM_NEON #if __aarch64__ - for (; nn>0; nn--) + if (nn > 0) { - float32x4_t _sum1 = vld1q_f32(outptr); - float32x4_t _sum3 = vld1q_f32(outptr2); + asm volatile( + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v9.4s, v10.4s}, [%3] \n"// r0 + "add %3, %3, #16 \n" - float32x4_t _r00 = vld1q_f32(r0); - float32x4_t _r00n = vld1q_f32(r0 + 4); - float32x4_t _r01 = vextq_f32(_r00, _r00n, 1); - float32x4_t _r02 = vextq_f32(_r00, _r00n, 2); + "ext v11.16b, v9.16b, v10.16b, #4 \n" + "ext v12.16b, v9.16b, v10.16b, #8 \n" - float32x4_t _r10 = vld1q_f32(r1); - float32x4_t _r10n = vld1q_f32(r1 + 4); - float32x4_t _r11 = vextq_f32(_r10, _r10n, 1); - float32x4_t _r12 = vextq_f32(_r10, _r10n, 2); + "0: \n" - float32x4_t _r20 = vld1q_f32(r2); - float32x4_t _r20n = vld1q_f32(r2 + 4); - float32x4_t _r21 = vextq_f32(_r20, _r20n, 1); - float32x4_t _r22 = vextq_f32(_r20, _r20n, 2); + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v7.4s}, [%1] \n"// _sum - float32x4_t _r30 = vld1q_f32(r3); - float32x4_t _r30n = vld1q_f32(r3 + 4); - float32x4_t _r31 = vextq_f32(_r30, _r30n, 1); - float32x4_t _r32 = vextq_f32(_r30, _r30n, 2); - - _sum1 = vfmaq_laneq_f32(_sum1, _r00, _k0123, 0); - float32x4_t _sum2 = vmulq_laneq_f32(_r01, _k0123, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r02, _k0123, 2); - _sum2 = vfmaq_laneq_f32(_sum2, _r10, _k3456, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r11, _k3456, 1); - _sum2 = vfmaq_laneq_f32(_sum2, _r12, _k3456, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r20, _k6789, 0); - _sum2 = vfmaq_laneq_f32(_sum2, _r21, _k6789, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r22, _k6789, 2); - - _sum3 = vfmaq_laneq_f32(_sum3, _r10, _k0123, 0); - float32x4_t _sum4 = vmulq_laneq_f32(_r11, _k0123, 1); - _sum3 = vfmaq_laneq_f32(_sum3, _r12, _k0123, 2); - _sum4 = vfmaq_laneq_f32(_sum4, _r20, _k3456, 0); - _sum3 = vfmaq_laneq_f32(_sum3, _r21, _k3456, 1); - _sum4 = vfmaq_laneq_f32(_sum4, _r22, _k3456, 2); - _sum3 = vfmaq_laneq_f32(_sum3, _r30, _k6789, 0); - _sum4 = vfmaq_laneq_f32(_sum4, _r31, _k6789, 1); - _sum3 = vfmaq_laneq_f32(_sum3, _r32, _k6789, 2); - - _sum1 = vaddq_f32(_sum1, _sum2); - _sum3 = vaddq_f32(_sum3, _sum4); - - vst1q_f32(outptr, _sum1); - vst1q_f32(outptr2, _sum3); + "fmla v7.4s, v9.4s, %14.s[0] \n" + "fmul v6.4s, v11.4s, %14.s[1] \n" + "fmul v13.4s, v12.4s, %14.s[2] \n" - r0 += 4; - r1 += 4; - r2 += 4; - r3 += 4; - outptr += 4; - outptr2 += 4; + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v9.4s, v10.4s}, [%4] \n"// r1 + "add %4, %4, #16 \n" + + "fmla v7.4s, v9.4s, %15.s[0] \n" + + "ext v11.16b, v9.16b, v10.16b, #4 \n" + "ext v12.16b, v9.16b, v10.16b, #8 \n" + + "fmla v6.4s, v11.4s, %15.s[1] \n" + "fmla v13.4s, v12.4s, %15.s[2] \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v8.4s}, [%2] \n"// _sum2 + + "fmla v8.4s, v9.4s, %14.s[0] \n" + "fmul v14.4s, v11.4s, %14.s[1] \n" + "fmul v15.4s, v12.4s, %14.s[2] \n" + + "prfm pldl1keep, [%5, #256] \n" + "ld1 {v9.4s, v10.4s}, [%5] \n"// r2 + "add %5, %5, #16 \n" + + "fmla v7.4s, v9.4s, %16.s[0] \n" + + "ext v11.16b, v9.16b, v10.16b, #4 \n" + "ext v12.16b, v9.16b, v10.16b, #8 \n" + + "fmla v6.4s, v11.4s, %16.s[1] \n" + "fmla v13.4s, v12.4s, %16.s[2] \n" + + "fmla v8.4s, v9.4s, %15.s[0] \n" + "fmla v14.4s, v11.4s, %15.s[1] \n" + "fmla v15.4s, v12.4s, %15.s[2] \n" + + "prfm pldl1keep, [%6, #256] \n" + "ld1 {v9.4s, v10.4s}, [%6] \n"// r3 + "add %6, %6, #16 \n" + + "fmla v8.4s, v9.4s, %16.s[0] \n" + + "ext v11.16b, v9.16b, v10.16b, #4 \n" + "ext v12.16b, v9.16b, v10.16b, #8 \n" + + "fmla v14.4s, v11.4s, %16.s[1] \n" + "fmla v15.4s, v12.4s, %16.s[2] \n" + + "fadd v7.4s, v7.4s, v6.4s \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v9.4s, v10.4s}, [%3] \n"// r0 + + "fadd v8.4s, v8.4s, v14.4s \n" + "fadd v7.4s, v7.4s, v13.4s \n" + "fadd v8.4s, v8.4s, v15.4s \n" + + "ext v11.16b, v9.16b, v10.16b, #4 \n" + "ext v12.16b, v9.16b, v10.16b, #8 \n" + + "add %3, %3, #16 \n" + + "st1 {v7.4s}, [%1], #16 \n" + "st1 {v8.4s}, [%2], #16 \n" + + "subs %w0, %w0, #1 \n" + "bne 0b \n" + + "sub %3, %3, #16 \n" + : "=r"(nn), // %0 + "=r"(outptr), // %1 + "=r"(outptr2), // %2 + "=r"(r0), // %3 + "=r"(r1), // %4 + "=r"(r2), // %5 + "=r"(r3) // %6 + : "0"(nn), + "1"(outptr), + "2"(outptr2), + "3"(r0), + "4"(r1), + "5"(r2), + "6"(r3), + "w"(_k0123), // %14 + "w"(_k3456), // %15 + "w"(_k6789) // %16 + : "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + ); } #else if (nn > 0) @@ -956,43 +1107,80 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke #if __ARM_NEON #if __aarch64__ - for (; nn>0; nn--) + if (nn > 0) { - float32x4_t _sum1 = vld1q_f32(outptr); + asm volatile( + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v8.4s, v9.4s}, [%2] \n"// r0 + "add %2, %2, #16 \n" - float32x4_t _r00 = vld1q_f32(r0); - float32x4_t _r00n = vld1q_f32(r0 + 4); - float32x4_t _r01 = vextq_f32(_r00, _r00n, 1); - float32x4_t _r02 = vextq_f32(_r00, _r00n, 2); + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" - float32x4_t _r10 = vld1q_f32(r1); - float32x4_t _r10n = vld1q_f32(r1 + 4); - float32x4_t _r11 = vextq_f32(_r10, _r10n, 1); - float32x4_t _r12 = vextq_f32(_r10, _r10n, 2); + "0: \n" - float32x4_t _r20 = vld1q_f32(r2); - float32x4_t _r20n = vld1q_f32(r2 + 4); - float32x4_t _r21 = vextq_f32(_r20, _r20n, 1); - float32x4_t _r22 = vextq_f32(_r20, _r20n, 2); + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v7.4s}, [%1] \n"// _sum - _sum1 = vfmaq_laneq_f32(_sum1, _r00, _k0123, 0); - float32x4_t _sum2 = vmulq_laneq_f32(_r01, _k0123, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r02, _k0123, 2); - _sum2 = vfmaq_laneq_f32(_sum2, _r10, _k3456, 0); - _sum1 = vfmaq_laneq_f32(_sum1, _r11, _k3456, 1); - _sum2 = vfmaq_laneq_f32(_sum2, _r12, _k3456, 2); - _sum1 = vfmaq_laneq_f32(_sum1, _r20, _k6789, 0); - _sum2 = vfmaq_laneq_f32(_sum2, _r21, _k6789, 1); - _sum1 = vfmaq_laneq_f32(_sum1, _r22, _k6789, 2); + "fmla v7.4s, v8.4s, %10.s[0] \n" + "fmul v13.4s, v10.4s, %10.s[1] \n" + "fmul v14.4s, v11.4s, %10.s[2] \n" - _sum1 = vaddq_f32(_sum1, _sum2); + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4s, v9.4s}, [%3] \n"// r1 + "add %3, %3, #16 \n" - vst1q_f32(outptr, _sum1); + "fmla v7.4s, v8.4s, %11.s[0] \n" - r0 += 4; - r1 += 4; - r2 += 4; - outptr += 4; + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v13.4s, v10.4s, %11.s[1] \n" + "fmla v14.4s, v11.4s, %11.s[2] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4s, v9.4s}, [%4] \n"// r2 + "add %4, %4, #16 \n" + + "fmla v7.4s, v8.4s, %12.s[0] \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "fmla v13.4s, v10.4s, %12.s[1] \n" + "fmla v14.4s, v11.4s, %12.s[2] \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v8.4s, v9.4s}, [%2] \n"// r0 + "add %2, %2, #16 \n" + + "fadd v7.4s, v7.4s, v13.4s \n" + "fadd v7.4s, v7.4s, v14.4s \n" + + "ext v10.16b, v8.16b, v9.16b, #4 \n" + "ext v11.16b, v8.16b, v9.16b, #8 \n" + + "st1 {v7.4s}, [%1], #16 \n" + + "subs %w0, %w0, #1 \n" + "bne 0b \n" + + "sub %2, %2, #16 \n" + : "=r"(nn), // %0 + "=r"(outptr), // %1 + "=r"(r0), // %2 + "=r"(r1), // %3 + "=r"(r2) // %4 + : "0"(nn), + "1"(outptr), + "2"(r0), + "3"(r1), + "4"(r2), + "w"(_k0123), // %10 + "w"(_k3456), // %11 + "w"(_k6789) // %12 + : "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + ); } #else if (nn > 0)