nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

aarch64 assembly for conv3x3s1, about 18% speed gain

tags/20181228
nihui 7 years ago
parent
5f1e678e7f
commit
a862a3432d
1 changed files with 385 additions and 197 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +385
    -197
      src/layer/arm/convolution_3x3.h

+ 385
- 197
src/layer/arm/convolution_3x3.h View File

@@ -85,86 +85,148 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke

#if __ARM_NEON
#if __aarch64__
for (; nn>0; nn--)
if (nn > 0)
{
float32x4_t _sum0 = vld1q_f32(outptr0);
float32x4_t _sum1 = vld1q_f32(outptr1);
float32x4_t _sum0n = vld1q_f32(outptr0n);
float32x4_t _sum1n = vld1q_f32(outptr1n);
asm volatile(
"prfm pldl1keep, [%5, #256] \n"
"ld1 {v8.4s, v9.4s}, [%5] \n"// r0
"add %5, %5, #16 \n"

float32x4_t _r00 = vld1q_f32(r0);
float32x4_t _r00n = vld1q_f32(r0 + 4);
float32x4_t _r01 = vextq_f32(_r00, _r00n, 1);
float32x4_t _r02 = vextq_f32(_r00, _r00n, 2);
"prfm pldl1keep, [%8, #256] \n"
"ld1 {v14.4s, v15.4s}, [%8] \n"// r3
"add %8, %8, #16 \n"

float32x4_t _r10 = vld1q_f32(r1);
float32x4_t _r10n = vld1q_f32(r1 + 4);
float32x4_t _r11 = vextq_f32(_r10, _r10n, 1);
float32x4_t _r12 = vextq_f32(_r10, _r10n, 2);
"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v14.16b, v15.16b, #8 \n"

float32x4_t _r20 = vld1q_f32(r2);
float32x4_t _r20n = vld1q_f32(r2 + 4);
float32x4_t _r21 = vextq_f32(_r20, _r20n, 1);
float32x4_t _r22 = vextq_f32(_r20, _r20n, 2);
"0: \n"

float32x4_t _r30 = vld1q_f32(r3);
float32x4_t _r30n = vld1q_f32(r3 + 4);
float32x4_t _r31 = vextq_f32(_r30, _r30n, 1);
float32x4_t _r32 = vextq_f32(_r30, _r30n, 2);

_sum0 = vfmaq_laneq_f32(_sum0, _r00, _k00, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r01, _k00, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r02, _k00, 2);
_sum0 = vfmaq_laneq_f32(_sum0, _r10, _k03, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r11, _k03, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r12, _k03, 2);
_sum0 = vfmaq_laneq_f32(_sum0, _r20, _k06, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r21, _k06, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r22, _k06, 2);

_sum1 = vfmaq_laneq_f32(_sum1, _r00, _k10, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r01, _k10, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r02, _k10, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r10, _k13, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r11, _k13, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r12, _k13, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r20, _k16, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r21, _k16, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r22, _k16, 2);

_sum0n = vfmaq_laneq_f32(_sum0n, _r10, _k00, 0);
_sum0n = vfmaq_laneq_f32(_sum0n, _r11, _k00, 1);
_sum0n = vfmaq_laneq_f32(_sum0n, _r12, _k00, 2);
_sum0n = vfmaq_laneq_f32(_sum0n, _r20, _k03, 0);
_sum0n = vfmaq_laneq_f32(_sum0n, _r21, _k03, 1);
_sum0n = vfmaq_laneq_f32(_sum0n, _r22, _k03, 2);
_sum0n = vfmaq_laneq_f32(_sum0n, _r30, _k06, 0);
_sum0n = vfmaq_laneq_f32(_sum0n, _r31, _k06, 1);
_sum0n = vfmaq_laneq_f32(_sum0n, _r32, _k06, 2);

_sum1n = vfmaq_laneq_f32(_sum1n, _r10, _k10, 0);
_sum1n = vfmaq_laneq_f32(_sum1n, _r11, _k10, 1);
_sum1n = vfmaq_laneq_f32(_sum1n, _r12, _k10, 2);
_sum1n = vfmaq_laneq_f32(_sum1n, _r20, _k13, 0);
_sum1n = vfmaq_laneq_f32(_sum1n, _r21, _k13, 1);
_sum1n = vfmaq_laneq_f32(_sum1n, _r22, _k13, 2);
_sum1n = vfmaq_laneq_f32(_sum1n, _r30, _k16, 0);
_sum1n = vfmaq_laneq_f32(_sum1n, _r31, _k16, 1);
_sum1n = vfmaq_laneq_f32(_sum1n, _r32, _k16, 2);

vst1q_f32(outptr0, _sum0);
vst1q_f32(outptr1, _sum1);
vst1q_f32(outptr0n, _sum0n);
vst1q_f32(outptr1n, _sum1n);
"prfm pldl1keep, [%1, #128] \n"
"ld1 {v6.4s}, [%1] \n"// _sum0

r0 += 4;
r1 += 4;
r2 += 4;
r3 += 4;
outptr0 += 4;
outptr1 += 4;
outptr0n += 4;
outptr1n += 4;
"prfm pldl1keep, [%2, #128] \n"
"ld1 {v7.4s}, [%2] \n"// _sum1

"fmla v6.4s, v8.4s, %18.s[0] \n"
"fmla v7.4s, v8.4s, %21.s[0] \n"

"prfm pldl1keep, [%3, #128] \n"
"ld1 {v12.4s}, [%3] \n"// _sum0n

"prfm pldl1keep, [%4, #128] \n"
"ld1 {v13.4s}, [%4] \n"// _sum1n

"fmla v12.4s, v14.4s, %20.s[0] \n"
"fmla v13.4s, v14.4s, %23.s[0] \n"

"ext v8.16b, v8.16b, v9.16b, #8 \n"
"ext v9.16b, v14.16b, v15.16b, #4 \n"

"fmla v6.4s, v10.4s, %18.s[1] \n"
"fmla v7.4s, v10.4s, %21.s[1] \n"
"fmla v12.4s, v11.4s, %20.s[2] \n"
"fmla v13.4s, v11.4s, %23.s[2] \n"

"prfm pldl1keep, [%6, #256] \n"
"ld1 {v14.4s, v15.4s}, [%6] \n"// r1
"add %6, %6, #16 \n"

"fmla v6.4s, v8.4s, %18.s[2] \n"
"fmla v7.4s, v8.4s, %21.s[2] \n"
"fmla v12.4s, v9.4s, %20.s[1] \n"
"fmla v13.4s, v9.4s, %23.s[1] \n"

"ext v10.16b, v14.16b, v15.16b, #4 \n"

"fmla v6.4s, v14.4s, %19.s[0] \n"
"fmla v7.4s, v14.4s, %22.s[0] \n"
"fmla v12.4s, v14.4s, %18.s[0] \n"
"fmla v13.4s, v14.4s, %21.s[0] \n"

"ext v11.16b, v14.16b, v15.16b, #8 \n"

"fmla v6.4s, v10.4s, %19.s[1] \n"
"fmla v7.4s, v10.4s, %22.s[1] \n"
"fmla v12.4s, v10.4s, %18.s[1] \n"
"fmla v13.4s, v10.4s, %21.s[1] \n"

"prfm pldl1keep, [%7, #256] \n"
"ld1 {v8.4s, v9.4s}, [%7] \n"// r2
"add %7, %7, #16 \n"

"fmla v6.4s, v11.4s, %19.s[2] \n"
"fmla v7.4s, v11.4s, %22.s[2] \n"
"fmla v12.4s, v11.4s, %18.s[2] \n"
"fmla v13.4s, v11.4s, %21.s[2] \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"

"fmla v6.4s, v8.4s, %20.s[0] \n"
"fmla v7.4s, v8.4s, %23.s[0] \n"
"fmla v12.4s, v8.4s, %19.s[0] \n"
"fmla v13.4s, v8.4s, %22.s[0] \n"

"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v6.4s, v10.4s, %20.s[1] \n"
"fmla v7.4s, v10.4s, %23.s[1] \n"
"fmla v12.4s, v10.4s, %19.s[1] \n"
"fmla v13.4s, v10.4s, %22.s[1] \n"

"prfm pldl1keep, [%5, #256] \n"
"ld1 {v8.4s, v9.4s}, [%5] \n"// r0
"add %5, %5, #16 \n"

"fmla v6.4s, v11.4s, %20.s[2] \n"
"fmla v7.4s, v11.4s, %23.s[2] \n"
"fmla v12.4s, v11.4s, %19.s[2] \n"
"fmla v13.4s, v11.4s, %22.s[2] \n"

"prfm pldl1keep, [%8, #256] \n"
"ld1 {v14.4s, v15.4s}, [%8] \n"// r3
"add %8, %8, #16 \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"

"st1 {v6.4s}, [%1], #16 \n"
"st1 {v7.4s}, [%2], #16 \n"

"ext v11.16b, v14.16b, v15.16b, #8 \n"

"st1 {v12.4s}, [%3], #16 \n"
"st1 {v13.4s}, [%4], #16 \n"

"subs %w0, %w0, #1 \n"
"bne 0b \n"

"sub %5, %5, #16 \n"
"sub %8, %8, #16 \n"
: "=r"(nn), // %0
"=r"(outptr0), // %1
"=r"(outptr1), // %2
"=r"(outptr0n), // %3
"=r"(outptr1n), // %4
"=r"(r0), // %5
"=r"(r1), // %6
"=r"(r2), // %7
"=r"(r3) // %8
: "0"(nn),
"1"(outptr0),
"2"(outptr1),
"3"(outptr0n),
"4"(outptr1n),
"5"(r0),
"6"(r1),
"7"(r2),
"8"(r3),
"w"(_k00), // %18
"w"(_k03), // %19
"w"(_k06), // %20
"w"(_k10), // %21
"w"(_k13), // %22
"w"(_k16) // %23
: "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
);
}
#else
if (nn > 0)
@@ -441,54 +503,94 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke

#if __ARM_NEON
#if __aarch64__
for (; nn>0; nn--)
if (nn > 0)
{
float32x4_t _sum0 = vld1q_f32(outptr0);
float32x4_t _sum1 = vld1q_f32(outptr1);
asm volatile(
"0: \n"

float32x4_t _r00 = vld1q_f32(r0);
float32x4_t _r00n = vld1q_f32(r0 + 4);
float32x4_t _r01 = vextq_f32(_r00, _r00n, 1);
float32x4_t _r02 = vextq_f32(_r00, _r00n, 2);
"prfm pldl1keep, [%3, #256] \n"
"ld1 {v8.4s, v9.4s}, [%3] \n"// r0
"add %3, %3, #16 \n"

float32x4_t _r10 = vld1q_f32(r1);
float32x4_t _r10n = vld1q_f32(r1 + 4);
float32x4_t _r11 = vextq_f32(_r10, _r10n, 1);
float32x4_t _r12 = vextq_f32(_r10, _r10n, 2);
"prfm pldl1keep, [%1, #128] \n"
"ld1 {v6.4s}, [%1] \n"// _sum0

float32x4_t _r20 = vld1q_f32(r2);
float32x4_t _r20n = vld1q_f32(r2 + 4);
float32x4_t _r21 = vextq_f32(_r20, _r20n, 1);
float32x4_t _r22 = vextq_f32(_r20, _r20n, 2);

_sum0 = vfmaq_laneq_f32(_sum0, _r00, _k00, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r01, _k00, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r02, _k00, 2);
_sum0 = vfmaq_laneq_f32(_sum0, _r10, _k03, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r11, _k03, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r12, _k03, 2);
_sum0 = vfmaq_laneq_f32(_sum0, _r20, _k06, 0);
_sum0 = vfmaq_laneq_f32(_sum0, _r21, _k06, 1);
_sum0 = vfmaq_laneq_f32(_sum0, _r22, _k06, 2);

_sum1 = vfmaq_laneq_f32(_sum1, _r00, _k10, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r01, _k10, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r02, _k10, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r10, _k13, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r11, _k13, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r12, _k13, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r20, _k16, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r21, _k16, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r22, _k16, 2);

vst1q_f32(outptr0, _sum0);
vst1q_f32(outptr1, _sum1);
"prfm pldl1keep, [%2, #128] \n"
"ld1 {v7.4s}, [%2] \n"// _sum1

r0 += 4;
r1 += 4;
r2 += 4;
outptr0 += 4;
outptr1 += 4;
"fmul v14.4s, v8.4s, %12.s[0] \n"
"fmul v15.4s, v8.4s, %15.s[0] \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v6.4s, v10.4s, %12.s[1] \n"
"fmla v7.4s, v10.4s, %15.s[1] \n"

"prfm pldl1keep, [%4, #256] \n"
"ld1 {v8.4s, v9.4s}, [%4] \n"// r1
"add %4, %4, #16 \n"

"fmla v14.4s, v11.4s, %12.s[2] \n"
"fmla v15.4s, v11.4s, %15.s[2] \n"

"fmla v6.4s, v8.4s, %13.s[0] \n"
"fmla v7.4s, v8.4s, %16.s[0] \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v14.4s, v10.4s, %13.s[1] \n"
"fmla v15.4s, v10.4s, %16.s[1] \n"

"prfm pldl1keep, [%5, #256] \n"
"ld1 {v8.4s, v9.4s}, [%5] \n"// r2
"add %5, %5, #16 \n"

"fmla v6.4s, v11.4s, %13.s[2] \n"
"fmla v7.4s, v11.4s, %16.s[2] \n"

"fmla v14.4s, v8.4s, %14.s[0] \n"
"fmla v15.4s, v8.4s, %17.s[0] \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v6.4s, v10.4s, %14.s[1] \n"
"fmla v7.4s, v10.4s, %17.s[1] \n"

"fmla v14.4s, v11.4s, %14.s[2] \n"
"fmla v15.4s, v11.4s, %17.s[2] \n"

"fadd v6.4s, v6.4s, v14.4s \n"
"fadd v7.4s, v7.4s, v15.4s \n"

"st1 {v6.4s}, [%1], #16 \n"
"st1 {v7.4s}, [%2], #16 \n"

"subs %w0, %w0, #1 \n"
"bne 0b \n"

: "=r"(nn), // %0
"=r"(outptr0), // %1
"=r"(outptr1), // %2
"=r"(r0), // %3
"=r"(r1), // %4
"=r"(r2) // %5
: "0"(nn),
"1"(outptr0),
"2"(outptr1),
"3"(r0),
"4"(r1),
"5"(r2),
"w"(_k00), // %12
"w"(_k03), // %13
"w"(_k06), // %14
"w"(_k10), // %15
"w"(_k13), // %16
"w"(_k16) // %17
: "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
);
}
#else
if (nn > 0)
@@ -701,63 +803,112 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke

#if __ARM_NEON
#if __aarch64__
for (; nn>0; nn--)
if (nn > 0)
{
float32x4_t _sum1 = vld1q_f32(outptr);
float32x4_t _sum3 = vld1q_f32(outptr2);
asm volatile(
"prfm pldl1keep, [%3, #256] \n"
"ld1 {v9.4s, v10.4s}, [%3] \n"// r0
"add %3, %3, #16 \n"

float32x4_t _r00 = vld1q_f32(r0);
float32x4_t _r00n = vld1q_f32(r0 + 4);
float32x4_t _r01 = vextq_f32(_r00, _r00n, 1);
float32x4_t _r02 = vextq_f32(_r00, _r00n, 2);
"ext v11.16b, v9.16b, v10.16b, #4 \n"
"ext v12.16b, v9.16b, v10.16b, #8 \n"

float32x4_t _r10 = vld1q_f32(r1);
float32x4_t _r10n = vld1q_f32(r1 + 4);
float32x4_t _r11 = vextq_f32(_r10, _r10n, 1);
float32x4_t _r12 = vextq_f32(_r10, _r10n, 2);
"0: \n"

float32x4_t _r20 = vld1q_f32(r2);
float32x4_t _r20n = vld1q_f32(r2 + 4);
float32x4_t _r21 = vextq_f32(_r20, _r20n, 1);
float32x4_t _r22 = vextq_f32(_r20, _r20n, 2);
"prfm pldl1keep, [%1, #128] \n"
"ld1 {v7.4s}, [%1] \n"// _sum

float32x4_t _r30 = vld1q_f32(r3);
float32x4_t _r30n = vld1q_f32(r3 + 4);
float32x4_t _r31 = vextq_f32(_r30, _r30n, 1);
float32x4_t _r32 = vextq_f32(_r30, _r30n, 2);

_sum1 = vfmaq_laneq_f32(_sum1, _r00, _k0123, 0);
float32x4_t _sum2 = vmulq_laneq_f32(_r01, _k0123, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r02, _k0123, 2);
_sum2 = vfmaq_laneq_f32(_sum2, _r10, _k3456, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r11, _k3456, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _r12, _k3456, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r20, _k6789, 0);
_sum2 = vfmaq_laneq_f32(_sum2, _r21, _k6789, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r22, _k6789, 2);

_sum3 = vfmaq_laneq_f32(_sum3, _r10, _k0123, 0);
float32x4_t _sum4 = vmulq_laneq_f32(_r11, _k0123, 1);
_sum3 = vfmaq_laneq_f32(_sum3, _r12, _k0123, 2);
_sum4 = vfmaq_laneq_f32(_sum4, _r20, _k3456, 0);
_sum3 = vfmaq_laneq_f32(_sum3, _r21, _k3456, 1);
_sum4 = vfmaq_laneq_f32(_sum4, _r22, _k3456, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _r30, _k6789, 0);
_sum4 = vfmaq_laneq_f32(_sum4, _r31, _k6789, 1);
_sum3 = vfmaq_laneq_f32(_sum3, _r32, _k6789, 2);

_sum1 = vaddq_f32(_sum1, _sum2);
_sum3 = vaddq_f32(_sum3, _sum4);

vst1q_f32(outptr, _sum1);
vst1q_f32(outptr2, _sum3);
"fmla v7.4s, v9.4s, %14.s[0] \n"
"fmul v6.4s, v11.4s, %14.s[1] \n"
"fmul v13.4s, v12.4s, %14.s[2] \n"

r0 += 4;
r1 += 4;
r2 += 4;
r3 += 4;
outptr += 4;
outptr2 += 4;
"prfm pldl1keep, [%4, #256] \n"
"ld1 {v9.4s, v10.4s}, [%4] \n"// r1
"add %4, %4, #16 \n"

"fmla v7.4s, v9.4s, %15.s[0] \n"

"ext v11.16b, v9.16b, v10.16b, #4 \n"
"ext v12.16b, v9.16b, v10.16b, #8 \n"

"fmla v6.4s, v11.4s, %15.s[1] \n"
"fmla v13.4s, v12.4s, %15.s[2] \n"

"prfm pldl1keep, [%2, #128] \n"
"ld1 {v8.4s}, [%2] \n"// _sum2

"fmla v8.4s, v9.4s, %14.s[0] \n"
"fmul v14.4s, v11.4s, %14.s[1] \n"
"fmul v15.4s, v12.4s, %14.s[2] \n"

"prfm pldl1keep, [%5, #256] \n"
"ld1 {v9.4s, v10.4s}, [%5] \n"// r2
"add %5, %5, #16 \n"

"fmla v7.4s, v9.4s, %16.s[0] \n"

"ext v11.16b, v9.16b, v10.16b, #4 \n"
"ext v12.16b, v9.16b, v10.16b, #8 \n"

"fmla v6.4s, v11.4s, %16.s[1] \n"
"fmla v13.4s, v12.4s, %16.s[2] \n"

"fmla v8.4s, v9.4s, %15.s[0] \n"
"fmla v14.4s, v11.4s, %15.s[1] \n"
"fmla v15.4s, v12.4s, %15.s[2] \n"

"prfm pldl1keep, [%6, #256] \n"
"ld1 {v9.4s, v10.4s}, [%6] \n"// r3
"add %6, %6, #16 \n"

"fmla v8.4s, v9.4s, %16.s[0] \n"

"ext v11.16b, v9.16b, v10.16b, #4 \n"
"ext v12.16b, v9.16b, v10.16b, #8 \n"

"fmla v14.4s, v11.4s, %16.s[1] \n"
"fmla v15.4s, v12.4s, %16.s[2] \n"

"fadd v7.4s, v7.4s, v6.4s \n"

"prfm pldl1keep, [%3, #256] \n"
"ld1 {v9.4s, v10.4s}, [%3] \n"// r0

"fadd v8.4s, v8.4s, v14.4s \n"
"fadd v7.4s, v7.4s, v13.4s \n"
"fadd v8.4s, v8.4s, v15.4s \n"

"ext v11.16b, v9.16b, v10.16b, #4 \n"
"ext v12.16b, v9.16b, v10.16b, #8 \n"

"add %3, %3, #16 \n"

"st1 {v7.4s}, [%1], #16 \n"
"st1 {v8.4s}, [%2], #16 \n"

"subs %w0, %w0, #1 \n"
"bne 0b \n"

"sub %3, %3, #16 \n"
: "=r"(nn), // %0
"=r"(outptr), // %1
"=r"(outptr2), // %2
"=r"(r0), // %3
"=r"(r1), // %4
"=r"(r2), // %5
"=r"(r3) // %6
: "0"(nn),
"1"(outptr),
"2"(outptr2),
"3"(r0),
"4"(r1),
"5"(r2),
"6"(r3),
"w"(_k0123), // %14
"w"(_k3456), // %15
"w"(_k6789) // %16
: "cc", "memory", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
);
}
#else
if (nn > 0)
@@ -956,43 +1107,80 @@ static void conv3x3s1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _ke

#if __ARM_NEON
#if __aarch64__
for (; nn>0; nn--)
if (nn > 0)
{
float32x4_t _sum1 = vld1q_f32(outptr);
asm volatile(
"prfm pldl1keep, [%2, #256] \n"
"ld1 {v8.4s, v9.4s}, [%2] \n"// r0
"add %2, %2, #16 \n"

float32x4_t _r00 = vld1q_f32(r0);
float32x4_t _r00n = vld1q_f32(r0 + 4);
float32x4_t _r01 = vextq_f32(_r00, _r00n, 1);
float32x4_t _r02 = vextq_f32(_r00, _r00n, 2);
"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

float32x4_t _r10 = vld1q_f32(r1);
float32x4_t _r10n = vld1q_f32(r1 + 4);
float32x4_t _r11 = vextq_f32(_r10, _r10n, 1);
float32x4_t _r12 = vextq_f32(_r10, _r10n, 2);
"0: \n"

float32x4_t _r20 = vld1q_f32(r2);
float32x4_t _r20n = vld1q_f32(r2 + 4);
float32x4_t _r21 = vextq_f32(_r20, _r20n, 1);
float32x4_t _r22 = vextq_f32(_r20, _r20n, 2);
"prfm pldl1keep, [%1, #128] \n"
"ld1 {v7.4s}, [%1] \n"// _sum

_sum1 = vfmaq_laneq_f32(_sum1, _r00, _k0123, 0);
float32x4_t _sum2 = vmulq_laneq_f32(_r01, _k0123, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r02, _k0123, 2);
_sum2 = vfmaq_laneq_f32(_sum2, _r10, _k3456, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _r11, _k3456, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _r12, _k3456, 2);
_sum1 = vfmaq_laneq_f32(_sum1, _r20, _k6789, 0);
_sum2 = vfmaq_laneq_f32(_sum2, _r21, _k6789, 1);
_sum1 = vfmaq_laneq_f32(_sum1, _r22, _k6789, 2);
"fmla v7.4s, v8.4s, %10.s[0] \n"
"fmul v13.4s, v10.4s, %10.s[1] \n"
"fmul v14.4s, v11.4s, %10.s[2] \n"

_sum1 = vaddq_f32(_sum1, _sum2);
"prfm pldl1keep, [%3, #256] \n"
"ld1 {v8.4s, v9.4s}, [%3] \n"// r1
"add %3, %3, #16 \n"

vst1q_f32(outptr, _sum1);
"fmla v7.4s, v8.4s, %11.s[0] \n"

r0 += 4;
r1 += 4;
r2 += 4;
outptr += 4;
"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v13.4s, v10.4s, %11.s[1] \n"
"fmla v14.4s, v11.4s, %11.s[2] \n"

"prfm pldl1keep, [%4, #256] \n"
"ld1 {v8.4s, v9.4s}, [%4] \n"// r2
"add %4, %4, #16 \n"

"fmla v7.4s, v8.4s, %12.s[0] \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"fmla v13.4s, v10.4s, %12.s[1] \n"
"fmla v14.4s, v11.4s, %12.s[2] \n"

"prfm pldl1keep, [%2, #256] \n"
"ld1 {v8.4s, v9.4s}, [%2] \n"// r0
"add %2, %2, #16 \n"

"fadd v7.4s, v7.4s, v13.4s \n"
"fadd v7.4s, v7.4s, v14.4s \n"

"ext v10.16b, v8.16b, v9.16b, #4 \n"
"ext v11.16b, v8.16b, v9.16b, #8 \n"

"st1 {v7.4s}, [%1], #16 \n"

"subs %w0, %w0, #1 \n"
"bne 0b \n"

"sub %2, %2, #16 \n"
: "=r"(nn), // %0
"=r"(outptr), // %1
"=r"(r0), // %2
"=r"(r1), // %3
"=r"(r2) // %4
: "0"(nn),
"1"(outptr),
"2"(r0),
"3"(r1),
"4"(r2),
"w"(_k0123), // %10
"w"(_k3456), // %11
"w"(_k6789) // %12
: "cc", "memory", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
);
}
#else
if (nn > 0)


Write Preview
Loading…
Cancel
Save