diff --git a/src/layer/arm/convolution_3x3_pack1to4.h b/src/layer/arm/convolution_3x3_pack1to4.h new file mode 100644 index 000000000..3e38ea732 --- /dev/null +++ b/src/layer/arm/convolution_3x3_pack1to4.h @@ -0,0 +1,574 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s2_pack1to4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int inch = bottom_blob.c; + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int tailstep = w - 2*outw + w; + + const float* bias = _bias; + + int nn_outch = 0; + int remain_outch_start = 0; + +#if __ARM_NEON && __aarch64__ + nn_outch = outch >> 1; + remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp=0; pp> 2; + int remain = outw & 3; + + if (nn > 0) + { + asm volatile( + "0: \n" + + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v6.4s, v7.4s, v8.4s, v9.4s}, [%1] \n"// sum0 + + // r0 + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4s, v1.4s}, [%3], #32 \n" + "ld1r {v4.4s}, [%3] \n" + + "fmla v6.4s, %12.4s, v0.s[0] \n" + "fmla v7.4s, %12.4s, v0.s[2] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v10.4s, v11.4s, v12.4s, v13.4s}, [%2] \n"// sum1 + + "fmla v8.4s, %12.4s, v1.s[0] \n" + "fmla v9.4s, %12.4s, v1.s[2] \n" + + "fmla v10.4s, %21.4s, v0.s[0] \n" + "fmla v11.4s, %21.4s, v0.s[2] \n" + "fmla v12.4s, %21.4s, v1.s[0] \n" + "fmla v13.4s, %21.4s, v1.s[2] \n" + + "fmla v6.4s, %13.4s, v0.s[1] \n" + "fmla v7.4s, %13.4s, v0.s[3] \n" + "fmla v8.4s, %13.4s, v1.s[1] \n" + "fmla v9.4s, %13.4s, v1.s[3] \n" + "fmla v10.4s, %22.4s, v0.s[1] \n" + "fmla v11.4s, %22.4s, v0.s[3] \n" + "fmla v12.4s, %22.4s, v1.s[1] \n" + "fmla v13.4s, %22.4s, v1.s[3] \n" + + // r1 + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v2.4s, v3.4s}, [%4], #32 \n" + "ld1r {v5.4s}, [%4] \n" + + "fmla v6.4s, %14.4s, v0.s[2] \n" + "fmla v7.4s, %14.4s, v1.s[0] \n" + "fmla v8.4s, %14.4s, v1.s[2] \n" + "fmla v9.4s, %14.4s, v4.s[0] \n" + "fmla v10.4s, %23.4s, v0.s[2] \n" + "fmla v11.4s, %23.4s, v1.s[0] \n" + "fmla v12.4s, %23.4s, v1.s[2] \n" + "fmla v13.4s, %23.4s, v4.s[0] \n" + + "fmla v6.4s, %15.4s, v2.s[0] \n" + "fmla v7.4s, %15.4s, v2.s[2] \n" + "fmla v8.4s, %15.4s, v3.s[0] \n" + "fmla v9.4s, %15.4s, v3.s[2] \n" + "fmla v10.4s, %24.4s, v2.s[0] \n" + "fmla v11.4s, %24.4s, v2.s[2] \n" + "fmla v12.4s, %24.4s, v3.s[0] \n" + "fmla v13.4s, %24.4s, v3.s[2] \n" + + "fmla v6.4s, %16.4s, v2.s[1] \n" + "fmla v7.4s, %16.4s, v2.s[3] \n" + "fmla v8.4s, %16.4s, v3.s[1] \n" + "fmla v9.4s, %16.4s, v3.s[3] \n" + "fmla v10.4s, %25.4s, v2.s[1] \n" + "fmla v11.4s, %25.4s, v2.s[3] \n" + "fmla v12.4s, %25.4s, v3.s[1] \n" + "fmla v13.4s, %25.4s, v3.s[3] \n" + + // r2 + "prfm pldl1keep, [%5, #256] \n" + "ld1 {v0.4s, v1.4s}, [%5], #32 \n" + "ld1r {v4.4s}, [%5] \n" + + "fmla v6.4s, %17.4s, v2.s[2] \n" + "fmla v7.4s, %17.4s, v3.s[0] \n" + "fmla v8.4s, %17.4s, v3.s[2] \n" + "fmla v9.4s, %17.4s, v5.s[0] \n" + "fmla v10.4s, %26.4s, v2.s[2] \n" + "fmla v11.4s, %26.4s, v3.s[0] \n" + "fmla v12.4s, %26.4s, v3.s[2] \n" + "fmla v13.4s, %26.4s, v5.s[0] \n" + + "fmla v6.4s, %18.4s, v0.s[0] \n" + "fmla v7.4s, %18.4s, v0.s[2] \n" + "fmla v8.4s, %18.4s, v1.s[0] \n" + "fmla v9.4s, %18.4s, v1.s[2] \n" + "fmla v10.4s, %27.4s, v0.s[0] \n" + "fmla v11.4s, %27.4s, v0.s[2] \n" + "fmla v12.4s, %27.4s, v1.s[0] \n" + "fmla v13.4s, %27.4s, v1.s[2] \n" + + "fmla v6.4s, %19.4s, v0.s[1] \n" + "fmla v7.4s, %19.4s, v0.s[3] \n" + "fmla v8.4s, %19.4s, v1.s[1] \n" + "fmla v9.4s, %19.4s, v1.s[3] \n" + "fmla v10.4s, %28.4s, v0.s[1] \n" + "fmla v11.4s, %28.4s, v0.s[3] \n" + "fmla v12.4s, %28.4s, v1.s[1] \n" + "fmla v13.4s, %28.4s, v1.s[3] \n" + + "fmla v6.4s, %20.4s, v0.s[2] \n" + "fmla v7.4s, %20.4s, v1.s[0] \n" + "fmla v8.4s, %20.4s, v1.s[2] \n" + "fmla v9.4s, %20.4s, v4.s[0] \n" + "fmla v10.4s, %29.4s, v0.s[2] \n" + "fmla v11.4s, %29.4s, v1.s[0] \n" + "fmla v12.4s, %29.4s, v1.s[2] \n" + "fmla v13.4s, %29.4s, v4.s[0] \n" + + "subs %w0, %w0, #1 \n" + + "st1 {v6.4s, v7.4s, v8.4s, v9.4s}, [%1], #64 \n" + "st1 {v10.4s, v11.4s, v12.4s, v13.4s}, [%2], #64 \n" + + "bne 0b \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(r0), // %3 + "=r"(r1), // %4 + "=r"(r2) // %5 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(r0), + "4"(r1), + "5"(r2), + "w"(_k00_0), // %12 + "w"(_k01_0), // %13 + "w"(_k02_0), // %14 + "w"(_k10_0), // %15 + "w"(_k11_0), // %16 + "w"(_k12_0), // %17 + "w"(_k20_0), // %18 + "w"(_k21_0), // %19 + "w"(_k22_0), // %20 + "w"(_k00_1), // %21 + "w"(_k01_1), // %22 + "w"(_k02_1), // %23 + "w"(_k10_1), // %24 + "w"(_k11_1), // %25 + "w"(_k12_1), // %26 + "w"(_k20_1), // %27 + "w"(_k21_1), // %28 + "w"(_k22_1) // %29 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13" + ); + } + + for (; remain>0; remain--) + { + float32x4_t _sum0 = vld1q_f32(outptr0); + float32x4_t _sum1 = vld1q_f32(outptr1); + + float32x4_t _r0 = vld1q_f32(r0); + float32x4_t _r1 = vld1q_f32(r1); + float32x4_t _r2 = vld1q_f32(r2); + + _sum0 = vfmaq_laneq_f32(_sum0, _k00_0, _r0, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k01_0, _r0, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k02_0, _r0, 2); + _sum0 = vfmaq_laneq_f32(_sum0, _k10_0, _r1, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k11_0, _r1, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k12_0, _r1, 2); + _sum0 = vfmaq_laneq_f32(_sum0, _k20_0, _r2, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k21_0, _r2, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k22_0, _r2, 2); + + _sum1 = vfmaq_laneq_f32(_sum1, _k00_1, _r0, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _k01_1, _r0, 1); + _sum1 = vfmaq_laneq_f32(_sum1, _k02_1, _r0, 2); + _sum1 = vfmaq_laneq_f32(_sum1, _k10_1, _r1, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _k11_1, _r1, 1); + _sum1 = vfmaq_laneq_f32(_sum1, _k12_1, _r1, 2); + _sum1 = vfmaq_laneq_f32(_sum1, _k20_1, _r2, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _k21_1, _r2, 1); + _sum1 = vfmaq_laneq_f32(_sum1, _k22_1, _r2, 2); + + vst1q_f32(outptr0, _sum0); + vst1q_f32(outptr1, _sum1); + + r0 += 2; + r1 += 2; + r2 += 2; + outptr0 += 4; + outptr1 += 4; + } + + r0 += tailstep; + r1 += tailstep; + r2 += tailstep; + } + + k0 += 9*4; + k1 += 9*4; + } + } +#endif // __ARM_NEON && __aarch64__ + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p=remain_outch_start; p> 2; + int remain = outw & 3; + +#if __aarch64__ + if (nn > 0) + { + asm volatile( + "0: \n" + + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v6.4s, v7.4s, v8.4s, v9.4s}, [%1] \n"// sum0 + + // r0 + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.4s, v1.4s}, [%2], #32 \n" + "ld1r {v4.4s}, [%2] \n" + + "fmla v6.4s, %10.4s, v0.s[0] \n" + "fmla v7.4s, %10.4s, v0.s[2] \n" + "fmla v8.4s, %10.4s, v1.s[0] \n" + "fmla v9.4s, %10.4s, v1.s[2] \n" + + "fmla v6.4s, %11.4s, v0.s[1] \n" + "fmla v7.4s, %11.4s, v0.s[3] \n" + "fmla v8.4s, %11.4s, v1.s[1] \n" + "fmla v9.4s, %11.4s, v1.s[3] \n" + + // r1 + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v2.4s, v3.4s}, [%3], #32 \n" + "ld1r {v5.4s}, [%3] \n" + + "fmla v6.4s, %12.4s, v0.s[2] \n" + "fmla v7.4s, %12.4s, v1.s[0] \n" + "fmla v8.4s, %12.4s, v1.s[2] \n" + "fmla v9.4s, %12.4s, v4.s[0] \n" + + "fmla v6.4s, %13.4s, v2.s[0] \n" + "fmla v7.4s, %13.4s, v2.s[2] \n" + "fmla v8.4s, %13.4s, v3.s[0] \n" + "fmla v9.4s, %13.4s, v3.s[2] \n" + + "fmla v6.4s, %14.4s, v2.s[1] \n" + "fmla v7.4s, %14.4s, v2.s[3] \n" + "fmla v8.4s, %14.4s, v3.s[1] \n" + "fmla v9.4s, %14.4s, v3.s[3] \n" + + // r2 + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v0.4s, v1.4s}, [%4], #32 \n" + "ld1r {v4.4s}, [%4] \n" + + "fmla v6.4s, %15.4s, v2.s[2] \n" + "fmla v7.4s, %15.4s, v3.s[0] \n" + "fmla v8.4s, %15.4s, v3.s[2] \n" + "fmla v9.4s, %15.4s, v5.s[0] \n" + + "fmla v6.4s, %16.4s, v0.s[0] \n" + "fmla v7.4s, %16.4s, v0.s[2] \n" + "fmla v8.4s, %16.4s, v1.s[0] \n" + "fmla v9.4s, %16.4s, v1.s[2] \n" + + "fmla v6.4s, %17.4s, v0.s[1] \n" + "fmla v7.4s, %17.4s, v0.s[3] \n" + "fmla v8.4s, %17.4s, v1.s[1] \n" + "fmla v9.4s, %17.4s, v1.s[3] \n" + + "fmla v6.4s, %18.4s, v0.s[2] \n" + "fmla v7.4s, %18.4s, v1.s[0] \n" + "fmla v8.4s, %18.4s, v1.s[2] \n" + "fmla v9.4s, %18.4s, v4.s[0] \n" + + "subs %w0, %w0, #1 \n" + + "st1 {v6.4s, v7.4s, v8.4s, v9.4s}, [%1], #64 \n" + + "bne 0b \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(r0), // %2 + "=r"(r1), // %3 + "=r"(r2) // %4 + : "0"(nn), + "1"(outptr0), + "2"(r0), + "3"(r1), + "4"(r2), + "w"(_k00), // %10 + "w"(_k01), // %11 + "w"(_k02), // %12 + "w"(_k10), // %13 + "w"(_k11), // %14 + "w"(_k12), // %15 + "w"(_k20), // %16 + "w"(_k21), // %17 + "w"(_k22) // %18 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9" + ); + } +#else // __aarch64__ + if (nn > 0) + { + asm volatile( + "0: \n" + + "pld [%1, #512] \n" + "vldm %1, {d0-d7} \n"// sum0 + + // r0 + "pld [%2, #256] \n" + "vld1.f32 {d8-d11}, [%2]! \n" + "vld1.f32 {d12[]}, [%2] \n" + + "vmla.f32 q0, %q10, d8[0] \n" + "vmla.f32 q1, %q10, d9[0] \n" + "vmla.f32 q2, %q10, d10[0] \n" + "vmla.f32 q3, %q10, d11[0] \n" + + + "vmla.f32 q0, %q11, d8[1] \n" + "vmla.f32 q1, %q11, d9[1] \n" + "vmla.f32 q2, %q11, d10[1] \n" + "vmla.f32 q3, %q11, d11[1] \n" + + "vmla.f32 q0, %q12, d9[0] \n" + "vmla.f32 q1, %q12, d10[0] \n" + "vmla.f32 q2, %q12, d11[0] \n" + + // r1 + "pld [%3, #256] \n" + "vld1.f32 {d8-d11}, [%3]! \n" + "vld1.f32 {d13[]}, [%3] \n" + + "vmla.f32 q3, %q12, d12[0] \n" + + "vmla.f32 q0, %q13, d8[0] \n" + "vmla.f32 q1, %q13, d9[0] \n" + "vmla.f32 q2, %q13, d10[0] \n" + "vmla.f32 q3, %q13, d11[0] \n" + + + "vmla.f32 q0, %q14, d8[1] \n" + "vmla.f32 q1, %q14, d9[1] \n" + "vmla.f32 q2, %q14, d10[1] \n" + "vmla.f32 q3, %q14, d11[1] \n" + + "vmla.f32 q0, %q15, d9[0] \n" + "vmla.f32 q1, %q15, d10[0] \n" + "vmla.f32 q2, %q15, d11[0] \n" + + // r2 + "pld [%4, #256] \n" + "vld1.f32 {d8-d11}, [%4]! \n" + "vld1.f32 {d12[]}, [%4] \n" + + "vmla.f32 q3, %q15, d13[0] \n" + + "vmla.f32 q0, %q16, d8[0] \n" + "vmla.f32 q1, %q16, d9[0] \n" + "vmla.f32 q2, %q16, d10[0] \n" + "vmla.f32 q3, %q16, d11[0] \n" + + + "vmla.f32 q0, %q17, d8[1] \n" + "vmla.f32 q1, %q17, d9[1] \n" + "vmla.f32 q2, %q17, d10[1] \n" + "vmla.f32 q3, %q17, d11[1] \n" + + "vmla.f32 q0, %q18, d9[0] \n" + "vmla.f32 q1, %q18, d10[0] \n" + "vmla.f32 q2, %q18, d11[0] \n" + "vmla.f32 q3, %q18, d12[0] \n" + + "subs %0, %0, #1 \n" + + "vstm %1!, {d0-d7} \n" + + "bne 0b \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(r0), // %2 + "=r"(r1), // %3 + "=r"(r2) // %4 + : "0"(nn), + "1"(outptr0), + "2"(r0), + "3"(r1), + "4"(r2), + "w"(_k00), // %10 + "w"(_k01), // %11 + "w"(_k02), // %12 + "w"(_k10), // %13 + "w"(_k11), // %14 + "w"(_k12), // %15 + "w"(_k20), // %16 + "w"(_k21), // %17 + "w"(_k22) // %18 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6" + ); + } +#endif // __aarch64__ + + for (; remain>0; remain--) + { + float32x4_t _sum0 = vld1q_f32(outptr0); + + float32x4_t _r0 = vld1q_f32(r0); + float32x4_t _r1 = vld1q_f32(r1); + float32x4_t _r2 = vld1q_f32(r2); + +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _k00, _r0, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k01, _r0, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k02, _r0, 2); + _sum0 = vfmaq_laneq_f32(_sum0, _k10, _r1, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k11, _r1, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k12, _r1, 2); + _sum0 = vfmaq_laneq_f32(_sum0, _k20, _r2, 0); + _sum0 = vfmaq_laneq_f32(_sum0, _k21, _r2, 1); + _sum0 = vfmaq_laneq_f32(_sum0, _k22, _r2, 2); +#else + _sum0 = vmlaq_lane_f32(_sum0, _k00, vget_low_f32(_r0), 0); + _sum0 = vmlaq_lane_f32(_sum0, _k01, vget_low_f32(_r0), 1); + _sum0 = vmlaq_lane_f32(_sum0, _k02, vget_high_f32(_r0), 0); + _sum0 = vmlaq_lane_f32(_sum0, _k10, vget_low_f32(_r1), 0); + _sum0 = vmlaq_lane_f32(_sum0, _k11, vget_low_f32(_r1), 1); + _sum0 = vmlaq_lane_f32(_sum0, _k12, vget_high_f32(_r1), 0); + _sum0 = vmlaq_lane_f32(_sum0, _k20, vget_low_f32(_r2), 0); + _sum0 = vmlaq_lane_f32(_sum0, _k21, vget_low_f32(_r2), 1); + _sum0 = vmlaq_lane_f32(_sum0, _k22, vget_high_f32(_r2), 0); +#endif + + vst1q_f32(outptr0, _sum0); + + r0 += 2; + r1 += 2; + r2 += 2; + outptr0 += 4; + } + + r0 += tailstep; + r1 += tailstep; + r2 += tailstep; + } + + k0 += 9*4; + } + } + +} diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index 50bb8da88..e8d86ff05 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -40,6 +40,7 @@ namespace ncnn { #if __ARM_NEON #include "convolution_1x1_pack4.h" #include "convolution_3x3_pack4.h" +#include "convolution_3x3_pack1to4.h" #endif // __ARM_NEON DEFINE_LAYER_CREATOR(Convolution_arm) @@ -702,6 +703,18 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option if (elempack == 1 && out_elempack == 4) { + if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + { + conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + + return 0; + } + // num_output #pragma omp parallel for num_threads(opt.num_threads) for (int p=0; p