diff --git a/src/layer/arm/convolution_1x1.h b/src/layer/arm/convolution_1x1.h index 29fcfcaaa..d3944b98d 100644 --- a/src/layer/arm/convolution_1x1.h +++ b/src/layer/arm/convolution_1x1.h @@ -12,2034 +12,17 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -static void conv1x1s1_sgemm_transform_kernel_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch) -{ - const float* kernel = _kernel; - - // interleave -#if __ARM_NEON && __aarch64__ - kernel_tm.create(4 * 8, inch / 4 + inch % 4, outch / 8 + (outch % 8) / 4 + outch % 4); -#else - kernel_tm.create(4 * 4, inch / 4 + inch % 4, outch / 4 + outch % 4); -#endif // __ARM_NEON && __aarch64__ - - int p = 0; -#if __ARM_NEON && __aarch64__ - for (; p + 7 < outch; p += 8) - { - const float* kernel0 = kernel + (p + 0) * inch; - const float* kernel1 = kernel + (p + 1) * inch; - const float* kernel2 = kernel + (p + 2) * inch; - const float* kernel3 = kernel + (p + 3) * inch; - const float* kernel4 = kernel + (p + 4) * inch; - const float* kernel5 = kernel + (p + 5) * inch; - const float* kernel6 = kernel + (p + 6) * inch; - const float* kernel7 = kernel + (p + 7) * inch; - - float* ktmp = kernel_tm.channel(p / 8); - - for (int q = 0; q < inch; q++) - { - // kernel0...7 0 - ktmp[0] = kernel0[0]; - ktmp[1] = kernel1[0]; - ktmp[2] = kernel2[0]; - ktmp[3] = kernel3[0]; - ktmp[4] = kernel4[0]; - ktmp[5] = kernel5[0]; - ktmp[6] = kernel6[0]; - ktmp[7] = kernel7[0]; - - ktmp += 8; - kernel0 += 1; - kernel1 += 1; - kernel2 += 1; - kernel3 += 1; - kernel4 += 1; - kernel5 += 1; - kernel6 += 1; - kernel7 += 1; - } - } -#endif // __ARM_NEON && __aarch64__ - for (; p + 3 < outch; p += 4) - { - const float* kernel0 = kernel + (p + 0) * inch; - const float* kernel1 = kernel + (p + 1) * inch; - const float* kernel2 = kernel + (p + 2) * inch; - const float* kernel3 = kernel + (p + 3) * inch; - -#if __ARM_NEON && __aarch64__ - float* ktmp = kernel_tm.channel(p / 8 + (p % 8) / 4); -#else - float* ktmp = kernel_tm.channel(p / 4); -#endif // __ARM_NEON && __aarch64__ - - for (int q = 0; q < inch; q++) - { - // kernel0...3 0 - ktmp[0] = kernel0[0]; - ktmp[1] = kernel1[0]; - ktmp[2] = kernel2[0]; - ktmp[3] = kernel3[0]; - - ktmp += 4; - kernel0 += 1; - kernel1 += 1; - kernel2 += 1; - kernel3 += 1; - } - } - for (; p < outch; p++) - { - const float* kernel0 = kernel + p * inch; - -#if __ARM_NEON && __aarch64__ - float* ktmp = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - float* ktmp = kernel_tm.channel(p / 4 + p % 4); -#endif // __ARM_NEON && __aarch64__ - - for (int q = 0; q < inch; q++) - { - ktmp[0] = kernel0[0]; - ktmp++; - kernel0++; - } - } -} - static void conv1x1s1_sgemm_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) { int w = bottom_blob.w; int h = bottom_blob.h; - int inch = bottom_blob.c; - int outch = top_blob.c; - const int size = w * h; - const float* bias = _bias; - - // interleave - Mat tmp(8 * 4, inch / 4 + inch % 4, size / 8 + (size % 8) / 4 + size % 4, 4u, opt.workspace_allocator); - { - int nn_size = size >> 3; - int remain_size_start = nn_size << 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = ii * 8; - - const float* img0 = bottom_blob.channel(0); - img0 += i; - - float* tmpptr = tmp.channel(i / 8); - - for (int q = 0; q < inch; q++) - { -#if __ARM_NEON -#if __aarch64__ - vst1q_f32(tmpptr, vld1q_f32(img0)); - vst1q_f32(tmpptr + 4, vld1q_f32(img0 + 4)); - - tmpptr += 8; - img0 += bottom_blob.cstep; -#else - asm volatile( - "pld [%0, #256] \n" - "vld1.f32 {d0-d3}, [%0 :128] \n" - "vst1.f32 {d0-d3}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1"); - - img0 += bottom_blob.cstep; -#endif // __aarch64__ -#else - tmpptr[0] = img0[0]; - tmpptr[1] = img0[1]; - tmpptr[2] = img0[2]; - tmpptr[3] = img0[3]; - tmpptr[4] = img0[4]; - tmpptr[5] = img0[5]; - tmpptr[6] = img0[6]; - tmpptr[7] = img0[7]; - - tmpptr += 8; - img0 += bottom_blob.cstep; -#endif // __ARM_NEON - } - } - - nn_size = (size - remain_size_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 4; - - const float* img0 = bottom_blob.channel(0); - img0 += i; - - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); - - for (int q = 0; q < inch; q++) - { -#if __ARM_NEON -#if __aarch64__ - vst1q_f32(tmpptr, vld1q_f32(img0)); - - tmpptr += 4; - img0 += bottom_blob.cstep; -#else - asm volatile( - "pld [%0, #128] \n" - "vld1.f32 {d0-d1}, [%0 :128] \n" - "vst1.f32 {d0-d1}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0"); - - img0 += bottom_blob.cstep; -#endif // __aarch64__ -#else - tmpptr[0] = img0[0]; - tmpptr[1] = img0[1]; - tmpptr[2] = img0[2]; - tmpptr[3] = img0[3]; - - tmpptr += 4; - img0 += bottom_blob.cstep; -#endif // __ARM_NEON - } - } - - remain_size_start += nn_size << 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_size_start; i < size; i++) - { - const float* img0 = bottom_blob.channel(0); - img0 += i; - - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); - - for (int q = 0; q < inch; q++) - { - tmpptr[0] = img0[0]; - tmpptr++; - img0 += bottom_blob.cstep; - } - } - } - - int nn_outch = 0; - int remain_outch_start = 0; - -#if __ARM_NEON && __aarch64__ - nn_outch = outch >> 3; - remain_outch_start = nn_outch << 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - float* outptr0 = top_blob.channel(p); - float* outptr1 = top_blob.channel(p + 1); - float* outptr2 = top_blob.channel(p + 2); - float* outptr3 = top_blob.channel(p + 3); - float* outptr4 = top_blob.channel(p + 4); - float* outptr5 = top_blob.channel(p + 5); - float* outptr6 = top_blob.channel(p + 6); - float* outptr7 = top_blob.channel(p + 7); - - const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p : zeros; - - int i = 0; - - for (; i + 7 < size; i += 8) - { - const float* tmpptr = tmp.channel(i / 8); - const float* kptr = kernel.channel(p / 8); - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%20] \n" - "dup v16.4s, v0.s[0] \n" - "dup v17.4s, v0.s[0] \n" - "dup v18.4s, v0.s[1] \n" - "dup v19.4s, v0.s[1] \n" - "dup v20.4s, v0.s[2] \n" - "dup v21.4s, v0.s[2] \n" - "dup v22.4s, v0.s[3] \n" - "dup v23.4s, v0.s[3] \n" - "dup v24.4s, v1.s[0] \n" - "dup v25.4s, v1.s[0] \n" - "dup v26.4s, v1.s[1] \n" - "dup v27.4s, v1.s[1] \n" - "dup v28.4s, v1.s[2] \n" - "dup v29.4s, v1.s[2] \n" - "dup v30.4s, v1.s[3] \n" - "dup v31.4s, v1.s[3] \n" - - // inch loop - "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%8, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64 \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v18.4s, v8.4s, v0.s[1] \n" - "fmla v20.4s, v8.4s, v0.s[2] \n" - "fmla v22.4s, v8.4s, v0.s[3] \n" - - "fmla v17.4s, v9.4s, v0.s[0] \n" - "fmla v19.4s, v9.4s, v0.s[1] \n" - "fmla v21.4s, v9.4s, v0.s[2] \n" - "fmla v23.4s, v9.4s, v0.s[3] \n" - - "fmla v24.4s, v8.4s, v1.s[0] \n" - "fmla v26.4s, v8.4s, v1.s[1] \n" - "fmla v28.4s, v8.4s, v1.s[2] \n" - "fmla v30.4s, v8.4s, v1.s[3] \n" - - "fmla v25.4s, v9.4s, v1.s[0] \n" - "fmla v27.4s, v9.4s, v1.s[1] \n" - "fmla v29.4s, v9.4s, v1.s[2] \n" - "fmla v31.4s, v9.4s, v1.s[3] \n" - - "prfm pldl1keep, [%8, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%8], #64 \n" - - "fmla v16.4s, v10.4s, v2.s[0] \n" - "fmla v18.4s, v10.4s, v2.s[1] \n" - "fmla v20.4s, v10.4s, v2.s[2] \n" - "fmla v22.4s, v10.4s, v2.s[3] \n" - - "fmla v17.4s, v11.4s, v2.s[0] \n" - "fmla v19.4s, v11.4s, v2.s[1] \n" - "fmla v21.4s, v11.4s, v2.s[2] \n" - "fmla v23.4s, v11.4s, v2.s[3] \n" - - "fmla v24.4s, v10.4s, v3.s[0] \n" - "fmla v26.4s, v10.4s, v3.s[1] \n" - "fmla v28.4s, v10.4s, v3.s[2] \n" - "fmla v30.4s, v10.4s, v3.s[3] \n" - - "fmla v25.4s, v11.4s, v3.s[0] \n" - "fmla v27.4s, v11.4s, v3.s[1] \n" - "fmla v29.4s, v11.4s, v3.s[2] \n" - "fmla v31.4s, v11.4s, v3.s[3] \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" - - "fmla v16.4s, v12.4s, v4.s[0] \n" - "fmla v18.4s, v12.4s, v4.s[1] \n" - "fmla v20.4s, v12.4s, v4.s[2] \n" - "fmla v22.4s, v12.4s, v4.s[3] \n" - - "fmla v17.4s, v13.4s, v4.s[0] \n" - "fmla v19.4s, v13.4s, v4.s[1] \n" - "fmla v21.4s, v13.4s, v4.s[2] \n" - "fmla v23.4s, v13.4s, v4.s[3] \n" - - "fmla v24.4s, v12.4s, v5.s[0] \n" - "fmla v26.4s, v12.4s, v5.s[1] \n" - "fmla v28.4s, v12.4s, v5.s[2] \n" - "fmla v30.4s, v12.4s, v5.s[3] \n" - - "fmla v25.4s, v13.4s, v5.s[0] \n" - "fmla v27.4s, v13.4s, v5.s[1] \n" - "fmla v29.4s, v13.4s, v5.s[2] \n" - "fmla v31.4s, v13.4s, v5.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v16.4s, v14.4s, v6.s[0] \n" - "fmla v18.4s, v14.4s, v6.s[1] \n" - "fmla v20.4s, v14.4s, v6.s[2] \n" - "fmla v22.4s, v14.4s, v6.s[3] \n" - - "fmla v17.4s, v15.4s, v6.s[0] \n" - "fmla v19.4s, v15.4s, v6.s[1] \n" - "fmla v21.4s, v15.4s, v6.s[2] \n" - "fmla v23.4s, v15.4s, v6.s[3] \n" - - "fmla v24.4s, v14.4s, v7.s[0] \n" - "fmla v26.4s, v14.4s, v7.s[1] \n" - "fmla v28.4s, v14.4s, v7.s[2] \n" - "fmla v30.4s, v14.4s, v7.s[3] \n" - - "fmla v25.4s, v15.4s, v7.s[0] \n" - "fmla v27.4s, v15.4s, v7.s[1] \n" - "fmla v29.4s, v15.4s, v7.s[2] \n" - "fmla v31.4s, v15.4s, v7.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w21, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%8, #256] \n" - "ld1 {v8.4s, v9.4s}, [%8], #32 \n" - - "prfm pldl1keep, [%9, #256] \n" - "ld1 {v0.4s, v1.4s}, [%9], #32 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v18.4s, v8.4s, v0.s[1] \n" - "fmla v20.4s, v8.4s, v0.s[2] \n" - "fmla v22.4s, v8.4s, v0.s[3] \n" - - "fmla v17.4s, v9.4s, v0.s[0] \n" - "fmla v19.4s, v9.4s, v0.s[1] \n" - "fmla v21.4s, v9.4s, v0.s[2] \n" - "fmla v23.4s, v9.4s, v0.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v24.4s, v8.4s, v1.s[0] \n" - "fmla v26.4s, v8.4s, v1.s[1] \n" - "fmla v28.4s, v8.4s, v1.s[2] \n" - "fmla v30.4s, v8.4s, v1.s[3] \n" - - "fmla v25.4s, v9.4s, v1.s[0] \n" - "fmla v27.4s, v9.4s, v1.s[1] \n" - "fmla v29.4s, v9.4s, v1.s[2] \n" - "fmla v31.4s, v9.4s, v1.s[3] \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v16.4s, v17.4s}, [%0], #32 \n" - "st1 {v18.4s, v19.4s}, [%1], #32 \n" - "st1 {v20.4s, v21.4s}, [%2], #32 \n" - "st1 {v22.4s, v23.4s}, [%3], #32 \n" - "st1 {v24.4s, v25.4s}, [%4], #32 \n" - "st1 {v26.4s, v27.4s}, [%5], #32 \n" - "st1 {v28.4s, v29.4s}, [%6], #32 \n" - "st1 {v30.4s, v31.4s}, [%7], #32 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(outptr4), // %4 - "=r"(outptr5), // %5 - "=r"(outptr6), // %6 - "=r"(outptr7), // %7 - "=r"(tmpptr), // %8 - "=r"(kptr) // %9 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(outptr4), - "5"(outptr5), - "6"(outptr6), - "7"(outptr7), - "8"(tmpptr), - "9"(kptr), - "r"(biasptr), // %20 - "r"(inch) // %21 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - - for (; i + 3 < size; i += 4) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); - const float* kptr = kernel.channel(p / 8); - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%20] \n" - "dup v16.4s, v0.s[0] \n" - "dup v17.4s, v0.s[1] \n" - "dup v18.4s, v0.s[2] \n" - "dup v19.4s, v0.s[3] \n" - "dup v20.4s, v1.s[0] \n" - "dup v21.4s, v1.s[1] \n" - "dup v22.4s, v1.s[2] \n" - "dup v23.4s, v1.s[3] \n" - - // inch loop - "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%8, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64 \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v0.s[1] \n" - "fmla v18.4s, v8.4s, v0.s[2] \n" - "fmla v19.4s, v8.4s, v0.s[3] \n" - "fmla v20.4s, v8.4s, v1.s[0] \n" - "fmla v21.4s, v8.4s, v1.s[1] \n" - "fmla v22.4s, v8.4s, v1.s[2] \n" - "fmla v23.4s, v8.4s, v1.s[3] \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" - - "fmla v16.4s, v9.4s, v2.s[0] \n" - "fmla v17.4s, v9.4s, v2.s[1] \n" - "fmla v18.4s, v9.4s, v2.s[2] \n" - "fmla v19.4s, v9.4s, v2.s[3] \n" - "fmla v20.4s, v9.4s, v3.s[0] \n" - "fmla v21.4s, v9.4s, v3.s[1] \n" - "fmla v22.4s, v9.4s, v3.s[2] \n" - "fmla v23.4s, v9.4s, v3.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v16.4s, v10.4s, v4.s[0] \n" - "fmla v17.4s, v10.4s, v4.s[1] \n" - "fmla v18.4s, v10.4s, v4.s[2] \n" - "fmla v19.4s, v10.4s, v4.s[3] \n" - "fmla v20.4s, v10.4s, v5.s[0] \n" - "fmla v21.4s, v10.4s, v5.s[1] \n" - "fmla v22.4s, v10.4s, v5.s[2] \n" - "fmla v23.4s, v10.4s, v5.s[3] \n" - - "fmla v16.4s, v11.4s, v6.s[0] \n" - "fmla v17.4s, v11.4s, v6.s[1] \n" - "fmla v18.4s, v11.4s, v6.s[2] \n" - "fmla v19.4s, v11.4s, v6.s[3] \n" - "fmla v20.4s, v11.4s, v7.s[0] \n" - "fmla v21.4s, v11.4s, v7.s[1] \n" - "fmla v22.4s, v11.4s, v7.s[2] \n" - "fmla v23.4s, v11.4s, v7.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w21, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%8, #128] \n" - "ld1 {v8.4s}, [%8], #16 \n" - - "prfm pldl1keep, [%9, #256] \n" - "ld1 {v0.4s, v1.4s}, [%9], #32 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v0.s[1] \n" - "fmla v18.4s, v8.4s, v0.s[2] \n" - "fmla v19.4s, v8.4s, v0.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v20.4s, v8.4s, v1.s[0] \n" - "fmla v21.4s, v8.4s, v1.s[1] \n" - "fmla v22.4s, v8.4s, v1.s[2] \n" - "fmla v23.4s, v8.4s, v1.s[3] \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v16.4s}, [%0], #16 \n" - "st1 {v17.4s}, [%1], #16 \n" - "st1 {v18.4s}, [%2], #16 \n" - "st1 {v19.4s}, [%3], #16 \n" - "st1 {v20.4s}, [%4], #16 \n" - "st1 {v21.4s}, [%5], #16 \n" - "st1 {v22.4s}, [%6], #16 \n" - "st1 {v23.4s}, [%7], #16 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(outptr4), // %4 - "=r"(outptr5), // %5 - "=r"(outptr6), // %6 - "=r"(outptr7), // %7 - "=r"(tmpptr), // %8 - "=r"(kptr) // %9 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(outptr4), - "5"(outptr5), - "6"(outptr6), - "7"(outptr7), - "8"(tmpptr), - "9"(kptr), - "r"(biasptr), // %20 - "r"(inch) // %21 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - } - - for (; i < size; i++) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); - const float* kptr = kernel.channel(p / 8); - - asm volatile( - "ld1 {v24.4s, v25.4s}, [%20] \n" - - // inch loop - "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "eor v16.16b, v16.16b, v16.16b \n" - "eor v17.16b, v17.16b, v17.16b \n" - "eor v18.16b, v18.16b, v18.16b \n" - "eor v19.16b, v19.16b, v19.16b \n" - "eor v20.16b, v20.16b, v20.16b \n" - "eor v21.16b, v21.16b, v21.16b \n" - "eor v22.16b, v22.16b, v22.16b \n" - "eor v23.16b, v23.16b, v23.16b \n" - - "0: \n" - - "prfm pldl1keep, [%8, #128] \n" - "ld1 {v8.4s}, [%8], #16 \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" - - "fmla v16.4s, v0.4s, v8.s[0] \n" - "fmla v17.4s, v1.4s, v8.s[0] \n" - "fmla v18.4s, v2.4s, v8.s[1] \n" - "fmla v19.4s, v3.4s, v8.s[1] \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" - - "subs w4, w4, #1 \n" - - "fmla v20.4s, v4.4s, v8.s[2] \n" - "fmla v21.4s, v5.4s, v8.s[2] \n" - "fmla v22.4s, v6.4s, v8.s[3] \n" - "fmla v23.4s, v7.4s, v8.s[3] \n" - - "bne 0b \n" - - "fadd v16.4s, v16.4s, v18.4s \n" - "fadd v17.4s, v17.4s, v19.4s \n" - "fadd v20.4s, v20.4s, v22.4s \n" - "fadd v21.4s, v21.4s, v23.4s \n" - "fadd v16.4s, v16.4s, v20.4s \n" - "fadd v17.4s, v17.4s, v21.4s \n" - "fadd v24.4s, v24.4s, v16.4s \n" - "fadd v25.4s, v25.4s, v17.4s \n" - - "1: \n" - - // remain loop - "and w4, %w21, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%8, #32] \n" - "ld1r {v8.4s}, [%8], #4 \n" - - "prfm pldl1keep, [%9, #256] \n" - "ld1 {v0.4s, v1.4s}, [%9], #32 \n" - - "subs w4, w4, #1 \n" - - "fmla v24.4s, v8.4s, v0.4s \n" - "fmla v25.4s, v8.4s, v1.4s \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v24.s}[0],[%0], #4 \n" - "st1 {v24.s}[1],[%1], #4 \n" - "st1 {v24.s}[2],[%2], #4 \n" - "st1 {v24.s}[3],[%3], #4 \n" - "st1 {v25.s}[0],[%4], #4 \n" - "st1 {v25.s}[1],[%5], #4 \n" - "st1 {v25.s}[2],[%6], #4 \n" - "st1 {v25.s}[3],[%7], #4 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(outptr4), // %4 - "=r"(outptr5), // %5 - "=r"(outptr6), // %6 - "=r"(outptr7), // %7 - "=r"(tmpptr), // %8 - "=r"(kptr) // %9 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(outptr4), - "5"(outptr5), - "6"(outptr6), - "7"(outptr7), - "8"(tmpptr), - "9"(kptr), - "r"(biasptr), // %20 - "r"(inch) // %21 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25"); - } - } -#endif // __ARM_NEON && __aarch64__ - - nn_outch = (outch - remain_outch_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - float* outptr0 = top_blob.channel(p); - float* outptr1 = top_blob.channel(p + 1); - float* outptr2 = top_blob.channel(p + 2); - float* outptr3 = top_blob.channel(p + 3); - - const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p : zeros; - - int i = 0; - - for (; i + 7 < size; i += 8) - { - const float* tmpptr = tmp.channel(i / 8); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); -#else - const float* kptr = kernel.channel(p / 4); -#endif // __ARM_NEON && __aarch64__ - -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%12] \n" - "dup v8.4s, v0.s[0] \n" - "dup v9.4s, v0.s[0] \n" - "dup v10.4s, v0.s[1] \n" - "dup v11.4s, v0.s[1] \n" - "dup v12.4s, v0.s[2] \n" - "dup v13.4s, v0.s[2] \n" - "dup v14.4s, v0.s[3] \n" - "dup v15.4s, v0.s[3] \n" - - // inch loop - "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" - - "prfm pldl1keep, [%5, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v10.4s, v4.4s, v0.s[1] \n" - "fmla v12.4s, v4.4s, v0.s[2] \n" - "fmla v14.4s, v4.4s, v0.s[3] \n" - - "fmla v9.4s, v5.4s, v0.s[0] \n" - "fmla v11.4s, v5.4s, v0.s[1] \n" - "fmla v13.4s, v5.4s, v0.s[2] \n" - "fmla v15.4s, v5.4s, v0.s[3] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%4], #64 \n" - - "fmla v8.4s, v6.4s, v1.s[0] \n" - "fmla v10.4s, v6.4s, v1.s[1] \n" - "fmla v12.4s, v6.4s, v1.s[2] \n" - "fmla v14.4s, v6.4s, v1.s[3] \n" - - "fmla v9.4s, v7.4s, v1.s[0] \n" - "fmla v11.4s, v7.4s, v1.s[1] \n" - "fmla v13.4s, v7.4s, v1.s[2] \n" - "fmla v15.4s, v7.4s, v1.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v16.4s, v2.s[0] \n" - "fmla v10.4s, v16.4s, v2.s[1] \n" - "fmla v12.4s, v16.4s, v2.s[2] \n" - "fmla v14.4s, v16.4s, v2.s[3] \n" - - "fmla v9.4s, v17.4s, v2.s[0] \n" - "fmla v11.4s, v17.4s, v2.s[1] \n" - "fmla v13.4s, v17.4s, v2.s[2] \n" - "fmla v15.4s, v17.4s, v2.s[3] \n" - - "fmla v8.4s, v18.4s, v3.s[0] \n" - "fmla v10.4s, v18.4s, v3.s[1] \n" - "fmla v12.4s, v18.4s, v3.s[2] \n" - "fmla v14.4s, v18.4s, v3.s[3] \n" - - "fmla v9.4s, v19.4s, v3.s[0] \n" - "fmla v11.4s, v19.4s, v3.s[1] \n" - "fmla v13.4s, v19.4s, v3.s[2] \n" - "fmla v15.4s, v19.4s, v3.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w13, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v4.4s, v5.4s}, [%4], #32 \n" - - "prfm pldl1keep, [%5, #128] \n" - "ld1 {v0.4s}, [%5], #16 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v10.4s, v4.4s, v0.s[1] \n" - "fmla v12.4s, v4.4s, v0.s[2] \n" - "fmla v14.4s, v4.4s, v0.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v9.4s, v5.4s, v0.s[0] \n" - "fmla v11.4s, v5.4s, v0.s[1] \n" - "fmla v13.4s, v5.4s, v0.s[2] \n" - "fmla v15.4s, v5.4s, v0.s[3] \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v8.4s, v9.4s}, [%0], #32 \n" - "st1 {v10.4s, v11.4s}, [%1], #32 \n" - "st1 {v12.4s, v13.4s}, [%2], #32 \n" - "st1 {v14.4s, v15.4s}, [%3], #32 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); -#else // __aarch64__ - asm volatile( - "vld1.f32 {d0-d1}, [%12] \n" - "vdup.f32 q8, d0[0] \n" - "vdup.f32 q9, d0[0] \n" - "vdup.f32 q10, d0[1] \n" - "vdup.f32 q11, d0[1] \n" - "vdup.f32 q12, d1[0] \n" - "vdup.f32 q13, d1[0] \n" - "vdup.f32 q14, d1[1] \n" - "vdup.f32 q15, d1[1] \n" - - // inch loop - "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" - - "pld [%4, #512] \n" - "vldm %4!, {d8-d15} \n" - // "vld1.f32 {d8-d11}, [%4 :128]! \n" - // "vld1.f32 {d12-d15}, [%4 :128]! \n" - - "pld [%5, #512] \n" - "vldm %5!, {d0-d7} \n" - // "vld1.f32 {d0-d3}, [%5 :128]! \n" - // "vld1.f32 {d4-d7}, [%5 :128]! \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q10, q4, d0[1] \n" - "vmla.f32 q12, q4, d1[0] \n" - "vmla.f32 q14, q4, d1[1] \n" - - "vmla.f32 q9, q5, d0[0] \n" - "vmla.f32 q11, q5, d0[1] \n" - "vmla.f32 q13, q5, d1[0] \n" - "vmla.f32 q15, q5, d1[1] \n" - - "vmla.f32 q8, q6, d2[0] \n" - "vmla.f32 q10, q6, d2[1] \n" - "vmla.f32 q12, q6, d3[0] \n" - "vmla.f32 q14, q6, d3[1] \n" - - "vmla.f32 q9, q7, d2[0] \n" - "vmla.f32 q11, q7, d2[1] \n" - "vmla.f32 q13, q7, d3[0] \n" - "vmla.f32 q15, q7, d3[1] \n" - - "pld [%4, #512] \n" - "vldm %4!, {d8-d15} \n" - // "vld1.f32 {d8-d11}, [%4 :128]! \n" - // "vld1.f32 {d12-d15}, [%4 :128]! \n" - - "vmla.f32 q8, q4, d4[0] \n" - "vmla.f32 q10, q4, d4[1] \n" - "vmla.f32 q12, q4, d5[0] \n" - "vmla.f32 q14, q4, d5[1] \n" - - "vmla.f32 q9, q5, d4[0] \n" - "vmla.f32 q11, q5, d4[1] \n" - "vmla.f32 q13, q5, d5[0] \n" - "vmla.f32 q15, q5, d5[1] \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q6, d6[0] \n" - "vmla.f32 q10, q6, d6[1] \n" - "vmla.f32 q12, q6, d7[0] \n" - "vmla.f32 q14, q6, d7[1] \n" - - "vmla.f32 q9, q7, d6[0] \n" - "vmla.f32 q11, q7, d6[1] \n" - "vmla.f32 q13, q7, d7[0] \n" - "vmla.f32 q15, q7, d7[1] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and r4, %13, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - - "pld [%4, #256] \n" - "vld1.f32 {d8-d11}, [%4 :128]! \n" - - "pld [%5, #128] \n" - "vld1.f32 {d0-d1}, [%5 :128]! \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q10, q4, d0[1] \n" - "vmla.f32 q12, q4, d1[0] \n" - "vmla.f32 q14, q4, d1[1] \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q9, q5, d0[0] \n" - "vmla.f32 q11, q5, d0[1] \n" - "vmla.f32 q13, q5, d1[0] \n" - "vmla.f32 q15, q5, d1[1] \n" - - "bne 2b \n" - - "3: \n" - - "vst1.f32 {d16-d19}, [%0 :128]! \n" - "vst1.f32 {d20-d23}, [%1 :128]! \n" - "vst1.f32 {d24-d27}, [%2 :128]! \n" - "vst1.f32 {d28-d31}, [%3 :128]! \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif // __aarch64__ -#else - float sum0_0 = biasptr[0]; - float sum0_1 = biasptr[0]; - float sum0_2 = biasptr[0]; - float sum0_3 = biasptr[0]; - float sum0_4 = biasptr[0]; - float sum0_5 = biasptr[0]; - float sum0_6 = biasptr[0]; - float sum0_7 = biasptr[0]; - - float sum1_0 = biasptr[1]; - float sum1_1 = biasptr[1]; - float sum1_2 = biasptr[1]; - float sum1_3 = biasptr[1]; - float sum1_4 = biasptr[1]; - float sum1_5 = biasptr[1]; - float sum1_6 = biasptr[1]; - float sum1_7 = biasptr[1]; - - float sum2_0 = biasptr[2]; - float sum2_1 = biasptr[2]; - float sum2_2 = biasptr[2]; - float sum2_3 = biasptr[2]; - float sum2_4 = biasptr[2]; - float sum2_5 = biasptr[2]; - float sum2_6 = biasptr[2]; - float sum2_7 = biasptr[2]; - - float sum3_0 = biasptr[3]; - float sum3_1 = biasptr[3]; - float sum3_2 = biasptr[3]; - float sum3_3 = biasptr[3]; - float sum3_4 = biasptr[3]; - float sum3_5 = biasptr[3]; - float sum3_6 = biasptr[3]; - float sum3_7 = biasptr[3]; - - for (int q = 0; q < inch; q++) - { - sum0_0 += tmpptr[0] * kptr[0]; - sum0_1 += tmpptr[1] * kptr[0]; - sum0_2 += tmpptr[2] * kptr[0]; - sum0_3 += tmpptr[3] * kptr[0]; - sum0_4 += tmpptr[4] * kptr[0]; - sum0_5 += tmpptr[5] * kptr[0]; - sum0_6 += tmpptr[6] * kptr[0]; - sum0_7 += tmpptr[7] * kptr[0]; - - sum1_0 += tmpptr[0] * kptr[1]; - sum1_1 += tmpptr[1] * kptr[1]; - sum1_2 += tmpptr[2] * kptr[1]; - sum1_3 += tmpptr[3] * kptr[1]; - sum1_4 += tmpptr[4] * kptr[1]; - sum1_5 += tmpptr[5] * kptr[1]; - sum1_6 += tmpptr[6] * kptr[1]; - sum1_7 += tmpptr[7] * kptr[1]; - - sum2_0 += tmpptr[0] * kptr[2]; - sum2_1 += tmpptr[1] * kptr[2]; - sum2_2 += tmpptr[2] * kptr[2]; - sum2_3 += tmpptr[3] * kptr[2]; - sum2_4 += tmpptr[4] * kptr[2]; - sum2_5 += tmpptr[5] * kptr[2]; - sum2_6 += tmpptr[6] * kptr[2]; - sum2_7 += tmpptr[7] * kptr[2]; - - sum3_0 += tmpptr[0] * kptr[3]; - sum3_1 += tmpptr[1] * kptr[3]; - sum3_2 += tmpptr[2] * kptr[3]; - sum3_3 += tmpptr[3] * kptr[3]; - sum3_4 += tmpptr[4] * kptr[3]; - sum3_5 += tmpptr[5] * kptr[3]; - sum3_6 += tmpptr[6] * kptr[3]; - sum3_7 += tmpptr[7] * kptr[3]; - - tmpptr += 8; - kptr += 4; - } - - outptr0[0] = sum0_0; - outptr0[1] = sum0_1; - outptr0[2] = sum0_2; - outptr0[3] = sum0_3; - outptr0[4] = sum0_4; - outptr0[5] = sum0_5; - outptr0[6] = sum0_6; - outptr0[7] = sum0_7; - - outptr1[0] = sum1_0; - outptr1[1] = sum1_1; - outptr1[2] = sum1_2; - outptr1[3] = sum1_3; - outptr1[4] = sum1_4; - outptr1[5] = sum1_5; - outptr1[6] = sum1_6; - outptr1[7] = sum1_7; - - outptr2[0] = sum2_0; - outptr2[1] = sum2_1; - outptr2[2] = sum2_2; - outptr2[3] = sum2_3; - outptr2[4] = sum2_4; - outptr2[5] = sum2_5; - outptr2[6] = sum2_6; - outptr2[7] = sum2_7; - - outptr3[0] = sum3_0; - outptr3[1] = sum3_1; - outptr3[2] = sum3_2; - outptr3[3] = sum3_3; - outptr3[4] = sum3_4; - outptr3[5] = sum3_5; - outptr3[6] = sum3_6; - outptr3[7] = sum3_7; - - outptr0 += 8; - outptr1 += 8; - outptr2 += 8; - outptr3 += 8; -#endif // __ARM_NEON - } - - for (; i + 3 < size; i += 4) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); -#else - const float* kptr = kernel.channel(p / 4); -#endif // __ARM_NEON && __aarch64__ - -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%12] \n" - "dup v8.4s, v0.s[0] \n" - "dup v9.4s, v0.s[1] \n" - "dup v10.4s, v0.s[2] \n" - "dup v11.4s, v0.s[3] \n" - - // inch loop - "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" - - "prfm pldl1keep, [%5, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - - "fmla v8.4s, v5.4s, v1.s[0] \n" - "fmla v9.4s, v5.4s, v1.s[1] \n" - "fmla v10.4s, v5.4s, v1.s[2] \n" - "fmla v11.4s, v5.4s, v1.s[3] \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v6.4s, v2.s[0] \n" - "fmla v9.4s, v6.4s, v2.s[1] \n" - "fmla v10.4s, v6.4s, v2.s[2] \n" - "fmla v11.4s, v6.4s, v2.s[3] \n" - - "fmla v8.4s, v7.4s, v3.s[0] \n" - "fmla v9.4s, v7.4s, v3.s[1] \n" - "fmla v10.4s, v7.4s, v3.s[2] \n" - "fmla v11.4s, v7.4s, v3.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w13, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%4, #128] \n" - "ld1 {v4.4s}, [%4], #16 \n" - - "prfm pldl1keep, [%5, #128] \n" - "ld1 {v0.4s}, [%5], #16 \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v8.4s}, [%0], #16 \n" - "st1 {v9.4s}, [%1], #16 \n" - "st1 {v10.4s}, [%2], #16 \n" - "st1 {v11.4s}, [%3], #16 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"); -#else // __aarch64__ - asm volatile( - "vld1.f32 {d0-d1}, [%12] \n" - "vdup.f32 q8, d0[0] \n" - "vdup.f32 q9, d0[1] \n" - "vdup.f32 q10, d1[0] \n" - "vdup.f32 q11, d1[1] \n" - - // inch loop - "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" - - "pld [%4, #512] \n" - "vldm %4!, {d8-d15} \n" - // "vld1.f32 {d8-d11}, [%4 :128]! \n" - // "vld1.f32 {d12-d15}, [%4 :128]! \n" - - "pld [%5, #512] \n" - "vldm %5!, {d0-d7} \n" - // "vld1.f32 {d0-d3}, [%5 :128]! \n" - // "vld1.f32 {d4-d7}, [%5 :128]! \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d0[1] \n" - "vmla.f32 q10, q4, d1[0] \n" - "vmla.f32 q11, q4, d1[1] \n" - - "vmla.f32 q8, q5, d2[0] \n" - "vmla.f32 q9, q5, d2[1] \n" - "vmla.f32 q10, q5, d3[0] \n" - "vmla.f32 q11, q5, d3[1] \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q6, d4[0] \n" - "vmla.f32 q9, q6, d4[1] \n" - "vmla.f32 q10, q6, d5[0] \n" - "vmla.f32 q11, q6, d5[1] \n" - - "vmla.f32 q8, q7, d6[0] \n" - "vmla.f32 q9, q7, d6[1] \n" - "vmla.f32 q10, q7, d7[0] \n" - "vmla.f32 q11, q7, d7[1] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and r4, %13, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - - "pld [%4, #128] \n" - "vld1.f32 {d8-d9}, [%4 :128]! \n" - - "pld [%5, #128] \n" - "vld1.f32 {d0-d1}, [%5 :128]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d0[1] \n" - "vmla.f32 q10, q4, d1[0] \n" - "vmla.f32 q11, q4, d1[1] \n" - - "bne 2b \n" - - "3: \n" - - "vst1.f32 {d16-d17}, [%0 :128]! \n" - "vst1.f32 {d18-d19}, [%1 :128]! \n" - "vst1.f32 {d20-d21}, [%2 :128]! \n" - "vst1.f32 {d22-d23}, [%3 :128]! \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); -#endif // __aarch64__ -#else - float sum0_0 = biasptr[0]; - float sum0_1 = biasptr[0]; - float sum0_2 = biasptr[0]; - float sum0_3 = biasptr[0]; - - float sum1_0 = biasptr[1]; - float sum1_1 = biasptr[1]; - float sum1_2 = biasptr[1]; - float sum1_3 = biasptr[1]; - - float sum2_0 = biasptr[2]; - float sum2_1 = biasptr[2]; - float sum2_2 = biasptr[2]; - float sum2_3 = biasptr[2]; - - float sum3_0 = biasptr[3]; - float sum3_1 = biasptr[3]; - float sum3_2 = biasptr[3]; - float sum3_3 = biasptr[3]; - - for (int q = 0; q < inch; q++) - { - sum0_0 += tmpptr[0] * kptr[0]; - sum0_1 += tmpptr[1] * kptr[0]; - sum0_2 += tmpptr[2] * kptr[0]; - sum0_3 += tmpptr[3] * kptr[0]; - - sum1_0 += tmpptr[0] * kptr[1]; - sum1_1 += tmpptr[1] * kptr[1]; - sum1_2 += tmpptr[2] * kptr[1]; - sum1_3 += tmpptr[3] * kptr[1]; - - sum2_0 += tmpptr[0] * kptr[2]; - sum2_1 += tmpptr[1] * kptr[2]; - sum2_2 += tmpptr[2] * kptr[2]; - sum2_3 += tmpptr[3] * kptr[2]; - - sum3_0 += tmpptr[0] * kptr[3]; - sum3_1 += tmpptr[1] * kptr[3]; - sum3_2 += tmpptr[2] * kptr[3]; - sum3_3 += tmpptr[3] * kptr[3]; - - tmpptr += 4; - kptr += 4; - } - - outptr0[0] = sum0_0; - outptr0[1] = sum0_1; - outptr0[2] = sum0_2; - outptr0[3] = sum0_3; - - outptr1[0] = sum1_0; - outptr1[1] = sum1_1; - outptr1[2] = sum1_2; - outptr1[3] = sum1_3; - - outptr2[0] = sum2_0; - outptr2[1] = sum2_1; - outptr2[2] = sum2_2; - outptr2[3] = sum2_3; - - outptr3[0] = sum3_0; - outptr3[1] = sum3_1; - outptr3[2] = sum3_2; - outptr3[3] = sum3_3; - - outptr0 += 4; - outptr1 += 4; - outptr2 += 4; - outptr3 += 4; -#endif // __ARM_NEON - } - - for (; i < size; i++) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); -#else - const float* kptr = kernel.channel(p / 4); -#endif // __ARM_NEON && __aarch64__ - -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "ld1 {v12.4s}, [%12] \n" - - // inch loop - "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "eor v8.16b, v8.16b, v8.16b \n" - "eor v9.16b, v9.16b, v9.16b \n" - "eor v10.16b, v10.16b, v10.16b \n" - "eor v11.16b, v11.16b, v11.16b \n" - - "0: \n" - - "prfm pldl1keep, [%4, #128] \n" - "ld1 {v4.4s}, [%4], #16 \n" - - "prfm pldl1keep, [%5, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v0.4s, v4.s[0] \n" - "fmla v9.4s, v1.4s, v4.s[1] \n" - "fmla v10.4s, v2.4s, v4.s[2] \n" - "fmla v11.4s, v3.4s, v4.s[3] \n" - - "bne 0b \n" - - "fadd v8.4s, v8.4s, v9.4s \n" - "fadd v10.4s, v10.4s, v11.4s \n" - "fadd v8.4s, v8.4s, v10.4s \n" - "fadd v12.4s, v12.4s, v8.4s \n" - - "1: \n" - - // remain loop - "and w4, %w13, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%4, #32] \n" - "ld1r {v4.4s}, [%4], #4 \n" - - "prfm pldl1keep, [%5, #128] \n" - "ld1 {v0.4s}, [%5], #16 \n" - - "subs w4, w4, #1 \n" - - "fmla v12.4s, v4.4s, v0.4s \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v12.s}[0], [%0], #4 \n" - "st1 {v12.s}[1], [%1], #4 \n" - "st1 {v12.s}[2], [%2], #4 \n" - "st1 {v12.s}[3], [%3], #4 \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v8", "v9", "v10", "v11", "v12"); -#else // __aarch64__ - asm volatile( - "vld1.f32 {d24-d25}, [%12] \n" - - // inch loop - "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "veor q8, q8, q8 \n" - "veor q9, q9, q9 \n" - "veor q10, q10, q10 \n" - "veor q11, q11, q11 \n" - - "0: \n" - - "pld [%4, #128] \n" - "vld1.f32 {d8-d9}, [%4 :128]! \n" - - "pld [%5, #512] \n" - "vldm %5!, {d0-d7} \n" - // "vld1.f32 {d0-d3}, [%5 :128]! \n" - // "vld1.f32 {d4-d7}, [%5 :128]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q0, d8[0] \n" - "vmla.f32 q9, q1, d8[1] \n" - "vmla.f32 q10, q2, d9[0] \n" - "vmla.f32 q11, q3, d9[1] \n" - - "bne 0b \n" - - "vadd.f32 q8, q8, q9 \n" - "vadd.f32 q10, q10, q11 \n" - "vadd.f32 q8, q8, q10 \n" - "vadd.f32 q12, q12, q8 \n" - - "1: \n" - - // remain loop - "and r4, %13, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - - "pld [%4, #32] \n" - "vld1.f32 {d8[],d9[]}, [%4]! \n" - - "pld [%5, #128] \n" - "vld1.f32 {d0-d1}, [%5 :128]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q12, q4, q0 \n" - - "bne 2b \n" - - "3: \n" - - "vst1.f32 {d24[0]}, [%0]! \n" - "vst1.f32 {d24[1]}, [%1]! \n" - "vst1.f32 {d25[0]}, [%2]! \n" - "vst1.f32 {d25[1]}, [%3]! \n" - - : "=r"(outptr0), // %0 - "=r"(outptr1), // %1 - "=r"(outptr2), // %2 - "=r"(outptr3), // %3 - "=r"(tmpptr), // %4 - "=r"(kptr) // %5 - : "0"(outptr0), - "1"(outptr1), - "2"(outptr2), - "3"(outptr3), - "4"(tmpptr), - "5"(kptr), - "r"(biasptr), // %12 - "r"(inch) // %13 - : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11", "q12"); -#endif // __aarch64__ -#else - float sum0 = biasptr[0]; - float sum1 = biasptr[1]; - float sum2 = biasptr[2]; - float sum3 = biasptr[3]; - - for (int q = 0; q < inch; q++) - { - sum0 += tmpptr[0] * kptr[0]; - sum1 += tmpptr[0] * kptr[1]; - sum2 += tmpptr[0] * kptr[2]; - sum3 += tmpptr[0] * kptr[3]; - - tmpptr++; - kptr += 4; - } - - outptr0[0] = sum0; - outptr1[0] = sum1; - outptr2[0] = sum2; - outptr3[0] = sum3; - - outptr0++; - outptr1++; - outptr2++; - outptr3++; -#endif // __ARM_NEON - } - } - - remain_outch_start += nn_outch << 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - Mat out0 = top_blob.channel(p); - - const float bias0 = bias ? bias[p] : 0.f; - - float* outptr0 = out0; - - int i = 0; - - for (; i + 7 < size; i += 8) - { - const float* tmpptr = tmp.channel(i / 8); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - const float* kptr = kernel.channel(p / 4 + p % 4); -#endif // __ARM_NEON && __aarch64__ - -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "dup v8.4s, %w6 \n" - "dup v9.4s, %w6 \n" - - // inch loop - "lsr w4, %w7, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%1, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" - - "prfm pldl1keep, [%2, #128] \n" - "ld1 {v0.4s}, [%2], #16 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v5.4s, v0.s[0] \n" - - "prfm pldl1keep, [%1, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - - "fmla v8.4s, v6.4s, v0.s[1] \n" - "fmla v9.4s, v7.4s, v0.s[1] \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v12.4s, v0.s[2] \n" - "fmla v9.4s, v13.4s, v0.s[2] \n" - - "fmla v8.4s, v14.4s, v0.s[3] \n" - "fmla v9.4s, v15.4s, v0.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w7, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%1, #256] \n" - "ld1 {v4.4s, v5.4s}, [%1], #32 \n" - - "prfm pldl1keep, [%2, #32] \n" - "ld1r {v0.4s}, [%2], #4 \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v4.4s, v0.4s \n" - "fmla v9.4s, v5.4s, v0.4s \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v8.4s, v9.4s}, [%0], #32 \n" - - : "=r"(outptr0), // %0 - "=r"(tmpptr), // %1 - "=r"(kptr) // %2 - : "0"(outptr0), - "1"(tmpptr), - "2"(kptr), - "r"(bias0), // %6 - "r"(inch) // %7 - : "cc", "memory", "x4", "v0", "v4", "v5", "v6", "v7", "v8", "v9", "v12", "v13", "v14", "v15"); -#else // __aarch64__ - asm volatile( - "vdup.f32 q8, %6 \n" - "vdup.f32 q9, %6 \n" - - // inch loop - "lsr r4, %7, #2 \n" // r4 = nn = inch >> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" - - "pld [%1, #512] \n" - "vldm %1!, {d8-d15} \n" - // "vld1.f32 {d8-d11}, [%1 :128]! \n" - // "vld1.f32 {d12-d15}, [%1 :128]! \n" - - "pld [%2, #128] \n" - "vld1.f32 {d0-d1}, [%2 :128]! \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q5, d0[0] \n" - - "pld [%1, #512] \n" - "vldm %1!, {d24-d31} \n" - // "vld1.f32 {d24-d27}, [%1 :128]! \n" - // "vld1.f32 {d28-d31}, [%1 :128]! \n" - - "vmla.f32 q8, q6, d0[1] \n" - "vmla.f32 q9, q7, d0[1] \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q12, d1[0] \n" - "vmla.f32 q9, q13, d1[0] \n" - - "vmla.f32 q8, q14, d1[1] \n" - "vmla.f32 q9, q15, d1[1] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and r4, %7, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - - "pld [%1, #256] \n" - "vld1.f32 {d8-d11}, [%1 :128]! \n" - - "pld [%2, #32] \n" - "vld1.f32 {d0[],d1[]}, [%2]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q4, q0 \n" - "vmla.f32 q9, q5, q0 \n" - - "bne 2b \n" - - "3: \n" - - "vst1.f32 {d16-d19}, [%0 :128]! \n" - - : "=r"(outptr0), // %0 - "=r"(tmpptr), // %1 - "=r"(kptr) // %2 - : "0"(outptr0), - "1"(tmpptr), - "2"(kptr), - "r"(bias0), // %6 - "r"(inch) // %7 - : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8", "q9", "q12", "q13", "q14", "q15"); -#endif // __aarch64__ -#else - float sum0 = bias0; - float sum1 = bias0; - float sum2 = bias0; - float sum3 = bias0; - float sum4 = bias0; - float sum5 = bias0; - float sum6 = bias0; - float sum7 = bias0; - - for (int q = 0; q < inch; q++) - { - sum0 += tmpptr[0] * kptr[0]; - sum1 += tmpptr[1] * kptr[0]; - sum2 += tmpptr[2] * kptr[0]; - sum3 += tmpptr[3] * kptr[0]; - sum4 += tmpptr[4] * kptr[0]; - sum5 += tmpptr[5] * kptr[0]; - sum6 += tmpptr[6] * kptr[0]; - sum7 += tmpptr[7] * kptr[0]; - - tmpptr += 8; - kptr++; - } - - outptr0[0] = sum0; - outptr0[1] = sum1; - outptr0[2] = sum2; - outptr0[3] = sum3; - outptr0[4] = sum4; - outptr0[5] = sum5; - outptr0[6] = sum6; - outptr0[7] = sum7; - - outptr0 += 8; -#endif // __ARM_NEON - } - - for (; i + 3 < size; i += 4) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - const float* kptr = kernel.channel(p / 4 + p % 4); -#endif // __ARM_NEON && __aarch64__ - -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "dup v8.4s, %w6 \n" - - // inch loop - "lsr w4, %w7, #2 \n" // w4 = nn = inch >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" - - "prfm pldl1keep, [%1, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" - - "prfm pldl1keep, [%2, #128] \n" - "ld1 {v0.4s}, [%2], #16 \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v8.4s, v5.4s, v0.s[1] \n" - "fmla v8.4s, v6.4s, v0.s[2] \n" - "fmla v8.4s, v7.4s, v0.s[3] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and w4, %w7, #3 \n" // w4 = remain = inch & 3; - "cmp w4, #0 \n" - "beq 3f \n" - - "2: \n" - - "prfm pldl1keep, [%1, #128] \n" - "ld1 {v4.4s}, [%1], #16 \n" - - "prfm pldl1keep, [%2, #32] \n" - "ld1r {v0.4s}, [%2], #4 \n" - - "subs w4, w4, #1 \n" - - "fmla v8.4s, v4.4s, v0.4s \n" - - "bne 2b \n" - - "3: \n" - - "st1 {v8.4s}, [%0], #16 \n" - - : "=r"(outptr0), // %0 - "=r"(tmpptr), // %1 - "=r"(kptr) // %2 - : "0"(outptr0), - "1"(tmpptr), - "2"(kptr), - "r"(bias0), // %6 - "r"(inch) // %7 - : "cc", "memory", "x4", "v0", "v4", "v5", "v6", "v7", "v8"); -#else // __aarch64__ - asm volatile( - "vdup.f32 q8, %6 \n" - - // inch loop - "lsr r4, %7, #2 \n" // r4 = nn = inch >> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" - - "pld [%1, #512] \n" - "vldm %1!, {d8-d15} \n" - // "vld1.f32 {d8-d11}, [%1 :128]! \n" - // "vld1.f32 {d12-d15}, [%1 :128]! \n" - - "pld [%2, #128] \n" - "vld1.f32 {d0-d1}, [%2]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q8, q5, d0[1] \n" - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q8, q7, d1[1] \n" - - "bne 0b \n" - - "1: \n" - - // remain loop - "and r4, %7, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - - "pld [%1, #128] \n" - "vld1.f32 {d8-d9}, [%1 :128]! \n" - - "pld [%2, #32] \n" - "vld1.f32 {d0[],d1[]}, [%2]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q4, q0 \n" - - "bne 2b \n" - - "3: \n" - - "vst1.f32 {d16-d17}, [%0 :128]! \n" - - : "=r"(outptr0), // %0 - "=r"(tmpptr), // %1 - "=r"(kptr) // %2 - : "0"(outptr0), - "1"(tmpptr), - "2"(kptr), - "r"(bias0), // %6 - "r"(inch) // %7 - : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8"); -#endif // __aarch64__ -#else - float sum0 = bias0; - float sum1 = bias0; - float sum2 = bias0; - float sum3 = bias0; - - for (int q = 0; q < inch; q++) - { - sum0 += tmpptr[0] * kptr[0]; - sum1 += tmpptr[1] * kptr[0]; - sum2 += tmpptr[2] * kptr[0]; - sum3 += tmpptr[3] * kptr[0]; - - tmpptr += 4; - kptr++; - } - - outptr0[0] = sum0; - outptr0[1] = sum1; - outptr0[2] = sum2; - outptr0[3] = sum3; - - outptr0 += 4; -#endif // __ARM_NEON - } - - for (; i < size; i++) - { - const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); -#if __ARM_NEON && __aarch64__ - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - const float* kptr = kernel.channel(p / 4 + p % 4); -#endif // __ARM_NEON && __aarch64__ - - int q = 0; - -#if __ARM_NEON - float32x4_t _sum0 = vdupq_n_f32(0.f); - - for (; q + 3 < inch; q += 4) - { - float32x4_t _p0 = vld1q_f32(tmpptr); - tmpptr += 4; - - float32x4_t _k0 = vld1q_f32(kptr); - kptr += 4; - -#if __aarch64__ - _sum0 = vfmaq_f32(_sum0, _p0, _k0); -#else - _sum0 = vmlaq_f32(_sum0, _p0, _k0); -#endif - } - -#if __aarch64__ - float sum0 = bias0 + vaddvq_f32(_sum0); -#else - float32x2_t _ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); - float sum0 = bias0 + vget_lane_f32(vpadd_f32(_ss, _ss), 0); -#endif -#else - float sum0 = bias0; -#endif // __ARM_NEON - - for (; q < inch; q++) - { - sum0 += tmpptr[0] * kptr[0]; - tmpptr++; - kptr++; - } - - outptr0[0] = sum0; - - outptr0++; - } - } + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; - // // NOTE sgemm - // for (; p> 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 8; - - const float* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); -#else - float* tmpptr = tmp.channel(i / 8); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n" - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%0] \n" - "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n" - "sub %0, %0, #64 \n" - "st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); -#else - asm volatile( - "pld [%0, #512] \n" - "vldm %0!, {d0-d7} \n" - "pld [%0, #512] \n" - "vldm %0, {d16-d23} \n" - - // transpose 8x4 - "vtrn.32 q0, q1 \n" - "vtrn.32 q2, q3 \n" - "vtrn.32 q8, q9 \n" - "vtrn.32 q10, q11 \n" - "vswp d1, d4 \n" - "vswp d3, d6 \n" - "vswp d17, d20 \n" - "vswp d19, d22 \n" - "vswp q1, q8 \n" - "vswp q3, q10 \n" - - "vst1.f32 {d0-d3}, [%1 :128]! \n" - "vst1.f32 {d16-d19}, [%1 :128]! \n" - "sub %0, %0, #64 \n" - "vst1.f32 {d4-d7}, [%1 :128]! \n" - "vst1.f32 {d20-d23}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 3; - nn_size = (size - remain_size_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 4; - - const float* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%0] \n" - "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3"); -#else - asm volatile( - "pld [%0, #512] \n" - "vldm %0, {d0-d7} \n" - "vstm %1!, {d0-d7} \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1", "q2", "q3"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 2; - nn_size = (size - remain_size_start) >> 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 2; - - const float* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #256] \n" - "ld1 {v0.4s, v1.4s}, [%0] \n" - "st1 {v0.4s, v1.4s}, [%1], #32 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1"); -#else - asm volatile( - "pld [%0, #256] \n" - "vld1.f32 {d0-d3}, [%0 :128] \n" - "vst1.f32 {d0-d3}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_size_start; i < size; i++) - { - const float* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #128] \n" - "ld1 {v0.4s}, [%0] \n" - "st1 {v0.4s}, [%1], #16 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0"); -#else - asm volatile( - "pld [%0, #128] \n" - "vld1.f32 {d0-d1}, [%0 :128] \n" - "vst1.f32 {d0-d1}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - } - - int remain_outch_start = 0; - -#if __ARM_NEON && __aarch64__ - int nn_outch = 0; - nn_outch = outch >> 1; - remain_outch_start = nn_outch << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 2; - - float* outptr0 = top_blob.channel(p); - float* outptr1 = top_blob.channel(p + 1); - - const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p * 4 : zeros; - - int i = 0; - for (; i + 11 < size; i += 12) - { - const float* tmpptr = tmp.channel(i / 12); - - const float* kptr01 = (const float*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v8.16b, v0.16b \n" - "mov v9.16b, v0.16b \n" - "mov v10.16b, v0.16b \n" - "mov v11.16b, v0.16b \n" - "mov v12.16b, v0.16b \n" - "mov v13.16b, v0.16b \n" - "mov v14.16b, v0.16b \n" - "mov v15.16b, v0.16b \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v1.16b \n" - "mov v21.16b, v1.16b \n" - "mov v22.16b, v1.16b \n" - "mov v23.16b, v1.16b \n" - "mov v24.16b, v1.16b \n" - "mov v25.16b, v1.16b \n" - "mov v26.16b, v1.16b \n" - "mov v27.16b, v1.16b \n" - "mov v28.16b, v1.16b \n" - "mov v29.16b, v1.16b \n" - "mov v30.16b, v1.16b \n" - "mov v31.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" // w0011_01 - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - "fmla v12.4s, v4.4s, v1.s[0] \n" - "fmla v13.4s, v4.4s, v1.s[1] \n" - "fmla v14.4s, v4.4s, v1.s[2] \n" - "fmla v15.4s, v4.4s, v1.s[3] \n" - "fmla v16.4s, v4.4s, v2.s[0] \n" - "fmla v17.4s, v4.4s, v2.s[1] \n" - "fmla v18.4s, v4.4s, v2.s[2] \n" - "fmla v19.4s, v4.4s, v2.s[3] \n" - - "fmla v20.4s, v5.4s, v0.s[0] \n" - "fmla v21.4s, v5.4s, v0.s[1] \n" - "fmla v22.4s, v5.4s, v0.s[2] \n" - "fmla v23.4s, v5.4s, v0.s[3] \n" - "fmla v24.4s, v5.4s, v1.s[0] \n" - "fmla v25.4s, v5.4s, v1.s[1] \n" - "fmla v26.4s, v5.4s, v1.s[2] \n" - "fmla v27.4s, v5.4s, v1.s[3] \n" - "fmla v28.4s, v5.4s, v2.s[0] \n" - "fmla v29.4s, v5.4s, v2.s[1] \n" - "fmla v30.4s, v5.4s, v2.s[2] \n" - "fmla v31.4s, v5.4s, v2.s[3] \n" - - "fmla v8.4s, v6.4s, v3.s[0] \n" - "fmla v9.4s, v6.4s, v3.s[1] \n" - "fmla v10.4s, v6.4s, v3.s[2] \n" - "fmla v11.4s, v6.4s, v3.s[3] \n" - - "fmla v20.4s, v7.4s, v3.s[0] \n" - "fmla v21.4s, v7.4s, v3.s[1] \n" - "fmla v22.4s, v7.4s, v3.s[2] \n" - "fmla v23.4s, v7.4s, v3.s[3] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" - - "fmla v12.4s, v6.4s, v0.s[0] \n" - "fmla v13.4s, v6.4s, v0.s[1] \n" - "fmla v14.4s, v6.4s, v0.s[2] \n" - "fmla v15.4s, v6.4s, v0.s[3] \n" - "fmla v16.4s, v6.4s, v1.s[0] \n" - "fmla v17.4s, v6.4s, v1.s[1] \n" - "fmla v18.4s, v6.4s, v1.s[2] \n" - "fmla v19.4s, v6.4s, v1.s[3] \n" - - "fmla v24.4s, v7.4s, v0.s[0] \n" - "fmla v25.4s, v7.4s, v0.s[1] \n" - "fmla v26.4s, v7.4s, v0.s[2] \n" - "fmla v27.4s, v7.4s, v0.s[3] \n" - "fmla v28.4s, v7.4s, v1.s[0] \n" - "fmla v29.4s, v7.4s, v1.s[1] \n" - "fmla v30.4s, v7.4s, v1.s[2] \n" - "fmla v31.4s, v7.4s, v1.s[3] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" // w2233_01 - - "fmla v8.4s, v4.4s, v2.s[0] \n" - "fmla v9.4s, v4.4s, v2.s[1] \n" - "fmla v10.4s, v4.4s, v2.s[2] \n" - "fmla v11.4s, v4.4s, v2.s[3] \n" - "fmla v12.4s, v4.4s, v3.s[0] \n" - "fmla v13.4s, v4.4s, v3.s[1] \n" - "fmla v14.4s, v4.4s, v3.s[2] \n" - "fmla v15.4s, v4.4s, v3.s[3] \n" - - "fmla v20.4s, v5.4s, v2.s[0] \n" - "fmla v21.4s, v5.4s, v2.s[1] \n" - "fmla v22.4s, v5.4s, v2.s[2] \n" - "fmla v23.4s, v5.4s, v2.s[3] \n" - "fmla v24.4s, v5.4s, v3.s[0] \n" - "fmla v25.4s, v5.4s, v3.s[1] \n" - "fmla v26.4s, v5.4s, v3.s[2] \n" - "fmla v27.4s, v5.4s, v3.s[3] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" - - "fmla v16.4s, v4.4s, v0.s[0] \n" - "fmla v17.4s, v4.4s, v0.s[1] \n" - "fmla v18.4s, v4.4s, v0.s[2] \n" - "fmla v19.4s, v4.4s, v0.s[3] \n" - - "fmla v28.4s, v5.4s, v0.s[0] \n" - "fmla v29.4s, v5.4s, v0.s[1] \n" - "fmla v30.4s, v5.4s, v0.s[2] \n" - "fmla v31.4s, v5.4s, v0.s[3] \n" - - "fmla v8.4s, v6.4s, v1.s[0] \n" - "fmla v9.4s, v6.4s, v1.s[1] \n" - "fmla v10.4s, v6.4s, v1.s[2] \n" - "fmla v11.4s, v6.4s, v1.s[3] \n" - "fmla v12.4s, v6.4s, v2.s[0] \n" - "fmla v13.4s, v6.4s, v2.s[1] \n" - "fmla v14.4s, v6.4s, v2.s[2] \n" - "fmla v15.4s, v6.4s, v2.s[3] \n" - "fmla v16.4s, v6.4s, v3.s[0] \n" - "fmla v17.4s, v6.4s, v3.s[1] \n" - "fmla v18.4s, v6.4s, v3.s[2] \n" - "fmla v19.4s, v6.4s, v3.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v20.4s, v7.4s, v1.s[0] \n" - "fmla v21.4s, v7.4s, v1.s[1] \n" - "fmla v22.4s, v7.4s, v1.s[2] \n" - "fmla v23.4s, v7.4s, v1.s[3] \n" - "fmla v24.4s, v7.4s, v2.s[0] \n" - "fmla v25.4s, v7.4s, v2.s[1] \n" - "fmla v26.4s, v7.4s, v2.s[2] \n" - "fmla v27.4s, v7.4s, v2.s[3] \n" - "fmla v28.4s, v7.4s, v3.s[0] \n" - "fmla v29.4s, v7.4s, v3.s[1] \n" - "fmla v30.4s, v7.4s, v3.s[2] \n" - "fmla v31.4s, v7.4s, v3.s[3] \n" - - "bne 0b \n" - - "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" - "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" - "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 7 < size; i += 8) - { - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - - const float* kptr01 = (const float*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v0.16b \n" - "mov v21.16b, v0.16b \n" - "mov v22.16b, v0.16b \n" - "mov v23.16b, v0.16b \n" - "mov v24.16b, v1.16b \n" - "mov v25.16b, v1.16b \n" - "mov v26.16b, v1.16b \n" - "mov v27.16b, v1.16b \n" - "mov v28.16b, v1.16b \n" - "mov v29.16b, v1.16b \n" - "mov v30.16b, v1.16b \n" - "mov v31.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // r4 r5 r6 r7 - - "fmla v20.4s, v8.4s, v4.s[0] \n" - "fmla v21.4s, v8.4s, v5.s[0] \n" - "fmla v22.4s, v8.4s, v6.s[0] \n" - "fmla v23.4s, v8.4s, v7.s[0] \n" - - "fmla v24.4s, v9.4s, v0.s[0] \n" - "fmla v25.4s, v9.4s, v1.s[0] \n" - "fmla v26.4s, v9.4s, v2.s[0] \n" - "fmla v27.4s, v9.4s, v3.s[0] \n" - "fmla v28.4s, v9.4s, v4.s[0] \n" - "fmla v29.4s, v9.4s, v5.s[0] \n" - "fmla v30.4s, v9.4s, v6.s[0] \n" - "fmla v31.4s, v9.4s, v7.s[0] \n" - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v10.4s, v2.s[1] \n" - "fmla v19.4s, v10.4s, v3.s[1] \n" - "fmla v20.4s, v10.4s, v4.s[1] \n" - "fmla v21.4s, v10.4s, v5.s[1] \n" - "fmla v22.4s, v10.4s, v6.s[1] \n" - "fmla v23.4s, v10.4s, v7.s[1] \n" - - "fmla v24.4s, v11.4s, v0.s[1] \n" - "fmla v25.4s, v11.4s, v1.s[1] \n" - "fmla v26.4s, v11.4s, v2.s[1] \n" - "fmla v27.4s, v11.4s, v3.s[1] \n" - "fmla v28.4s, v11.4s, v4.s[1] \n" - "fmla v29.4s, v11.4s, v5.s[1] \n" - "fmla v30.4s, v11.4s, v6.s[1] \n" - "fmla v31.4s, v11.4s, v7.s[1] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v12.4s, v2.s[2] \n" - "fmla v19.4s, v12.4s, v3.s[2] \n" - "fmla v20.4s, v12.4s, v4.s[2] \n" - "fmla v21.4s, v12.4s, v5.s[2] \n" - "fmla v22.4s, v12.4s, v6.s[2] \n" - "fmla v23.4s, v12.4s, v7.s[2] \n" - - "fmla v24.4s, v13.4s, v0.s[2] \n" - "fmla v25.4s, v13.4s, v1.s[2] \n" - "fmla v26.4s, v13.4s, v2.s[2] \n" - "fmla v27.4s, v13.4s, v3.s[2] \n" - "fmla v28.4s, v13.4s, v4.s[2] \n" - "fmla v29.4s, v13.4s, v5.s[2] \n" - "fmla v30.4s, v13.4s, v6.s[2] \n" - "fmla v31.4s, v13.4s, v7.s[2] \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v14.4s, v2.s[3] \n" - "fmla v19.4s, v14.4s, v3.s[3] \n" - "fmla v20.4s, v14.4s, v4.s[3] \n" - "fmla v21.4s, v14.4s, v5.s[3] \n" - "fmla v22.4s, v14.4s, v6.s[3] \n" - "fmla v23.4s, v14.4s, v7.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v24.4s, v15.4s, v0.s[3] \n" - "fmla v25.4s, v15.4s, v1.s[3] \n" - "fmla v26.4s, v15.4s, v2.s[3] \n" - "fmla v27.4s, v15.4s, v3.s[3] \n" - "fmla v28.4s, v15.4s, v4.s[3] \n" - "fmla v29.4s, v15.4s, v5.s[3] \n" - "fmla v30.4s, v15.4s, v6.s[3] \n" - "fmla v31.4s, v15.4s, v7.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" - "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n" - "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 3 < size; i += 4) - { - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - - const float* kptr01 = (const float*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v1.16b \n" - "mov v21.16b, v1.16b \n" - "mov v22.16b, v1.16b \n" - "mov v23.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "fmla v20.4s, v9.4s, v0.s[0] \n" - "fmla v21.4s, v9.4s, v1.s[0] \n" - "fmla v22.4s, v9.4s, v2.s[0] \n" - "fmla v23.4s, v9.4s, v3.s[0] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v10.4s, v2.s[1] \n" - "fmla v19.4s, v10.4s, v3.s[1] \n" - - "fmla v20.4s, v11.4s, v0.s[1] \n" - "fmla v21.4s, v11.4s, v1.s[1] \n" - "fmla v22.4s, v11.4s, v2.s[1] \n" - "fmla v23.4s, v11.4s, v3.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v12.4s, v2.s[2] \n" - "fmla v19.4s, v12.4s, v3.s[2] \n" - - "fmla v20.4s, v13.4s, v0.s[2] \n" - "fmla v21.4s, v13.4s, v1.s[2] \n" - "fmla v22.4s, v13.4s, v2.s[2] \n" - "fmla v23.4s, v13.4s, v3.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v14.4s, v2.s[3] \n" - "fmla v19.4s, v14.4s, v3.s[3] \n" - - "fmla v20.4s, v15.4s, v0.s[3] \n" - "fmla v21.4s, v15.4s, v1.s[3] \n" - "fmla v22.4s, v15.4s, v2.s[3] \n" - "fmla v23.4s, v15.4s, v3.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - } - for (; i + 1 < size; i += 2) - { - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - - const float* kptr01 = (const float*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v1.16b \n" - "mov v19.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4s, v1.4s}, [%3], #32 \n" // r0 r1 - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v9.4s, v0.s[0] \n" - "fmla v19.4s, v9.4s, v1.s[0] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v11.4s, v0.s[1] \n" - "fmla v19.4s, v11.4s, v1.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v13.4s, v0.s[2] \n" - "fmla v19.4s, v13.4s, v1.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v15.4s, v0.s[3] \n" - "fmla v19.4s, v15.4s, v1.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s}, [%1], #32 \n" - "st1 {v18.4s, v19.4s}, [%2], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); - } - for (; i < size; i++) - { - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - - const float* kptr01 = (const float*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.4s, v17.4s}, [%10] \n" - - "0: \n" - - "prfm pldl1keep, [%3, #128] \n" - "ld1 {v0.4s}, [%3], #16 \n" // r0 - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v9.4s, v0.s[0] \n" - - "prfm pldl1keep, [%4, #512] \n" - "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v11.4s, v0.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v13.4s, v0.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v15.4s, v0.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s}, [%1], #16 \n" - "st1 {v17.4s}, [%2], #16 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); - } - } - -#endif // __ARM_NEON && __aarch64__ - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - float* outptr0 = top_blob.channel(p); - - const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p * 4 : zeros; - - int i = 0; -#if __aarch64__ - for (; i + 11 < size; i += 12) - { - float* tmpptr = tmp.channel(i / 12); - - const float* kptr0 = (const float*)kernel.channel(p / 2 + p % 2); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v8.16b, v0.16b \n" - "mov v9.16b, v0.16b \n" - "mov v10.16b, v0.16b \n" - "mov v11.16b, v0.16b \n" - "mov v12.16b, v0.16b \n" - "mov v13.16b, v0.16b \n" - "mov v14.16b, v0.16b \n" - "mov v15.16b, v0.16b \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // w0123_0 - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - "fmla v12.4s, v4.4s, v1.s[0] \n" - "fmla v13.4s, v4.4s, v1.s[1] \n" - "fmla v14.4s, v4.4s, v1.s[2] \n" - "fmla v15.4s, v4.4s, v1.s[3] \n" - "fmla v16.4s, v4.4s, v2.s[0] \n" - "fmla v17.4s, v4.4s, v2.s[1] \n" - "fmla v18.4s, v4.4s, v2.s[2] \n" - "fmla v19.4s, v4.4s, v2.s[3] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" - - "fmla v8.4s, v5.4s, v3.s[0] \n" - "fmla v9.4s, v5.4s, v3.s[1] \n" - "fmla v10.4s, v5.4s, v3.s[2] \n" - "fmla v11.4s, v5.4s, v3.s[3] \n" - "fmla v12.4s, v5.4s, v20.s[0] \n" - "fmla v13.4s, v5.4s, v20.s[1] \n" - "fmla v14.4s, v5.4s, v20.s[2] \n" - "fmla v15.4s, v5.4s, v20.s[3] \n" - "fmla v16.4s, v5.4s, v21.s[0] \n" - "fmla v17.4s, v5.4s, v21.s[1] \n" - "fmla v18.4s, v5.4s, v21.s[2] \n" - "fmla v19.4s, v5.4s, v21.s[3] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" - - "fmla v8.4s, v6.4s, v22.s[0] \n" - "fmla v9.4s, v6.4s, v22.s[1] \n" - "fmla v10.4s, v6.4s, v22.s[2] \n" - "fmla v11.4s, v6.4s, v22.s[3] \n" - "fmla v12.4s, v6.4s, v23.s[0] \n" - "fmla v13.4s, v6.4s, v23.s[1] \n" - "fmla v14.4s, v6.4s, v23.s[2] \n" - "fmla v15.4s, v6.4s, v23.s[3] \n" - "fmla v16.4s, v6.4s, v24.s[0] \n" - "fmla v17.4s, v6.4s, v24.s[1] \n" - "fmla v18.4s, v6.4s, v24.s[2] \n" - "fmla v19.4s, v6.4s, v24.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v8.4s, v7.4s, v25.s[0] \n" - "fmla v9.4s, v7.4s, v25.s[1] \n" - "fmla v10.4s, v7.4s, v25.s[2] \n" - "fmla v11.4s, v7.4s, v25.s[3] \n" - "fmla v12.4s, v7.4s, v26.s[0] \n" - "fmla v13.4s, v7.4s, v26.s[1] \n" - "fmla v14.4s, v7.4s, v26.s[2] \n" - "fmla v15.4s, v7.4s, v26.s[3] \n" - "fmla v16.4s, v7.4s, v27.s[0] \n" - "fmla v17.4s, v7.4s, v27.s[1] \n" - "fmla v18.4s, v7.4s, v27.s[2] \n" - "fmla v19.4s, v7.4s, v27.s[3] \n" - - "bne 0b \n" - - "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" - "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); - } -#endif - for (; i + 7 < size; i += 8) - { -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - const float* kptr0 = (const float*)kernel.channel(p / 2 + p % 2); -#else - float* tmpptr = tmp.channel(i / 8); - const float* kptr0 = (const float*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v0.16b \n" - "mov v21.16b, v0.16b \n" - "mov v22.16b, v0.16b \n" - "mov v23.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%2], #64 \n" // r4 r5 r6 r7 - - "fmla v20.4s, v8.4s, v4.s[0] \n" - "fmla v21.4s, v8.4s, v5.s[0] \n" - "fmla v22.4s, v8.4s, v6.s[0] \n" - "fmla v23.4s, v8.4s, v7.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - "fmla v18.4s, v9.4s, v2.s[1] \n" - "fmla v19.4s, v9.4s, v3.s[1] \n" - "fmla v20.4s, v9.4s, v4.s[1] \n" - "fmla v21.4s, v9.4s, v5.s[1] \n" - "fmla v22.4s, v9.4s, v6.s[1] \n" - "fmla v23.4s, v9.4s, v7.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - "fmla v18.4s, v10.4s, v2.s[2] \n" - "fmla v19.4s, v10.4s, v3.s[2] \n" - "fmla v20.4s, v10.4s, v4.s[2] \n" - "fmla v21.4s, v10.4s, v5.s[2] \n" - "fmla v22.4s, v10.4s, v6.s[2] \n" - "fmla v23.4s, v10.4s, v7.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - "fmla v18.4s, v11.4s, v2.s[3] \n" - "fmla v19.4s, v11.4s, v3.s[3] \n" - "fmla v20.4s, v11.4s, v4.s[3] \n" - "fmla v21.4s, v11.4s, v5.s[3] \n" - "fmla v22.4s, v11.4s, v6.s[3] \n" - "fmla v23.4s, v11.4s, v7.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - "vmov q10, q0 \n" - "vmov q11, q0 \n" - "vmov q12, q0 \n" - "vmov q13, q0 \n" - "vmov q14, q0 \n" - "vmov q15, q0 \n" - - "0: \n" - - "pld [%2, #512] \n" - "vldm %2!, {d0-d7} \n" - - "pld [%3, #512] \n" - "vldm %3!, {d8-d15} \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d0[1] \n" - "vmla.f32 q10, q4, d1[0] \n" - "vmla.f32 q11, q4, d1[1] \n" - "vmla.f32 q12, q4, d2[0] \n" - "vmla.f32 q13, q4, d2[1] \n" - "vmla.f32 q14, q4, d3[0] \n" - "vmla.f32 q15, q4, d3[1] \n" - - "vmla.f32 q8, q5, d4[0] \n" - "vmla.f32 q9, q5, d4[1] \n" - "vmla.f32 q10, q5, d5[0] \n" - "vmla.f32 q11, q5, d5[1] \n" - "vmla.f32 q12, q5, d6[0] \n" - "vmla.f32 q13, q5, d6[1] \n" - "vmla.f32 q14, q5, d7[0] \n" - "vmla.f32 q15, q5, d7[1] \n" - - "pld [%2, #512] \n" - "vldm %2!, {d0-d7} \n" - - "vmla.f32 q8, q6, d0[0] \n" - "vmla.f32 q9, q6, d0[1] \n" - "vmla.f32 q10, q6, d1[0] \n" - "vmla.f32 q11, q6, d1[1] \n" - "vmla.f32 q12, q6, d2[0] \n" - "vmla.f32 q13, q6, d2[1] \n" - "vmla.f32 q14, q6, d3[0] \n" - "vmla.f32 q15, q6, d3[1] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d4[0] \n" - "vmla.f32 q9, q7, d4[1] \n" - "vmla.f32 q10, q7, d5[0] \n" - "vmla.f32 q11, q7, d5[1] \n" - "vmla.f32 q12, q7, d6[0] \n" - "vmla.f32 q13, q7, d6[1] \n" - "vmla.f32 q14, q7, d7[0] \n" - "vmla.f32 q15, q7, d7[1] \n" - - "bne 0b \n" - - "vstm %1!, {d16-d23} \n" - "vstm %1!, {d24-d31} \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif - } - for (; i + 3 < size; i += 4) - { -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - const float* kptr0 = (const float*)kernel.channel(p / 2 + p % 2); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); - const float* kptr0 = (const float*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - "fmla v18.4s, v9.4s, v2.s[1] \n" - "fmla v19.4s, v9.4s, v3.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - "fmla v18.4s, v10.4s, v2.s[2] \n" - "fmla v19.4s, v10.4s, v3.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - "fmla v18.4s, v11.4s, v2.s[3] \n" - "fmla v19.4s, v11.4s, v3.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - "vmov q10, q0 \n" - "vmov q11, q0 \n" - - "0: \n" - - "pld [%2, #512] \n" - "vldm %2!, {d0-d7} \n" - - "pld [%3, #512] \n" - "vldm %3!, {d8-d15} \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d2[0] \n" - "vmla.f32 q10, q4, d4[0] \n" - "vmla.f32 q11, q4, d6[0] \n" - - "vmla.f32 q8, q5, d0[1] \n" - "vmla.f32 q9, q5, d2[1] \n" - "vmla.f32 q10, q5, d4[1] \n" - "vmla.f32 q11, q5, d6[1] \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q9, q6, d3[0] \n" - "vmla.f32 q10, q6, d5[0] \n" - "vmla.f32 q11, q6, d7[0] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d1[1] \n" - "vmla.f32 q9, q7, d3[1] \n" - "vmla.f32 q10, q7, d5[1] \n" - "vmla.f32 q11, q7, d7[1] \n" - - "bne 0b \n" - - "vstm %1!, {d16-d23} \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); -#endif - } - for (; i + 1 < size; i += 2) - { -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - const float* kptr0 = (const float*)kernel.channel(p / 2 + p % 2); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); - const float* kptr0 = (const float*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v0.4s, v1.4s}, [%2], #32 \n" // r0 r1 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s}, [%1], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v16", "v17"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - - "0: \n" - - "pld [%2, #256] \n" - "vld1.f32 {d0-d3}, [%2 :128]! \n" - - "pld [%3, #512] \n" - "vldm %3!, {d8-d15} \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d2[0] \n" - - "vmla.f32 q8, q5, d0[1] \n" - "vmla.f32 q9, q5, d2[1] \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q9, q6, d3[0] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d1[1] \n" - "vmla.f32 q9, q7, d3[1] \n" - - "bne 0b \n" - - "vst1.f32 {d16-d19}, [%1 :128]! \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q4", "q5", "q6", "q7", "q8", "q9"); -#endif - } - for (; i < size; i++) - { -#if __aarch64__ - float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - const float* kptr0 = (const float*)kernel.channel(p / 2 + p % 2); -#else - float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); - const float* kptr0 = (const float*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v16.4s}, [%8] \n" - - "0: \n" - - "prfm pldl1keep, [%2, #128] \n" - "ld1 {v0.4s}, [%2], #16 \n" // r0 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v16.4s, v9.4s, v0.s[1] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v16.4s, v11.4s, v0.s[3] \n" - - "bne 0b \n" - - "st1 {v16.4s}, [%1], #16 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v16"); -#else - asm volatile( - "vld1.f32 {d16-d17}, [%8] \n" - - "0: \n" - - "pld [%2, #128] \n" - "vld1.f32 {d0-d1}, [%2 :128]! \n" - - "pld [%3, #512] \n" - "vldm %3!, {d8-d15} \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q8, q5, d0[1] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q8, q7, d1[1] \n" - - "bne 0b \n" - - "vst1.f32 {d16-d17}, [%1 :128]! \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q4", "q5", "q6", "q7", "q8"); -#endif - } - } + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; - // // NOTE sgemm - // for (; p= 12) - tmp.create(12, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 8) - tmp.create(8, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 4) - tmp.create(4, inch, size / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 2) - tmp.create(2, inch, size / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else // if (size >= 1) - tmp.create(1, inch, size, elemsize, elempack, opt.workspace_allocator); -#else - if (size >= 8) - tmp.create(8, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 4) - tmp.create(4, inch, size / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 2) - tmp.create(2, inch, size / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else // if (size >= 1) - tmp.create(1, inch, size, elemsize, elempack, opt.workspace_allocator); -#endif - { - int nn_size; - int remain_size_start; - -#if __aarch64__ - nn_size = size / 12; - remain_size_start = nn_size * 12; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = ii * 12; - - const unsigned short* img0 = bottom_blob.channel(0); - img0 += i * 4; - - unsigned short* tmpptr = tmp.channel(i / 12); - - for (int q = 0; q < inch; q++) - { - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.4h, v5.4h, v6.4h, v7.4h}, [%0] \n" - "st1 {v0.8h}, [%1], #16 \n" - "st1 {v4.4h}, [%1], #8 \n" - "st1 {v1.8h}, [%1], #16 \n" - "st1 {v5.4h}, [%1], #8 \n" - "sub %0, %0, #64 \n" - "st1 {v2.8h}, [%1], #16 \n" - "st1 {v6.4h}, [%1], #8 \n" - "st1 {v3.8h}, [%1], #16 \n" - "st1 {v7.4h}, [%1], #8 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); - img0 += bottom_blob.cstep * 4; - } - } -#else - remain_size_start = 0; -#endif - nn_size = (size - remain_size_start) >> 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 8; - - const unsigned short* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); -#else - unsigned short* tmpptr = tmp.channel(i / 8); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" - "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3"); -#else - asm volatile( - "pld [%0, #256] \n" - "vld4.u16 {d0-d3}, [%0]! \n" - "pld [%0, #256] \n" - "vld4.u16 {d4-d7}, [%0] \n" - "sub %0, %0, #32 \n" - "vst1.u16 {d0}, [%1 :64]! \n" - "vst1.u16 {d4}, [%1 :64]! \n" - "vst1.u16 {d1}, [%1 :64]! \n" - "vst1.u16 {d5}, [%1 :64]! \n" - "vst1.u16 {d2}, [%1 :64]! \n" - "vst1.u16 {d6}, [%1 :64]! \n" - "vst1.u16 {d3}, [%1 :64]! \n" - "vst1.u16 {d7}, [%1 :64]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1", "q2", "q3"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 3; - nn_size = (size - remain_size_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 4; - - const unsigned short* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #256] \n" - "ld1 {v0.8h, v1.8h}, [%0] \n" - "st1 {v0.8h, v1.8h}, [%1], #32 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1"); -#else - asm volatile( - "pld [%0, #256] \n" - "vld1.u16 {d0-d3}, [%0 :128] \n" - "vst1.u16 {d0-d3}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 2; - nn_size = (size - remain_size_start) >> 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 2; - - const unsigned short* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #128] \n" - "ld1 {v0.8h}, [%0] \n" - "st1 {v0.8h}, [%1], #16 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0"); -#else - asm volatile( - "pld [%0, #128] \n" - "vld1.u16 {d0-d1}, [%0 :128] \n" - "vst1.u16 {d0-d1}, [%1 :128]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - - remain_size_start += nn_size << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_size_start; i < size; i++) - { - const unsigned short* img0 = bottom_blob.channel(0); - img0 += i * 4; - -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); -#endif - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #64] \n" - "ld1 {v0.4h}, [%0] \n" - "st1 {v0.4h}, [%1], #8 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0"); -#else - asm volatile( - "pld [%0, #64] \n" - "vld1.u16 {d0}, [%0 :64] \n" - "vst1.u16 {d0}, [%1 :64]! \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0"); -#endif // __aarch64__ - img0 += bottom_blob.cstep * 4; - } - } - } - - int remain_outch_start = 0; - -#if __ARM_NEON && __aarch64__ - int nn_outch = 0; - nn_outch = outch >> 1; - remain_outch_start = nn_outch << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 2; - - unsigned short* outptr0 = top_blob.channel(p); - unsigned short* outptr1 = top_blob.channel(p + 1); - - const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p * 4 : zeros; - - int i = 0; - for (; i + 11 < size; i += 12) - { - const unsigned short* tmpptr = tmp.channel(i / 12); - - const unsigned short* kptr01 = (const unsigned short*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v8.16b, v0.16b \n" - "mov v9.16b, v0.16b \n" - "mov v10.16b, v0.16b \n" - "mov v11.16b, v0.16b \n" - "mov v12.16b, v0.16b \n" - "mov v13.16b, v0.16b \n" - "mov v14.16b, v0.16b \n" - "mov v15.16b, v0.16b \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v1.16b \n" - "mov v21.16b, v1.16b \n" - "mov v22.16b, v1.16b \n" - "mov v23.16b, v1.16b \n" - "mov v24.16b, v1.16b \n" - "mov v25.16b, v1.16b \n" - "mov v26.16b, v1.16b \n" - "mov v27.16b, v1.16b \n" - "mov v28.16b, v1.16b \n" - "mov v29.16b, v1.16b \n" - "mov v30.16b, v1.16b \n" - "mov v31.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%4], #32 \n" // w0011_01 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v4.4s, v4.4h, #16 \n" - "shll v5.4s, v5.4h, #16 \n" - "shll v6.4s, v6.4h, #16 \n" - "shll v7.4s, v7.4h, #16 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - "fmla v12.4s, v4.4s, v1.s[0] \n" - "fmla v13.4s, v4.4s, v1.s[1] \n" - "fmla v14.4s, v4.4s, v1.s[2] \n" - "fmla v15.4s, v4.4s, v1.s[3] \n" - "fmla v16.4s, v4.4s, v2.s[0] \n" - "fmla v17.4s, v4.4s, v2.s[1] \n" - "fmla v18.4s, v4.4s, v2.s[2] \n" - "fmla v19.4s, v4.4s, v2.s[3] \n" - - "fmla v20.4s, v5.4s, v0.s[0] \n" - "fmla v21.4s, v5.4s, v0.s[1] \n" - "fmla v22.4s, v5.4s, v0.s[2] \n" - "fmla v23.4s, v5.4s, v0.s[3] \n" - "fmla v24.4s, v5.4s, v1.s[0] \n" - "fmla v25.4s, v5.4s, v1.s[1] \n" - "fmla v26.4s, v5.4s, v1.s[2] \n" - "fmla v27.4s, v5.4s, v1.s[3] \n" - "fmla v28.4s, v5.4s, v2.s[0] \n" - "fmla v29.4s, v5.4s, v2.s[1] \n" - "fmla v30.4s, v5.4s, v2.s[2] \n" - "fmla v31.4s, v5.4s, v2.s[3] \n" - - "fmla v8.4s, v6.4s, v3.s[0] \n" - "fmla v9.4s, v6.4s, v3.s[1] \n" - "fmla v10.4s, v6.4s, v3.s[2] \n" - "fmla v11.4s, v6.4s, v3.s[3] \n" - - "fmla v20.4s, v7.4s, v3.s[0] \n" - "fmla v21.4s, v7.4s, v3.s[1] \n" - "fmla v22.4s, v7.4s, v3.s[2] \n" - "fmla v23.4s, v7.4s, v3.s[3] \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "fmla v12.4s, v6.4s, v0.s[0] \n" - "fmla v13.4s, v6.4s, v0.s[1] \n" - "fmla v14.4s, v6.4s, v0.s[2] \n" - "fmla v15.4s, v6.4s, v0.s[3] \n" - "fmla v16.4s, v6.4s, v1.s[0] \n" - "fmla v17.4s, v6.4s, v1.s[1] \n" - "fmla v18.4s, v6.4s, v1.s[2] \n" - "fmla v19.4s, v6.4s, v1.s[3] \n" - - "fmla v24.4s, v7.4s, v0.s[0] \n" - "fmla v25.4s, v7.4s, v0.s[1] \n" - "fmla v26.4s, v7.4s, v0.s[2] \n" - "fmla v27.4s, v7.4s, v0.s[3] \n" - "fmla v28.4s, v7.4s, v1.s[0] \n" - "fmla v29.4s, v7.4s, v1.s[1] \n" - "fmla v30.4s, v7.4s, v1.s[2] \n" - "fmla v31.4s, v7.4s, v1.s[3] \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%4], #32 \n" // w2233_01 - - "shll v4.4s, v4.4h, #16 \n" - "shll v5.4s, v5.4h, #16 \n" - "shll v6.4s, v6.4h, #16 \n" - "shll v7.4s, v7.4h, #16 \n" - - "fmla v8.4s, v4.4s, v2.s[0] \n" - "fmla v9.4s, v4.4s, v2.s[1] \n" - "fmla v10.4s, v4.4s, v2.s[2] \n" - "fmla v11.4s, v4.4s, v2.s[3] \n" - "fmla v12.4s, v4.4s, v3.s[0] \n" - "fmla v13.4s, v4.4s, v3.s[1] \n" - "fmla v14.4s, v4.4s, v3.s[2] \n" - "fmla v15.4s, v4.4s, v3.s[3] \n" - - "fmla v20.4s, v5.4s, v2.s[0] \n" - "fmla v21.4s, v5.4s, v2.s[1] \n" - "fmla v22.4s, v5.4s, v2.s[2] \n" - "fmla v23.4s, v5.4s, v2.s[3] \n" - "fmla v24.4s, v5.4s, v3.s[0] \n" - "fmla v25.4s, v5.4s, v3.s[1] \n" - "fmla v26.4s, v5.4s, v3.s[2] \n" - "fmla v27.4s, v5.4s, v3.s[3] \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "fmla v16.4s, v4.4s, v0.s[0] \n" - "fmla v17.4s, v4.4s, v0.s[1] \n" - "fmla v18.4s, v4.4s, v0.s[2] \n" - "fmla v19.4s, v4.4s, v0.s[3] \n" - - "fmla v28.4s, v5.4s, v0.s[0] \n" - "fmla v29.4s, v5.4s, v0.s[1] \n" - "fmla v30.4s, v5.4s, v0.s[2] \n" - "fmla v31.4s, v5.4s, v0.s[3] \n" - - "fmla v8.4s, v6.4s, v1.s[0] \n" - "fmla v9.4s, v6.4s, v1.s[1] \n" - "fmla v10.4s, v6.4s, v1.s[2] \n" - "fmla v11.4s, v6.4s, v1.s[3] \n" - "fmla v12.4s, v6.4s, v2.s[0] \n" - "fmla v13.4s, v6.4s, v2.s[1] \n" - "fmla v14.4s, v6.4s, v2.s[2] \n" - "fmla v15.4s, v6.4s, v2.s[3] \n" - "fmla v16.4s, v6.4s, v3.s[0] \n" - "fmla v17.4s, v6.4s, v3.s[1] \n" - "fmla v18.4s, v6.4s, v3.s[2] \n" - "fmla v19.4s, v6.4s, v3.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v20.4s, v7.4s, v1.s[0] \n" - "fmla v21.4s, v7.4s, v1.s[1] \n" - "fmla v22.4s, v7.4s, v1.s[2] \n" - "fmla v23.4s, v7.4s, v1.s[3] \n" - "fmla v24.4s, v7.4s, v2.s[0] \n" - "fmla v25.4s, v7.4s, v2.s[1] \n" - "fmla v26.4s, v7.4s, v2.s[2] \n" - "fmla v27.4s, v7.4s, v2.s[3] \n" - "fmla v28.4s, v7.4s, v3.s[0] \n" - "fmla v29.4s, v7.4s, v3.s[1] \n" - "fmla v30.4s, v7.4s, v3.s[2] \n" - "fmla v31.4s, v7.4s, v3.s[3] \n" - - "bne 0b \n" - - "shrn v8.4h, v8.4s, #16 \n" - "shrn v9.4h, v9.4s, #16 \n" - "shrn v10.4h, v10.4s, #16 \n" - "shrn v11.4h, v11.4s, #16 \n" - - "shrn v12.4h, v12.4s, #16 \n" - "shrn v13.4h, v13.4s, #16 \n" - "shrn v14.4h, v14.4s, #16 \n" - "shrn v15.4h, v15.4s, #16 \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "shrn v20.4h, v20.4s, #16 \n" - "shrn v21.4h, v21.4s, #16 \n" - "shrn v22.4h, v22.4s, #16 \n" - "shrn v23.4h, v23.4s, #16 \n" - - "shrn v24.4h, v24.4s, #16 \n" - "shrn v25.4h, v25.4s, #16 \n" - "shrn v26.4h, v26.4s, #16 \n" - "shrn v27.4h, v27.4s, #16 \n" - - "shrn v28.4h, v28.4s, #16 \n" - "shrn v29.4h, v29.4s, #16 \n" - "shrn v30.4h, v30.4s, #16 \n" - "shrn v31.4h, v31.4s, #16 \n" - - "st1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%1], #32 \n" - "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" - "st1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%1], #32 \n" - "st1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%2], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 7 < size; i += 8) - { - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - - const unsigned short* kptr01 = (const unsigned short*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v0.16b \n" - "mov v21.16b, v0.16b \n" - "mov v22.16b, v0.16b \n" - "mov v23.16b, v0.16b \n" - "mov v24.16b, v1.16b \n" - "mov v25.16b, v1.16b \n" - "mov v26.16b, v1.16b \n" - "mov v27.16b, v1.16b \n" - "mov v28.16b, v1.16b \n" - "mov v29.16b, v1.16b \n" - "mov v30.16b, v1.16b \n" - "mov v31.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%3], #32 \n" // r4 r5 r6 r7 - - "shll v4.4s, v4.4h, #16 \n" - "shll v5.4s, v5.4h, #16 \n" - "shll v6.4s, v6.4h, #16 \n" - "shll v7.4s, v7.4h, #16 \n" - - "fmla v20.4s, v8.4s, v4.s[0] \n" - "fmla v21.4s, v8.4s, v5.s[0] \n" - "fmla v22.4s, v8.4s, v6.s[0] \n" - "fmla v23.4s, v8.4s, v7.s[0] \n" - - "fmla v24.4s, v9.4s, v0.s[0] \n" - "fmla v25.4s, v9.4s, v1.s[0] \n" - "fmla v26.4s, v9.4s, v2.s[0] \n" - "fmla v27.4s, v9.4s, v3.s[0] \n" - "fmla v28.4s, v9.4s, v4.s[0] \n" - "fmla v29.4s, v9.4s, v5.s[0] \n" - "fmla v30.4s, v9.4s, v6.s[0] \n" - "fmla v31.4s, v9.4s, v7.s[0] \n" - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v10.4s, v2.s[1] \n" - "fmla v19.4s, v10.4s, v3.s[1] \n" - "fmla v20.4s, v10.4s, v4.s[1] \n" - "fmla v21.4s, v10.4s, v5.s[1] \n" - "fmla v22.4s, v10.4s, v6.s[1] \n" - "fmla v23.4s, v10.4s, v7.s[1] \n" - - "fmla v24.4s, v11.4s, v0.s[1] \n" - "fmla v25.4s, v11.4s, v1.s[1] \n" - "fmla v26.4s, v11.4s, v2.s[1] \n" - "fmla v27.4s, v11.4s, v3.s[1] \n" - "fmla v28.4s, v11.4s, v4.s[1] \n" - "fmla v29.4s, v11.4s, v5.s[1] \n" - "fmla v30.4s, v11.4s, v6.s[1] \n" - "fmla v31.4s, v11.4s, v7.s[1] \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 - - "shll v12.4s, v12.4h, #16 \n" - "shll v13.4s, v13.4h, #16 \n" - "shll v14.4s, v14.4h, #16 \n" - "shll v15.4s, v15.4h, #16 \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v12.4s, v2.s[2] \n" - "fmla v19.4s, v12.4s, v3.s[2] \n" - "fmla v20.4s, v12.4s, v4.s[2] \n" - "fmla v21.4s, v12.4s, v5.s[2] \n" - "fmla v22.4s, v12.4s, v6.s[2] \n" - "fmla v23.4s, v12.4s, v7.s[2] \n" - - "fmla v24.4s, v13.4s, v0.s[2] \n" - "fmla v25.4s, v13.4s, v1.s[2] \n" - "fmla v26.4s, v13.4s, v2.s[2] \n" - "fmla v27.4s, v13.4s, v3.s[2] \n" - "fmla v28.4s, v13.4s, v4.s[2] \n" - "fmla v29.4s, v13.4s, v5.s[2] \n" - "fmla v30.4s, v13.4s, v6.s[2] \n" - "fmla v31.4s, v13.4s, v7.s[2] \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v14.4s, v2.s[3] \n" - "fmla v19.4s, v14.4s, v3.s[3] \n" - "fmla v20.4s, v14.4s, v4.s[3] \n" - "fmla v21.4s, v14.4s, v5.s[3] \n" - "fmla v22.4s, v14.4s, v6.s[3] \n" - "fmla v23.4s, v14.4s, v7.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v24.4s, v15.4s, v0.s[3] \n" - "fmla v25.4s, v15.4s, v1.s[3] \n" - "fmla v26.4s, v15.4s, v2.s[3] \n" - "fmla v27.4s, v15.4s, v3.s[3] \n" - "fmla v28.4s, v15.4s, v4.s[3] \n" - "fmla v29.4s, v15.4s, v5.s[3] \n" - "fmla v30.4s, v15.4s, v6.s[3] \n" - "fmla v31.4s, v15.4s, v7.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "shrn v20.4h, v20.4s, #16 \n" - "shrn v21.4h, v21.4s, #16 \n" - "shrn v22.4h, v22.4s, #16 \n" - "shrn v23.4h, v23.4s, #16 \n" - - "shrn v24.4h, v24.4s, #16 \n" - "shrn v25.4h, v25.4s, #16 \n" - "shrn v26.4h, v26.4s, #16 \n" - "shrn v27.4h, v27.4s, #16 \n" - - "shrn v28.4h, v28.4s, #16 \n" - "shrn v29.4h, v29.4s, #16 \n" - "shrn v30.4h, v30.4s, #16 \n" - "shrn v31.4h, v31.4s, #16 \n" - - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - "st1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" - "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%1], #32 \n" - "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%2], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 3 < size; i += 4) - { - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - - const unsigned short* kptr01 = (const unsigned short*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v1.16b \n" - "mov v21.16b, v1.16b \n" - "mov v22.16b, v1.16b \n" - "mov v23.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "fmla v20.4s, v9.4s, v0.s[0] \n" - "fmla v21.4s, v9.4s, v1.s[0] \n" - "fmla v22.4s, v9.4s, v2.s[0] \n" - "fmla v23.4s, v9.4s, v3.s[0] \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 - - "shll v12.4s, v12.4h, #16 \n" - "shll v13.4s, v13.4h, #16 \n" - "shll v14.4s, v14.4h, #16 \n" - "shll v15.4s, v15.4h, #16 \n" - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v10.4s, v2.s[1] \n" - "fmla v19.4s, v10.4s, v3.s[1] \n" - - "fmla v20.4s, v11.4s, v0.s[1] \n" - "fmla v21.4s, v11.4s, v1.s[1] \n" - "fmla v22.4s, v11.4s, v2.s[1] \n" - "fmla v23.4s, v11.4s, v3.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v12.4s, v2.s[2] \n" - "fmla v19.4s, v12.4s, v3.s[2] \n" - - "fmla v20.4s, v13.4s, v0.s[2] \n" - "fmla v21.4s, v13.4s, v1.s[2] \n" - "fmla v22.4s, v13.4s, v2.s[2] \n" - "fmla v23.4s, v13.4s, v3.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v14.4s, v2.s[3] \n" - "fmla v19.4s, v14.4s, v3.s[3] \n" - - "fmla v20.4s, v15.4s, v0.s[3] \n" - "fmla v21.4s, v15.4s, v1.s[3] \n" - "fmla v22.4s, v15.4s, v2.s[3] \n" - "fmla v23.4s, v15.4s, v3.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "shrn v20.4h, v20.4s, #16 \n" - "shrn v21.4h, v21.4s, #16 \n" - "shrn v22.4h, v22.4s, #16 \n" - "shrn v23.4h, v23.4s, #16 \n" - - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - } - for (; i + 1 < size; i += 2) - { - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - - const unsigned short* kptr01 = (const unsigned short*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s, v1.4s}, [%10] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v1.16b \n" - "mov v19.16b, v1.16b \n" - - "0: \n" - - "prfm pldl1keep, [%3, #128] \n" - "ld1 {v0.4h, v1.4h}, [%3], #16 \n" // r0 r1 - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v9.4s, v0.s[0] \n" - "fmla v19.4s, v9.4s, v1.s[0] \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 - - "shll v12.4s, v12.4h, #16 \n" - "shll v13.4s, v13.4h, #16 \n" - "shll v14.4s, v14.4h, #16 \n" - "shll v15.4s, v15.4h, #16 \n" - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v10.4s, v1.s[1] \n" - "fmla v18.4s, v11.4s, v0.s[1] \n" - "fmla v19.4s, v11.4s, v1.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v12.4s, v1.s[2] \n" - "fmla v18.4s, v13.4s, v0.s[2] \n" - "fmla v19.4s, v13.4s, v1.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v14.4s, v1.s[3] \n" - "fmla v18.4s, v15.4s, v0.s[3] \n" - "fmla v19.4s, v15.4s, v1.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "st1 {v16.4h, v17.4h}, [%1], #16 \n" - "st1 {v18.4h, v19.4h}, [%2], #16 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); - } - for (; i < size; i++) - { - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - - const unsigned short* kptr01 = (const unsigned short*)kernel.channel(pp); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.4s, v17.4s}, [%10] \n" - - "0: \n" - - "prfm pldl1keep, [%3, #64] \n" - "ld1 {v0.4h}, [%3], #8 \n" // r0 - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 - - "shll v0.4s, v0.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v9.4s, v0.s[0] \n" - - "prfm pldl1keep, [%4, #256] \n" - "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 - - "shll v12.4s, v12.4h, #16 \n" - "shll v13.4s, v13.4h, #16 \n" - "shll v14.4s, v14.4h, #16 \n" - "shll v15.4s, v15.4h, #16 \n" - - "fmla v16.4s, v10.4s, v0.s[1] \n" - "fmla v17.4s, v11.4s, v0.s[1] \n" - - "fmla v16.4s, v12.4s, v0.s[2] \n" - "fmla v17.4s, v13.4s, v0.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v14.4s, v0.s[3] \n" - "fmla v17.4s, v15.4s, v0.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - - "st1 {v16.4h}, [%1], #8 \n" - "st1 {v17.4h}, [%2], #8 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(outptr1), // %2 - "=r"(tmpptr), // %3 - "=r"(kptr01) // %4 - : "0"(nn), - "1"(outptr0), - "2"(outptr1), - "3"(tmpptr), - "4"(kptr01), - "r"(biasptr) // %10 - : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); - } - } - -#endif // __ARM_NEON && __aarch64__ - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - unsigned short* outptr0 = top_blob.channel(p); - - const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + p * 4 : zeros; - - int i = 0; -#if __aarch64__ - for (; i + 11 < size; i += 12) - { - unsigned short* tmpptr = tmp.channel(i / 12); - - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p / 2 + p % 2); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v8.16b, v0.16b \n" - "mov v9.16b, v0.16b \n" - "mov v10.16b, v0.16b \n" - "mov v11.16b, v0.16b \n" - "mov v12.16b, v0.16b \n" - "mov v13.16b, v0.16b \n" - "mov v14.16b, v0.16b \n" - "mov v15.16b, v0.16b \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%3], #32 \n" // w0123_0 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v4.4s, v4.4h, #16 \n" - "shll v5.4s, v5.4h, #16 \n" - "shll v6.4s, v6.4h, #16 \n" - "shll v7.4s, v7.4h, #16 \n" - - "fmla v8.4s, v4.4s, v0.s[0] \n" - "fmla v9.4s, v4.4s, v0.s[1] \n" - "fmla v10.4s, v4.4s, v0.s[2] \n" - "fmla v11.4s, v4.4s, v0.s[3] \n" - "fmla v12.4s, v4.4s, v1.s[0] \n" - "fmla v13.4s, v4.4s, v1.s[1] \n" - "fmla v14.4s, v4.4s, v1.s[2] \n" - "fmla v15.4s, v4.4s, v1.s[3] \n" - "fmla v16.4s, v4.4s, v2.s[0] \n" - "fmla v17.4s, v4.4s, v2.s[1] \n" - "fmla v18.4s, v4.4s, v2.s[2] \n" - "fmla v19.4s, v4.4s, v2.s[3] \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" - - "shll v20.4s, v20.4h, #16 \n" - "shll v21.4s, v21.4h, #16 \n" - "shll v22.4s, v22.4h, #16 \n" - "shll v23.4s, v23.4h, #16 \n" - - "fmla v8.4s, v5.4s, v3.s[0] \n" - "fmla v9.4s, v5.4s, v3.s[1] \n" - "fmla v10.4s, v5.4s, v3.s[2] \n" - "fmla v11.4s, v5.4s, v3.s[3] \n" - "fmla v12.4s, v5.4s, v20.s[0] \n" - "fmla v13.4s, v5.4s, v20.s[1] \n" - "fmla v14.4s, v5.4s, v20.s[2] \n" - "fmla v15.4s, v5.4s, v20.s[3] \n" - "fmla v16.4s, v5.4s, v21.s[0] \n" - "fmla v17.4s, v5.4s, v21.s[1] \n" - "fmla v18.4s, v5.4s, v21.s[2] \n" - "fmla v19.4s, v5.4s, v21.s[3] \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" - - "shll v24.4s, v24.4h, #16 \n" - "shll v25.4s, v25.4h, #16 \n" - "shll v26.4s, v26.4h, #16 \n" - "shll v27.4s, v27.4h, #16 \n" - - "fmla v8.4s, v6.4s, v22.s[0] \n" - "fmla v9.4s, v6.4s, v22.s[1] \n" - "fmla v10.4s, v6.4s, v22.s[2] \n" - "fmla v11.4s, v6.4s, v22.s[3] \n" - "fmla v12.4s, v6.4s, v23.s[0] \n" - "fmla v13.4s, v6.4s, v23.s[1] \n" - "fmla v14.4s, v6.4s, v23.s[2] \n" - "fmla v15.4s, v6.4s, v23.s[3] \n" - "fmla v16.4s, v6.4s, v24.s[0] \n" - "fmla v17.4s, v6.4s, v24.s[1] \n" - "fmla v18.4s, v6.4s, v24.s[2] \n" - "fmla v19.4s, v6.4s, v24.s[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v8.4s, v7.4s, v25.s[0] \n" - "fmla v9.4s, v7.4s, v25.s[1] \n" - "fmla v10.4s, v7.4s, v25.s[2] \n" - "fmla v11.4s, v7.4s, v25.s[3] \n" - "fmla v12.4s, v7.4s, v26.s[0] \n" - "fmla v13.4s, v7.4s, v26.s[1] \n" - "fmla v14.4s, v7.4s, v26.s[2] \n" - "fmla v15.4s, v7.4s, v26.s[3] \n" - "fmla v16.4s, v7.4s, v27.s[0] \n" - "fmla v17.4s, v7.4s, v27.s[1] \n" - "fmla v18.4s, v7.4s, v27.s[2] \n" - "fmla v19.4s, v7.4s, v27.s[3] \n" - - "bne 0b \n" - - "shrn v8.4h, v8.4s, #16 \n" - "shrn v9.4h, v9.4s, #16 \n" - "shrn v10.4h, v10.4s, #16 \n" - "shrn v11.4h, v11.4s, #16 \n" - - "shrn v12.4h, v12.4s, #16 \n" - "shrn v13.4h, v13.4s, #16 \n" - "shrn v14.4h, v14.4s, #16 \n" - "shrn v15.4h, v15.4s, #16 \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "st1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%1], #32 \n" - "st1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%1], #32 \n" - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); - } -#endif - for (; i + 7 < size; i += 8) - { -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p / 2 + p % 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - "mov v20.16b, v0.16b \n" - "mov v21.16b, v0.16b \n" - "mov v22.16b, v0.16b \n" - "mov v23.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%2], #32 \n" // r4 r5 r6 r7 - - "shll v4.4s, v4.4h, #16 \n" - "shll v5.4s, v5.4h, #16 \n" - "shll v6.4s, v6.4h, #16 \n" - "shll v7.4s, v7.4h, #16 \n" - - "fmla v20.4s, v8.4s, v4.s[0] \n" - "fmla v21.4s, v8.4s, v5.s[0] \n" - "fmla v22.4s, v8.4s, v6.s[0] \n" - "fmla v23.4s, v8.4s, v7.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - "fmla v18.4s, v9.4s, v2.s[1] \n" - "fmla v19.4s, v9.4s, v3.s[1] \n" - "fmla v20.4s, v9.4s, v4.s[1] \n" - "fmla v21.4s, v9.4s, v5.s[1] \n" - "fmla v22.4s, v9.4s, v6.s[1] \n" - "fmla v23.4s, v9.4s, v7.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - "fmla v18.4s, v10.4s, v2.s[2] \n" - "fmla v19.4s, v10.4s, v3.s[2] \n" - "fmla v20.4s, v10.4s, v4.s[2] \n" - "fmla v21.4s, v10.4s, v5.s[2] \n" - "fmla v22.4s, v10.4s, v6.s[2] \n" - "fmla v23.4s, v10.4s, v7.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - "fmla v18.4s, v11.4s, v2.s[3] \n" - "fmla v19.4s, v11.4s, v3.s[3] \n" - "fmla v20.4s, v11.4s, v4.s[3] \n" - "fmla v21.4s, v11.4s, v5.s[3] \n" - "fmla v22.4s, v11.4s, v6.s[3] \n" - "fmla v23.4s, v11.4s, v7.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "shrn v20.4h, v20.4s, #16 \n" - "shrn v21.4h, v21.4s, #16 \n" - "shrn v22.4h, v22.4s, #16 \n" - "shrn v23.4h, v23.4s, #16 \n" - - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%1], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - "vmov q10, q0 \n" - "vmov q11, q0 \n" - "vmov q12, q0 \n" - "vmov q13, q0 \n" - "vmov q14, q0 \n" - "vmov q15, q0 \n" - - "0: \n" - - "pld [%2, #256] \n" - "vld1.u16 {d4-d7}, [%2]! \n" - - "pld [%3, #256] \n" - "vld1.u16 {d12-d15}, [%3]! \n" - - "vshll.u16 q0, d4, #16 \n" - "vshll.u16 q1, d5, #16 \n" - "vshll.u16 q2, d6, #16 \n" - "vshll.u16 q3, d7, #16 \n" - - "vshll.u16 q4, d12, #16 \n" - "vshll.u16 q5, d13, #16 \n" - "vshll.u16 q6, d14, #16 \n" - "vshll.u16 q7, d15, #16 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d0[1] \n" - "vmla.f32 q10, q4, d1[0] \n" - "vmla.f32 q11, q4, d1[1] \n" - "vmla.f32 q12, q4, d2[0] \n" - "vmla.f32 q13, q4, d2[1] \n" - "vmla.f32 q14, q4, d3[0] \n" - "vmla.f32 q15, q4, d3[1] \n" - - "vmla.f32 q8, q5, d4[0] \n" - "vmla.f32 q9, q5, d4[1] \n" - "vmla.f32 q10, q5, d5[0] \n" - "vmla.f32 q11, q5, d5[1] \n" - "vmla.f32 q12, q5, d6[0] \n" - "vmla.f32 q13, q5, d6[1] \n" - "vmla.f32 q14, q5, d7[0] \n" - "vmla.f32 q15, q5, d7[1] \n" - - "pld [%2, #256] \n" - "vld1.u16 {d4-d7}, [%2]! \n" - - "vshll.u16 q0, d4, #16 \n" - "vshll.u16 q1, d5, #16 \n" - "vshll.u16 q2, d6, #16 \n" - "vshll.u16 q3, d7, #16 \n" - - "vmla.f32 q8, q6, d0[0] \n" - "vmla.f32 q9, q6, d0[1] \n" - "vmla.f32 q10, q6, d1[0] \n" - "vmla.f32 q11, q6, d1[1] \n" - "vmla.f32 q12, q6, d2[0] \n" - "vmla.f32 q13, q6, d2[1] \n" - "vmla.f32 q14, q6, d3[0] \n" - "vmla.f32 q15, q6, d3[1] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d4[0] \n" - "vmla.f32 q9, q7, d4[1] \n" - "vmla.f32 q10, q7, d5[0] \n" - "vmla.f32 q11, q7, d5[1] \n" - "vmla.f32 q12, q7, d6[0] \n" - "vmla.f32 q13, q7, d6[1] \n" - "vmla.f32 q14, q7, d7[0] \n" - "vmla.f32 q15, q7, d7[1] \n" - - "bne 0b \n" - - "vshrn.u32 d16, q8, #16 \n" - "vshrn.u32 d17, q9, #16 \n" - "vshrn.u32 d18, q10, #16 \n" - "vshrn.u32 d19, q11, #16 \n" - - "vshrn.u32 d20, q12, #16 \n" - "vshrn.u32 d21, q13, #16 \n" - "vshrn.u32 d22, q14, #16 \n" - "vshrn.u32 d23, q15, #16 \n" - - "vstm %1!, {d16-d23} \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif - } - for (; i + 3 < size; i += 4) - { -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p / 2 + p % 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - "mov v18.16b, v0.16b \n" - "mov v19.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r0 r1 r2 r3 - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - "shll v2.4s, v2.4h, #16 \n" - "shll v3.4s, v3.4h, #16 \n" - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - "fmla v18.4s, v8.4s, v2.s[0] \n" - "fmla v19.4s, v8.4s, v3.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - "fmla v18.4s, v9.4s, v2.s[1] \n" - "fmla v19.4s, v9.4s, v3.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - "fmla v18.4s, v10.4s, v2.s[2] \n" - "fmla v19.4s, v10.4s, v3.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - "fmla v18.4s, v11.4s, v2.s[3] \n" - "fmla v19.4s, v11.4s, v3.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - "shrn v18.4h, v18.4s, #16 \n" - "shrn v19.4h, v19.4s, #16 \n" - - "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - "vmov q10, q0 \n" - "vmov q11, q0 \n" - - "0: \n" - - "pld [%2, #256] \n" - "vld1.u16 {d4-d7}, [%2]! \n" - - "pld [%3, #256] \n" - "vld1.u16 {d12-d15}, [%3]! \n" - - "vshll.u16 q0, d4, #16 \n" - "vshll.u16 q1, d5, #16 \n" - "vshll.u16 q2, d6, #16 \n" - "vshll.u16 q3, d7, #16 \n" - - "vshll.u16 q4, d12, #16 \n" - "vshll.u16 q5, d13, #16 \n" - "vshll.u16 q6, d14, #16 \n" - "vshll.u16 q7, d15, #16 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d2[0] \n" - "vmla.f32 q10, q4, d4[0] \n" - "vmla.f32 q11, q4, d6[0] \n" - - "vmla.f32 q8, q5, d0[1] \n" - "vmla.f32 q9, q5, d2[1] \n" - "vmla.f32 q10, q5, d4[1] \n" - "vmla.f32 q11, q5, d6[1] \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q9, q6, d3[0] \n" - "vmla.f32 q10, q6, d5[0] \n" - "vmla.f32 q11, q6, d7[0] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d1[1] \n" - "vmla.f32 q9, q7, d3[1] \n" - "vmla.f32 q10, q7, d5[1] \n" - "vmla.f32 q11, q7, d7[1] \n" - - "bne 0b \n" - - "vshrn.u32 d16, q8, #16 \n" - "vshrn.u32 d17, q9, #16 \n" - "vshrn.u32 d18, q10, #16 \n" - "vshrn.u32 d19, q11, #16 \n" - - "vst1.u16 {d16-d19}, [%1]! \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); -#endif - } - for (; i + 1 < size; i += 2) - { -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p / 2 + p % 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%8] \n" - "mov v16.16b, v0.16b \n" - "mov v17.16b, v0.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #128] \n" - "ld1 {v0.4h, v1.4h}, [%2], #16 \n" // r0 r1 - - "shll v0.4s, v0.4h, #16 \n" - "shll v1.4s, v1.4h, #16 \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v17.4s, v8.4s, v1.s[0] \n" - - "fmla v16.4s, v9.4s, v0.s[1] \n" - "fmla v17.4s, v9.4s, v1.s[1] \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v17.4s, v10.4s, v1.s[2] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v11.4s, v0.s[3] \n" - "fmla v17.4s, v11.4s, v1.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - "shrn v17.4h, v17.4s, #16 \n" - - "st1 {v16.4h, v17.4h}, [%1], #16 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v16", "v17"); -#else - asm volatile( - "vld1.f32 {d0-d1}, [%8] \n" - "vmov q8, q0 \n" - "vmov q9, q0 \n" - - "0: \n" - - "pld [%2, #128] \n" - "vld1.u16 {d4-d5}, [%2 :128]! \n" - - "vshll.u16 q0, d4, #16 \n" - "vshll.u16 q1, d5, #16 \n" - - "pld [%3, #256] \n" - "vld1.u16 {d12-d15}, [%3]! \n" - - "vshll.u16 q4, d12, #16 \n" - "vshll.u16 q5, d13, #16 \n" - "vshll.u16 q6, d14, #16 \n" - "vshll.u16 q7, d15, #16 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q4, d2[0] \n" - - "vmla.f32 q8, q5, d0[1] \n" - "vmla.f32 q9, q5, d2[1] \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q9, q6, d3[0] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q7, d1[1] \n" - "vmla.f32 q9, q7, d3[1] \n" - - "bne 0b \n" - - "vshrn.u32 d16, q8, #16 \n" - "vshrn.u32 d17, q9, #16 \n" - - "vst1.u16 {d16-d17}, [%1 :128]! \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q1", "q4", "q5", "q6", "q7", "q8", "q9"); -#endif - } - for (; i < size; i++) - { -#if __aarch64__ - unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p / 2 + p % 2); -#else - unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); - const unsigned short* kptr0 = (const unsigned short*)kernel.channel(p); -#endif - - int nn = inch; // inch always > 0 - -#if __aarch64__ - asm volatile( - "ld1 {v16.4s}, [%8] \n" - - "0: \n" - - "prfm pldl1keep, [%2, #64] \n" - "ld1 {v0.4h}, [%2], #8 \n" // r0 - - "shll v0.4s, v0.4h, #16 \n" - - "prfm pldl1keep, [%3, #256] \n" - "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 - - "shll v8.4s, v8.4h, #16 \n" - "shll v9.4s, v9.4h, #16 \n" - "shll v10.4s, v10.4h, #16 \n" - "shll v11.4s, v11.4h, #16 \n" - - "fmla v16.4s, v8.4s, v0.s[0] \n" - "fmla v16.4s, v9.4s, v0.s[1] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.4s, v10.4s, v0.s[2] \n" - "fmla v16.4s, v11.4s, v0.s[3] \n" - - "bne 0b \n" - - "shrn v16.4h, v16.4s, #16 \n" - - "st1 {v16.4h}, [%1], #8 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v16"); -#else - asm volatile( - "vld1.f32 {d16-d17}, [%8] \n" - - "0: \n" - - "pld [%2, #64] \n" - "vld1.u16 {d1}, [%2 :64]! \n" - - "vshll.u16 q0, d1, #16 \n" - - "pld [%3, #256] \n" - "vld1.u16 {d12-d15}, [%3]! \n" - - "vshll.u16 q4, d12, #16 \n" - "vshll.u16 q5, d13, #16 \n" - "vshll.u16 q6, d14, #16 \n" - "vshll.u16 q7, d15, #16 \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q8, q5, d0[1] \n" - - "subs %0, %0, #1 \n" - - "vmla.f32 q8, q6, d1[0] \n" - "vmla.f32 q8, q7, d1[1] \n" - - "bne 0b \n" - - "vshrn.u32 d16, q8, #16 \n" - - "vst1.u16 {d16}, [%1 :64]! \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "q0", "q4", "q5", "q6", "q7", "q8"); -#endif - } - } + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; - // // NOTE sgemm - // for (; p= 12) - tmp.create(12, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 8) - tmp.create(8, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 4) - tmp.create(4, inch, size / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else if (size >= 2) - tmp.create(2, inch, size / 2 + size % 2, elemsize, elempack, opt.workspace_allocator); - else // if (size >= 1) - tmp.create(1, inch, size, elemsize, elempack, opt.workspace_allocator); - { - int nn_size; - int remain_size_start; - - nn_size = size / 12; - remain_size_start = nn_size * 12; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = ii * 12; - - const __fp16* img0 = bottom_blob.channel(0); - img0 += i * 8; - - __fp16* tmpptr = tmp.channel(i / 12); - - for (int q = 0; q < inch; q++) - { - // transpose 12x8 - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n" - "ld4 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n" - - "sub %0, %0, #128 \n" - - "uzp1 v20.8h, v0.8h, v4.8h \n" // 0 - "uzp1 v21.8h, v16.8h, v1.8h \n" // 1 - "uzp1 v22.8h, v5.8h, v17.8h \n" // 2 - "uzp1 v23.8h, v2.8h, v6.8h \n" // 3 - "uzp1 v24.8h, v18.8h, v3.8h \n" // 4 - "uzp1 v25.8h, v7.8h, v19.8h \n" // 5 - "uzp2 v26.8h, v0.8h, v4.8h \n" // 6 - "uzp2 v27.8h, v16.8h, v1.8h \n" // 7 - "uzp2 v28.8h, v5.8h, v17.8h \n" // 8 - "uzp2 v29.8h, v2.8h, v6.8h \n" // 9 - "uzp2 v30.8h, v18.8h, v3.8h \n" // 10 - "uzp2 v31.8h, v7.8h, v19.8h \n" // 11 - - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" - "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - img0 += bottom_blob.cstep * 8; - } - } - - nn_size = (size - remain_size_start) >> 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 8; - - const __fp16* img0 = bottom_blob.channel(0); - img0 += i * 8; - - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - - for (int q = 0; q < inch; q++) - { - // transpose 8x8 - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" - "sub %0, %0, #64 \n" - - "uzp1 v16.8h, v0.8h, v4.8h \n" - "uzp2 v20.8h, v0.8h, v4.8h \n" - "uzp1 v17.8h, v1.8h, v5.8h \n" - "uzp2 v21.8h, v1.8h, v5.8h \n" - "uzp1 v18.8h, v2.8h, v6.8h \n" - "uzp2 v22.8h, v2.8h, v6.8h \n" - "uzp1 v19.8h, v3.8h, v7.8h \n" - "uzp2 v23.8h, v3.8h, v7.8h \n" - - "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - - img0 += bottom_blob.cstep * 8; - } - } - - remain_size_start += nn_size << 3; - nn_size = (size - remain_size_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 4; - - const __fp16* img0 = bottom_blob.channel(0); - img0 += i * 8; - - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - - for (int q = 0; q < inch; q++) - { - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" - "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3"); - - img0 += bottom_blob.cstep * 8; - } - } - - remain_size_start += nn_size << 2; - nn_size = (size - remain_size_start) >> 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = remain_size_start + ii * 2; - - const __fp16* img0 = bottom_blob.channel(0); - img0 += i * 8; - - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - - for (int q = 0; q < inch; q++) - { - asm volatile( - "prfm pldl1keep, [%0, #256] \n" - "ld1 {v0.8h, v1.8h}, [%0] \n" - "st1 {v0.8h, v1.8h}, [%1], #32 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0", "v1"); - - img0 += bottom_blob.cstep * 8; - } - } - - remain_size_start += nn_size << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_size_start; i < size; i++) - { - const __fp16* img0 = bottom_blob.channel(0); - img0 += i * 8; - - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - - for (int q = 0; q < inch; q++) - { - asm volatile( - "prfm pldl1keep, [%0, #128] \n" - "ld1 {v0.8h}, [%0] \n" - "st1 {v0.8h}, [%1], #16 \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "v0"); - - img0 += bottom_blob.cstep * 8; - } - } - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - __fp16* outptr0 = top_blob.channel(p); - - const __fp16 zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; - const __fp16* biasptr = bias ? bias + p * 8 : zeros; - - int i = 0; - for (; i + 11 < size; i += 12) - { - __fp16* tmpptr = tmp.channel(i / 12); - const __fp16* kptr0 = kernel.channel(p); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v20.8h}, [%8] \n" - "mov v21.16b, v20.16b \n" - "mov v22.16b, v20.16b \n" - "mov v23.16b, v20.16b \n" - "mov v24.16b, v20.16b \n" - "mov v25.16b, v20.16b \n" - "mov v26.16b, v20.16b \n" - "mov v27.16b, v20.16b \n" - "mov v28.16b, v20.16b \n" - "mov v29.16b, v20.16b \n" - "mov v30.16b, v20.16b \n" - "mov v31.16b, v20.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w0123 - - "fmla v20.8h, v12.8h, v0.h[0] \n" - "fmla v21.8h, v12.8h, v0.h[1] \n" - "fmla v22.8h, v12.8h, v0.h[2] \n" - "fmla v23.8h, v12.8h, v0.h[3] \n" - "fmla v24.8h, v12.8h, v0.h[4] \n" - "fmla v25.8h, v12.8h, v0.h[5] \n" - "fmla v26.8h, v12.8h, v0.h[6] \n" - "fmla v27.8h, v12.8h, v0.h[7] \n" - "fmla v28.8h, v12.8h, v1.h[0] \n" - "fmla v29.8h, v12.8h, v1.h[1] \n" - "fmla v30.8h, v12.8h, v1.h[2] \n" - "fmla v31.8h, v12.8h, v1.h[3] \n" - - "fmla v20.8h, v13.8h, v1.h[4] \n" - "fmla v21.8h, v13.8h, v1.h[5] \n" - "fmla v22.8h, v13.8h, v1.h[6] \n" - "fmla v23.8h, v13.8h, v1.h[7] \n" - "fmla v24.8h, v13.8h, v2.h[0] \n" - "fmla v25.8h, v13.8h, v2.h[1] \n" - "fmla v26.8h, v13.8h, v2.h[2] \n" - "fmla v27.8h, v13.8h, v2.h[3] \n" - "fmla v28.8h, v13.8h, v2.h[4] \n" - "fmla v29.8h, v13.8h, v2.h[5] \n" - "fmla v30.8h, v13.8h, v2.h[6] \n" - "fmla v31.8h, v13.8h, v2.h[7] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567 - - "fmla v20.8h, v14.8h, v3.h[0] \n" - "fmla v21.8h, v14.8h, v3.h[1] \n" - "fmla v22.8h, v14.8h, v3.h[2] \n" - "fmla v23.8h, v14.8h, v3.h[3] \n" - "fmla v24.8h, v14.8h, v3.h[4] \n" - "fmla v25.8h, v14.8h, v3.h[5] \n" - "fmla v26.8h, v14.8h, v3.h[6] \n" - "fmla v27.8h, v14.8h, v3.h[7] \n" - "fmla v28.8h, v14.8h, v4.h[0] \n" - "fmla v29.8h, v14.8h, v4.h[1] \n" - "fmla v30.8h, v14.8h, v4.h[2] \n" - "fmla v31.8h, v14.8h, v4.h[3] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%3], #64 \n" // w4567 - - "fmla v20.8h, v15.8h, v4.h[4] \n" - "fmla v21.8h, v15.8h, v4.h[5] \n" - "fmla v22.8h, v15.8h, v4.h[6] \n" - "fmla v23.8h, v15.8h, v4.h[7] \n" - "fmla v24.8h, v15.8h, v5.h[0] \n" - "fmla v25.8h, v15.8h, v5.h[1] \n" - "fmla v26.8h, v15.8h, v5.h[2] \n" - "fmla v27.8h, v15.8h, v5.h[3] \n" - "fmla v28.8h, v15.8h, v5.h[4] \n" - "fmla v29.8h, v15.8h, v5.h[5] \n" - "fmla v30.8h, v15.8h, v5.h[6] \n" - "fmla v31.8h, v15.8h, v5.h[7] \n" - - "fmla v20.8h, v16.8h, v6.h[0] \n" - "fmla v21.8h, v16.8h, v6.h[1] \n" - "fmla v22.8h, v16.8h, v6.h[2] \n" - "fmla v23.8h, v16.8h, v6.h[3] \n" - "fmla v24.8h, v16.8h, v6.h[4] \n" - "fmla v25.8h, v16.8h, v6.h[5] \n" - "fmla v26.8h, v16.8h, v6.h[6] \n" - "fmla v27.8h, v16.8h, v6.h[7] \n" - "fmla v28.8h, v16.8h, v7.h[0] \n" - "fmla v29.8h, v16.8h, v7.h[1] \n" - "fmla v30.8h, v16.8h, v7.h[2] \n" - "fmla v31.8h, v16.8h, v7.h[3] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%2], #64 \n" // r891011 - - "fmla v20.8h, v17.8h, v7.h[4] \n" - "fmla v21.8h, v17.8h, v7.h[5] \n" - "fmla v22.8h, v17.8h, v7.h[6] \n" - "fmla v23.8h, v17.8h, v7.h[7] \n" - "fmla v24.8h, v17.8h, v8.h[0] \n" - "fmla v25.8h, v17.8h, v8.h[1] \n" - "fmla v26.8h, v17.8h, v8.h[2] \n" - "fmla v27.8h, v17.8h, v8.h[3] \n" - "fmla v28.8h, v17.8h, v8.h[4] \n" - "fmla v29.8h, v17.8h, v8.h[5] \n" - "fmla v30.8h, v17.8h, v8.h[6] \n" - "fmla v31.8h, v17.8h, v8.h[7] \n" - - "fmla v20.8h, v18.8h, v9.h[0] \n" - "fmla v21.8h, v18.8h, v9.h[1] \n" - "fmla v22.8h, v18.8h, v9.h[2] \n" - "fmla v23.8h, v18.8h, v9.h[3] \n" - "fmla v24.8h, v18.8h, v9.h[4] \n" - "fmla v25.8h, v18.8h, v9.h[5] \n" - "fmla v26.8h, v18.8h, v9.h[6] \n" - "fmla v27.8h, v18.8h, v9.h[7] \n" - "fmla v28.8h, v18.8h, v10.h[0] \n" - "fmla v29.8h, v18.8h, v10.h[1] \n" - "fmla v30.8h, v18.8h, v10.h[2] \n" - "fmla v31.8h, v18.8h, v10.h[3] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v20.8h, v19.8h, v10.h[4] \n" - "fmla v21.8h, v19.8h, v10.h[5] \n" - "fmla v22.8h, v19.8h, v10.h[6] \n" - "fmla v23.8h, v19.8h, v10.h[7] \n" - "fmla v24.8h, v19.8h, v11.h[0] \n" - "fmla v25.8h, v19.8h, v11.h[1] \n" - "fmla v26.8h, v19.8h, v11.h[2] \n" - "fmla v27.8h, v19.8h, v11.h[3] \n" - "fmla v28.8h, v19.8h, v11.h[4] \n" - "fmla v29.8h, v19.8h, v11.h[5] \n" - "fmla v30.8h, v19.8h, v11.h[6] \n" - "fmla v31.8h, v19.8h, v11.h[7] \n" - - "bne 0b \n" - - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" - "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 7 < size; i += 8) - { - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); - const __fp16* kptr0 = kernel.channel(p); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.8h}, [%8] \n" - "mov v17.16b, v16.16b \n" - "mov v18.16b, v16.16b \n" - "mov v19.16b, v16.16b \n" - "mov v20.16b, v16.16b \n" - "mov v21.16b, v16.16b \n" - "mov v22.16b, v16.16b \n" - "mov v23.16b, v16.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 - - "fmla v16.8h, v8.8h, v0.h[0] \n" - "fmla v17.8h, v8.8h, v0.h[1] \n" - "fmla v18.8h, v8.8h, v0.h[2] \n" - "fmla v19.8h, v8.8h, v0.h[3] \n" - "fmla v20.8h, v8.8h, v0.h[4] \n" - "fmla v21.8h, v8.8h, v0.h[5] \n" - "fmla v22.8h, v8.8h, v0.h[6] \n" - "fmla v23.8h, v8.8h, v0.h[7] \n" - - "fmla v16.8h, v9.8h, v1.h[0] \n" - "fmla v17.8h, v9.8h, v1.h[1] \n" - "fmla v18.8h, v9.8h, v1.h[2] \n" - "fmla v19.8h, v9.8h, v1.h[3] \n" - "fmla v20.8h, v9.8h, v1.h[4] \n" - "fmla v21.8h, v9.8h, v1.h[5] \n" - "fmla v22.8h, v9.8h, v1.h[6] \n" - "fmla v23.8h, v9.8h, v1.h[7] \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567 - - "fmla v16.8h, v10.8h, v2.h[0] \n" - "fmla v17.8h, v10.8h, v2.h[1] \n" - "fmla v18.8h, v10.8h, v2.h[2] \n" - "fmla v19.8h, v10.8h, v2.h[3] \n" - "fmla v20.8h, v10.8h, v2.h[4] \n" - "fmla v21.8h, v10.8h, v2.h[5] \n" - "fmla v22.8h, v10.8h, v2.h[6] \n" - "fmla v23.8h, v10.8h, v2.h[7] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 - - "fmla v16.8h, v11.8h, v3.h[0] \n" - "fmla v17.8h, v11.8h, v3.h[1] \n" - "fmla v18.8h, v11.8h, v3.h[2] \n" - "fmla v19.8h, v11.8h, v3.h[3] \n" - "fmla v20.8h, v11.8h, v3.h[4] \n" - "fmla v21.8h, v11.8h, v3.h[5] \n" - "fmla v22.8h, v11.8h, v3.h[6] \n" - "fmla v23.8h, v11.8h, v3.h[7] \n" - - "fmla v16.8h, v12.8h, v4.h[0] \n" - "fmla v17.8h, v12.8h, v4.h[1] \n" - "fmla v18.8h, v12.8h, v4.h[2] \n" - "fmla v19.8h, v12.8h, v4.h[3] \n" - "fmla v20.8h, v12.8h, v4.h[4] \n" - "fmla v21.8h, v12.8h, v4.h[5] \n" - "fmla v22.8h, v12.8h, v4.h[6] \n" - "fmla v23.8h, v12.8h, v4.h[7] \n" - - "fmla v16.8h, v13.8h, v5.h[0] \n" - "fmla v17.8h, v13.8h, v5.h[1] \n" - "fmla v18.8h, v13.8h, v5.h[2] \n" - "fmla v19.8h, v13.8h, v5.h[3] \n" - "fmla v20.8h, v13.8h, v5.h[4] \n" - "fmla v21.8h, v13.8h, v5.h[5] \n" - "fmla v22.8h, v13.8h, v5.h[6] \n" - "fmla v23.8h, v13.8h, v5.h[7] \n" - - "fmla v16.8h, v14.8h, v6.h[0] \n" - "fmla v17.8h, v14.8h, v6.h[1] \n" - "fmla v18.8h, v14.8h, v6.h[2] \n" - "fmla v19.8h, v14.8h, v6.h[3] \n" - "fmla v20.8h, v14.8h, v6.h[4] \n" - "fmla v21.8h, v14.8h, v6.h[5] \n" - "fmla v22.8h, v14.8h, v6.h[6] \n" - "fmla v23.8h, v14.8h, v6.h[7] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.8h, v15.8h, v7.h[0] \n" - "fmla v17.8h, v15.8h, v7.h[1] \n" - "fmla v18.8h, v15.8h, v7.h[2] \n" - "fmla v19.8h, v15.8h, v7.h[3] \n" - "fmla v20.8h, v15.8h, v7.h[4] \n" - "fmla v21.8h, v15.8h, v7.h[5] \n" - "fmla v22.8h, v15.8h, v7.h[6] \n" - "fmla v23.8h, v15.8h, v7.h[7] \n" - - "bne 0b \n" - - "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - } - for (; i + 3 < size; i += 4) - { - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - const __fp16* kptr0 = kernel.channel(p); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.8h}, [%8] \n" - "mov v17.16b, v16.16b \n" - "mov v18.16b, v16.16b \n" - "mov v19.16b, v16.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 - - "fmla v16.8h, v8.8h, v0.h[0] \n" - "fmla v17.8h, v8.8h, v1.h[0] \n" - "fmla v18.8h, v8.8h, v2.h[0] \n" - "fmla v19.8h, v8.8h, v3.h[0] \n" - - "fmla v16.8h, v9.8h, v0.h[1] \n" - "fmla v17.8h, v9.8h, v1.h[1] \n" - "fmla v18.8h, v9.8h, v2.h[1] \n" - "fmla v19.8h, v9.8h, v3.h[1] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 - - "fmla v16.8h, v10.8h, v0.h[2] \n" - "fmla v17.8h, v10.8h, v1.h[2] \n" - "fmla v18.8h, v10.8h, v2.h[2] \n" - "fmla v19.8h, v10.8h, v3.h[2] \n" - - "fmla v16.8h, v11.8h, v0.h[3] \n" - "fmla v17.8h, v11.8h, v1.h[3] \n" - "fmla v18.8h, v11.8h, v2.h[3] \n" - "fmla v19.8h, v11.8h, v3.h[3] \n" - - "fmla v16.8h, v12.8h, v0.h[4] \n" - "fmla v17.8h, v12.8h, v1.h[4] \n" - "fmla v18.8h, v12.8h, v2.h[4] \n" - "fmla v19.8h, v12.8h, v3.h[4] \n" - - "fmla v16.8h, v13.8h, v0.h[5] \n" - "fmla v17.8h, v13.8h, v1.h[5] \n" - "fmla v18.8h, v13.8h, v2.h[5] \n" - "fmla v19.8h, v13.8h, v3.h[5] \n" - - "fmla v16.8h, v14.8h, v0.h[6] \n" - "fmla v17.8h, v14.8h, v1.h[6] \n" - "fmla v18.8h, v14.8h, v2.h[6] \n" - "fmla v19.8h, v14.8h, v3.h[6] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.8h, v15.8h, v0.h[7] \n" - "fmla v17.8h, v15.8h, v1.h[7] \n" - "fmla v18.8h, v15.8h, v2.h[7] \n" - "fmla v19.8h, v15.8h, v3.h[7] \n" - - "bne 0b \n" - - "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); - } - for (; i + 1 < size; i += 2) - { - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - const __fp16* kptr0 = kernel.channel(p); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.8h}, [%8] \n" - "mov v17.16b, v16.16b \n" - - "0: \n" - - "prfm pldl1keep, [%2, #256] \n" - "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 - - "fmla v16.8h, v8.8h, v0.h[0] \n" - "fmla v17.8h, v8.8h, v1.h[0] \n" - - "fmla v16.8h, v9.8h, v0.h[1] \n" - "fmla v17.8h, v9.8h, v1.h[1] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 - - "fmla v16.8h, v10.8h, v0.h[2] \n" - "fmla v17.8h, v10.8h, v1.h[2] \n" - - "fmla v16.8h, v11.8h, v0.h[3] \n" - "fmla v17.8h, v11.8h, v1.h[3] \n" - - "fmla v16.8h, v12.8h, v0.h[4] \n" - "fmla v17.8h, v12.8h, v1.h[4] \n" - - "fmla v16.8h, v13.8h, v0.h[5] \n" - "fmla v17.8h, v13.8h, v1.h[5] \n" - - "fmla v16.8h, v14.8h, v0.h[6] \n" - "fmla v17.8h, v14.8h, v1.h[6] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.8h, v15.8h, v0.h[7] \n" - "fmla v17.8h, v15.8h, v1.h[7] \n" - - "bne 0b \n" - - "st1 {v16.8h, v17.8h}, [%1], #32 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); - } - for (; i < size; i++) - { - __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - const __fp16* kptr0 = kernel.channel(p); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v16.8h}, [%8] \n" - - "0: \n" - - "prfm pldl1keep, [%2, #128] \n" - "ld1 {v0.8h}, [%2], #16 \n" // r0 - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 - - "fmla v16.8h, v8.8h, v0.h[0] \n" - "fmla v16.8h, v9.8h, v0.h[1] \n" - - "prfm pldl1keep, [%3, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 - - "fmla v16.8h, v10.8h, v0.h[2] \n" - "fmla v16.8h, v11.8h, v0.h[3] \n" - - "fmla v16.8h, v12.8h, v0.h[4] \n" - "fmla v16.8h, v13.8h, v0.h[5] \n" - - "subs %w0, %w0, #1 \n" - - "fmla v16.8h, v14.8h, v0.h[6] \n" - "fmla v16.8h, v15.8h, v0.h[7] \n" - - "bne 0b \n" - - "st1 {v16.8h}, [%1], #16 \n" - - : "=r"(nn), // %0 - "=r"(outptr0), // %1 - "=r"(tmpptr), // %2 - "=r"(kptr0) // %3 - : "0"(nn), - "1"(outptr0), - "2"(tmpptr), - "3"(kptr0), - "r"(biasptr) // %8 - : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16"); - } - } + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; - // // NOTE sgemm - // for (; p= 12 && num_output >= 12) + || (dilation_w == 1 && dilation_h == 1 && (stride_w >= 2 || stride_h >= 2) && num_input >= 16 && num_output >= 16) + || ((dilation_w >= 2 || dilation_h >= 2) && num_input >= 16 && num_output >= 16); + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); conv3x3s1_winograd42_transform_kernel_pack4_neon(weight_data, weight_3x3_winograd42_data_pack4, num_input, num_output); } - else + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - // src = kw-kh-inch-outch - // dst = 4b-4a-kw-kh-inch/4a-outch/4b - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16); - - for (int q = 0; q + 3 < num_output; q += 4) + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 24 && num_output >= 24) { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack4.channel(q / 4); - - for (int p = 0; p + 3 < num_input; p += 4) - { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - const float* k10 = k1.row(p); - const float* k11 = k1.row(p + 1); - const float* k12 = k1.row(p + 2); - const float* k13 = k1.row(p + 3); - - const float* k20 = k2.row(p); - const float* k21 = k2.row(p + 1); - const float* k22 = k2.row(p + 2); - const float* k23 = k2.row(p + 3); - - const float* k30 = k3.row(p); - const float* k31 = k3.row(p + 1); - const float* k32 = k3.row(p + 2); - const float* k33 = k3.row(p + 3); + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_sgemm_data_pack4, num_input, num_output, kernel_w, kernel_h); + } - float* g00 = g0.row(p / 4); + convolution_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); + } + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 48 && num_output >= 48) + { + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_sgemm_data_pack4, num_input, num_output, kernel_w, kernel_h); + } - for (int k = 0; k < maxk; k++) - { - g00[0] = k00[k]; - g00[1] = k10[k]; - g00[2] = k20[k]; - g00[3] = k30[k]; - - g00[4] = k01[k]; - g00[5] = k11[k]; - g00[6] = k21[k]; - g00[7] = k31[k]; - - g00[8] = k02[k]; - g00[9] = k12[k]; - g00[10] = k22[k]; - g00[11] = k32[k]; - - g00[12] = k03[k]; - g00[13] = k13[k]; - g00[14] = k23[k]; - g00[15] = k33[k]; - - g00 += 16; - } - } + convolution_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); + } + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 72 && num_output >= 72) + { + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_sgemm_data_pack4, num_input, num_output, kernel_w, kernel_h); } + + convolution_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); + } + else if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_transform_kernel_pack4_neon(weight_data, weight_sgemm_data_pack4, num_input, num_output, kernel_w, kernel_h); + } + else + { + convolution_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output, kernel_w, kernel_h); } } // pack1to4 if (elempack == 1 && out_elempack == 4) { - // src = kw-kh-inch-outch - // dst = 4b-kw-kh-inch-outch/4b - { - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack1to4.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4); - - for (int q = 0; q + 3 < num_output; q += 4) - { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack1to4.channel(q / 4); - - for (int p = 0; p < num_input; p++) - { - const float* k00 = k0.row(p); - const float* k10 = k1.row(p); - const float* k20 = k2.row(p); - const float* k30 = k3.row(p); - - float* g00 = g0.row(p); - - for (int k = 0; k < maxk; k++) - { - g00[0] = k00[k]; - g00[1] = k10[k]; - g00[2] = k20[k]; - g00[3] = k30[k]; - - g00 += 4; - } - } - } - } + convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_pack1to4, num_input, num_output, kernel_w, kernel_h); } // pack4to1 @@ -349,37 +306,7 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - // src = kw-kh-inch-outch - // dst = 4a-kw-kh-inch/4a-outch - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4to1.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4); - - for (int q = 0; q < num_output; q++) - { - const Mat k0 = weight_data_r2.channel(q); - Mat g0 = weight_data_pack4to1.channel(q); - - for (int p = 0; p + 3 < num_input; p += 4) - { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - float* g00 = g0.row(p / 4); - - for (int k = 0; k < maxk; k++) - { - g00[0] = k00[k]; - g00[1] = k01[k]; - g00[2] = k02[k]; - g00[3] = k03[k]; - - g00 += 4; - } - } - } + convolution_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output, kernel_w, kernel_h); } } #endif // __ARM_NEON @@ -411,7 +338,7 @@ int Convolution_arm::create_pipeline(const Option& opt) if (use_sgemm1x1) { - conv1x1s1_sgemm_transform_kernel_neon(weight_data, weight_1x1_sgemm_data, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } } @@ -424,12 +351,10 @@ int Convolution_arm::create_pipeline(const Option& opt) conv3x3s1_winograd64_transform_kernel_neon5(weight_data, weight_3x3_winograd64_data, num_input, num_output); break; case 2: - // pointwise - conv1x1s1_sgemm_transform_kernel_neon(weight_data, weight_1x1_sgemm_data, num_input, num_output); - break; + // pointwise case 3: // im2col - conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk); + convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); break; // case 4: // // direct @@ -448,12 +373,12 @@ int Convolution_arm::create_pipeline(const Option& opt) if (opt.use_sgemm_convolution && kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk); + convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } if (opt.use_sgemm_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk); + convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } } @@ -524,6 +449,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option w = bottom_blob_bordered.w; h = bottom_blob_bordered.h; + int size = w * h; int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; @@ -542,30 +468,15 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } } - const int maxk = kernel_w * kernel_h; - - // kernel offsets - std::vector _space_ofs(maxk); - int* space_ofs = &_space_ofs[0]; - { - int p1 = 0; - int p2 = 0; - int gap = w * dilation_h - kernel_w * dilation_w; - for (int i = 0; i < kernel_h; i++) - { - for (int j = 0; j < kernel_w; j++) - { - space_ofs[p1] = p2; - p1++; - p2 += dilation_w; - } - p2 += gap; - } - } + const int num_input = channels * elempack; #if __ARM_NEON if (elempack == 4 && out_elempack == 4) { + bool prefer_sgemm = (dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 12 && num_output >= 12) + || (dilation_w == 1 && dilation_h == 1 && (stride_w >= 2 || stride_h >= 2) && num_input >= 16 && num_output >= 16) + || ((dilation_w >= 2 || dilation_h >= 2) && num_input >= 16 && num_output >= 16); + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -603,7 +514,20 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + // we need more proper conditions + prefer_sgemm = (size >= 44 * 44 && num_input >= 48 && num_output >= 48) + || (size >= 28 * 28 && size < 44 * 44 && num_input >= 56 && num_output >= 56) + || (size >= 19 * 19 && size < 28 * 28 && num_input >= 64 && num_output >= 64) + || (size >= 17 * 17 && size < 19 * 19 && num_input >= 96 && num_output >= 96) + || (size >= 5 * 5 && size < 17 * 17 && num_input >= 24 && num_output >= 24); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + conv3x3s2_im2col_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_pack4, bias_data, opt); + } + else + { + conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + } if (activation) { @@ -612,7 +536,24 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + // we need more proper conditions + prefer_sgemm = (size >= 16 * 16 && size <= 17 * 17 && num_input >= 200 && num_output >= 200) + || (size >= 15 * 15 && size < 16 * 16 && num_input >= 128 && num_output >= 128) + || (size >= 13 * 13 && size < 15 * 15 && num_input >= 160 && num_output >= 160) + || (size >= 12 * 12 && size < 13 * 13 && num_input >= 184 && num_output >= 184) + || (size >= 11 * 11 && size < 12 * 12 && num_input >= 88 && num_output >= 88) + || (size >= 10 * 10 && size < 11 * 11 && num_input >= 128 && num_output >= 128) + || (size >= 9 * 9 && size < 10 * 10 && num_input >= 120 && num_output >= 120) + || (size >= 8 * 8 && size < 9 * 9 && num_input >= 192 && num_output >= 192) + || (size >= 6 * 6 && size < 8 * 8 && num_input >= 48 && num_output >= 48); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_pack4, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + } + else + { + conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + } if (activation) { @@ -621,74 +562,37 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + // we need more proper conditions + prefer_sgemm = (size >= 28 * 28 && num_input >= 144 && num_output >= 144) + || (size >= 12 * 12 && size < 28 * 28 && num_input >= 128 && num_output >= 128) + || (size >= 7 * 7 && size < 12 * 12 && num_input >= 72 && num_output >= 72); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_pack4, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + } + else + { + conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + } if (activation) { activation->forward_inplace(top_blob, opt); } } - else + else if (opt.use_sgemm_convolution && prefer_sgemm) { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - float* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32(((const float*)bias_data) + p * 4); - } - - const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16; - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const float* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) // 29.23 - { - float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4); - - float32x4_t _w0 = vld1q_f32(kptr); - float32x4_t _w1 = vld1q_f32(kptr + 4); - float32x4_t _w2 = vld1q_f32(kptr + 8); - float32x4_t _w3 = vld1q_f32(kptr + 12); - -#if __aarch64__ - _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0); - _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1); - _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2); - _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3); -#else - _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0); - _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1); - _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0); - _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1); -#endif - - kptr += 16; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1q_f32(outptr + j * 4, _sum); - } + convolution_im2col_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_pack4, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - outptr += outw * 4; - } + if (activation) + { + activation->forward_inplace(top_blob, opt); } } + else + { + convolution_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } } if (elempack == 1 && out_elempack == 4) @@ -722,49 +626,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - float* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32(((const float*)bias_data) + p * 4); - } - - const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4; - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const float* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) // 29.23 - { - float32x4_t _val = vdupq_n_f32(sptr[space_ofs[k]]); - float32x4_t _w = vld1q_f32(kptr); - _sum = vmlaq_f32(_sum, _val, _w); - - kptr += 4; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1q_f32(outptr + j * 4, _sum); - } - - outptr += outw * 4; - } - } + convolution_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -802,56 +664,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - float* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4; - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const float* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) // 29.23 - { - float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4); - float32x4_t _w = vld1q_f32(kptr); - float32x4_t _s4 = vmulq_f32(_val, _w); -#if __aarch64__ - sum += vaddvq_f32(_s4); // dot -#else - float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4)); - _ss = vpadd_f32(_ss, _ss); - sum += vget_lane_f32(_ss, 0); -#endif - - kptr += 4; - } - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = sum; - } - - outptr += outw; - } - } + convolution_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __ARM_NEON @@ -867,10 +680,10 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option conv3x3s1_winograd64_neon5(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data, opt); break; case 2: - conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_1x1_sgemm_data, bias_data, opt); + conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt); break; case 3: - conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); + convolution_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); break; // case 4: FIXME fallback to auto path // conv(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -889,7 +702,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option { if (use_sgemm1x1) { - conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_1x1_sgemm_data, bias_data, opt); + conv1x1s1_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt); } else { @@ -904,7 +717,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { if (opt.use_sgemm_convolution) - conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); + convolution_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); else conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -933,7 +746,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { if (opt.use_sgemm_convolution && !(outw >= 8 && outh >= 8)) - conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); + convolution_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); else conv3x3s2_packed_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data, bias_data, opt); @@ -989,6 +802,27 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + // num_output #pragma omp parallel for num_threads(opt.num_threads) for (int p = 0; p < num_output; p++) @@ -1114,19 +948,51 @@ int Convolution_arm::create_pipeline_fp16s(const Option& opt) // pack8 if (elempack == 8 && out_elempack == 8) { + bool prefer_sgemm = (dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 64 && num_output >= 64) + || (dilation_w == 1 && dilation_h == 1 && (stride_w >= 2 || stride_h >= 2) && num_input >= 32 && num_output >= 32) + || ((dilation_w >= 2 || dilation_h >= 2) && num_input >= 32 && num_output >= 32); + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_data_fp16, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_data_fp16, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s1_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_data_fp16, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_data_fp16, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_winograd64_transform_kernel_pack8_fp16sa_neon(weight_data, weight_data_fp16, num_input, num_output); conv3x3s1_winograd42_transform_kernel_pack8_fp16sa_neon(weight_data, weight_3x3_winograd42_data_pack4, num_input, num_output); } + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 48 && num_output >= 48) + { + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } + } + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 56 && num_output >= 56) + { + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } + } + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + // we need more proper conditions + if (opt.use_sgemm_convolution && num_input >= 64 && num_output >= 64) + { + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } + } + else if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } } // pack4to8 @@ -1243,243 +1109,36 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const // return forwardDilation_arm(bottom_blob_bordered, top_blob, opt); // } - const int maxk = kernel_w * kernel_h; + if (elempack == 4 && out_elempack == 4) + { + convolution_pack4_fp16s_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } - // kernel offsets - std::vector _space_ofs(maxk); - int* space_ofs = &_space_ofs[0]; + if (elempack == 1 && out_elempack == 4) { - int p1 = 0; - int p2 = 0; - int gap = w * dilation_h - kernel_w * dilation_w; - for (int i = 0; i < kernel_h; i++) - { - for (int j = 0; j < kernel_w; j++) - { - space_ofs[p1] = p2; - p1++; - p2 += dilation_w; - } - p2 += gap; - } + convolution_pack1to4_fp16s_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } - if (elempack == 4 && out_elempack == 4) + if (elempack == 4 && out_elempack == 1) { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); + convolution_pack4to1_fp16s_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); + if (elempack == 1 && out_elempack == 1) + { + convolution_fp16s(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } - if (bias_term) - { - _sum = vld1q_f32((const float*)bias_data + p * 4); - } + return 0; +} - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4)); - - float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr)); - float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4)); - float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8)); - float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12)); - - _sum = vfmaq_laneq_f32(_sum, _w0, _val, 0); - _sum = vfmaq_laneq_f32(_sum, _w1, _val, 1); - _sum = vfmaq_laneq_f32(_sum, _w2, _val, 2); - _sum = vfmaq_laneq_f32(_sum, _w3, _val, 3); - - kptr += 16; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum)); - } - - outptr += outw * 4; - } - } - } - } - - if (elempack == 1 && out_elempack == 4) - { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32((const float*)bias_data + p * 4); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vcvt_f32_f16(vdup_n_f16(sptr[space_ofs[k]])); - float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); - _sum = vfmaq_f32(_sum, _val, _w); - - kptr += 4; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum)); - } - - outptr += outw * 4; - } - } - } - } - - if (elempack == 4 && out_elempack == 1) - { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4)); - float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); - float32x4_t _s4 = vmulq_f32(_val, _w); - - sum += vaddvq_f32(_s4); // dot - - kptr += 4; - } - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = (__fp16)sum; - } - - outptr += outw; - } - } - } - } - - if (elempack == 1 && out_elempack == 1) - { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row<__fp16>(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float val = (float)sptr[space_ofs[k]]; - float w = (float)kptr[k]; - sum += val * w; - } - - kptr += maxk; - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = (__fp16)sum; - } - - outptr += outw; - } - } - } - } - - return 0; -} - -int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; - int elempack = bottom_blob.elempack; +int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; // NCNN_LOGE("Convolution input %d x %d pad = %d %d ksize=%d %d stride=%d %d", w, h, pad_w, pad_h, kernel_w, kernel_h, stride_w, stride_h); @@ -1493,6 +1152,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const w = bottom_blob_bordered.w; h = bottom_blob_bordered.h; + int size = w * h; int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; @@ -1513,29 +1173,14 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const // return forwardDilation_arm(bottom_blob_bordered, top_blob, opt); // } - const int maxk = kernel_w * kernel_h; - - // kernel offsets - std::vector _space_ofs(maxk); - int* space_ofs = &_space_ofs[0]; - { - int p1 = 0; - int p2 = 0; - int gap = w * dilation_h - kernel_w * dilation_w; - for (int i = 0; i < kernel_h; i++) - { - for (int j = 0; j < kernel_w; j++) - { - space_ofs[p1] = p2; - p1++; - p2 += dilation_w; - } - p2 += gap; - } - } + const int num_input = channels * elempack; if (elempack == 8 && out_elempack == 8) { + bool prefer_sgemm = (dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 64 && num_output >= 64) + || (dilation_w == 1 && dilation_h == 1 && (stride_w >= 2 || stride_h >= 2) && num_input >= 32 && num_output >= 32) + || ((dilation_w >= 2 || dilation_h >= 2) && num_input >= 32 && num_output >= 32); + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); @@ -1573,7 +1218,24 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + // we need more proper conditions + prefer_sgemm = (size >= 44 * 44 && num_input >= 192 && num_output >= 192) + || (size >= 28 * 28 && size < 44 * 44 && num_input >= 144 && num_output >= 144) + || (size >= 19 * 19 && size < 28 * 28 && num_input >= 160 && num_output >= 160) + || (size >= 17 * 17 && size < 19 * 19 && num_input >= 192 && num_output >= 192) + || (size >= 15 * 15 && size < 17 * 17 && num_input >= 112 && num_output >= 112) + || (size >= 13 * 13 && size < 15 * 15 && num_input >= 48 && num_output >= 48) + || (size >= 11 * 11 && size < 13 * 13 && num_input >= 56 && num_output >= 56) + || (size >= 9 * 9 && size < 11 * 11 && num_input >= 80 && num_output >= 80) + || (size >= 5 * 5 && size < 9 * 9 && num_input >= 64 && num_output >= 64); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + } + else + { + conv3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + } if (activation) { @@ -1582,7 +1244,19 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + // we need more proper conditions + prefer_sgemm = (size >= 12 * 12 && size <= 15 * 15 && num_input >= 256 && num_output >= 256) + || (size >= 10 * 10 && size < 12 * 12 && num_input >= 152 && num_output >= 152) + || (size >= 8 * 8 && size < 10 * 10 && num_input >= 232 && num_output >= 232) + || (size >= 6 * 6 && size < 8 * 8 && num_input >= 56 && num_output >= 56); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + } + else + { + conv5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + } if (activation) { @@ -1591,75 +1265,37 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + // we need more proper conditions + prefer_sgemm = (size >= 48 * 48 && num_input >= 160 && num_output >= 160) + || (size >= 11 * 11 && size < 48 * 48 && num_input >= 96 && num_output >= 96) + || (size >= 7 * 7 && size < 11 * 11 && num_input >= 64 && num_output >= 64); + if (opt.use_sgemm_convolution && prefer_sgemm) + { + convolution_im2col_sgemm_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + } + else + { + conv5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + } if (activation) { activation->forward_inplace(top_blob, opt); } } - else + else if (opt.use_sgemm_convolution && prefer_sgemm) { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x8_t _sum = vdupq_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; - - for (int k = 0; k < maxk; k++) - { - float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); - - float16x8_t _w0 = vld1q_f16(kptr); - float16x8_t _w1 = vld1q_f16(kptr + 8); - float16x8_t _w2 = vld1q_f16(kptr + 16); - float16x8_t _w3 = vld1q_f16(kptr + 24); - float16x8_t _w4 = vld1q_f16(kptr + 32); - float16x8_t _w5 = vld1q_f16(kptr + 40); - float16x8_t _w6 = vld1q_f16(kptr + 48); - float16x8_t _w7 = vld1q_f16(kptr + 56); - - _sum = vfmaq_laneq_f16(_sum, _w0, _val, 0); - _sum = vfmaq_laneq_f16(_sum, _w1, _val, 1); - _sum = vfmaq_laneq_f16(_sum, _w2, _val, 2); - _sum = vfmaq_laneq_f16(_sum, _w3, _val, 3); - _sum = vfmaq_laneq_f16(_sum, _w4, _val, 4); - _sum = vfmaq_laneq_f16(_sum, _w5, _val, 5); - _sum = vfmaq_laneq_f16(_sum, _w6, _val, 6); - _sum = vfmaq_laneq_f16(_sum, _w7, _val, 7); - - kptr += 64; - } - } + convolution_im2col_sgemm_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1q_f16(outptr + j * 8, _sum); - } - - outptr += outw * 8; - } + if (activation) + { + activation->forward_inplace(top_blob, opt); } } + else + { + convolution_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } } if (elempack == 1 && out_elempack == 8) @@ -1693,49 +1329,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x8_t _sum = vdupq_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float16x8_t _val = vdupq_n_f16(sptr[space_ofs[k]]); - float16x8_t _w = vld1q_f16(kptr); - _sum = vfmaq_f16(_sum, _val, _w); - - kptr += 8; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1q_f16(outptr + j * 8, _sum); - } - - outptr += outw * 8; - } - } + convolution_pack1to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1761,57 +1355,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x8_t _sum = vdupq_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); - - float16x8_t _w0 = vld1q_f16(kptr); - float16x8_t _w1 = vld1q_f16(kptr + 8); - float16x8_t _w2 = vld1q_f16(kptr + 16); - float16x8_t _w3 = vld1q_f16(kptr + 24); - - _sum = vfmaq_lane_f16(_sum, _w0, _val, 0); - _sum = vfmaq_lane_f16(_sum, _w1, _val, 1); - _sum = vfmaq_lane_f16(_sum, _w2, _val, 2); - _sum = vfmaq_lane_f16(_sum, _w3, _val, 3); - - kptr += 32; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1q_f16(outptr + j * 8, _sum); - } - - outptr += outw * 8; - } - } + convolution_pack4to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1849,52 +1393,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; - - for (int k = 0; k < maxk; k++) - { - float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); - float16x8_t _w = vld1q_f16(kptr); - float16x8_t _s8 = vmulq_f16(_val, _w); - - float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8)); - sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot - - kptr += 8; - } - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = sum; - } - - outptr += outw; - } - } + convolution_pack8to1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1932,65 +1431,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x4_t _sum = vdup_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; - - for (int k = 0; k < maxk; k++) - { - float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); - - float16x4_t _w0 = vld1_f16(kptr); - float16x4_t _w1 = vld1_f16(kptr + 4); - float16x4_t _w2 = vld1_f16(kptr + 8); - float16x4_t _w3 = vld1_f16(kptr + 12); - float16x4_t _w4 = vld1_f16(kptr + 16); - float16x4_t _w5 = vld1_f16(kptr + 20); - float16x4_t _w6 = vld1_f16(kptr + 24); - float16x4_t _w7 = vld1_f16(kptr + 28); - - _sum = vfma_laneq_f16(_sum, _w0, _val, 0); - _sum = vfma_laneq_f16(_sum, _w1, _val, 1); - _sum = vfma_laneq_f16(_sum, _w2, _val, 2); - _sum = vfma_laneq_f16(_sum, _w3, _val, 3); - _sum = vfma_laneq_f16(_sum, _w4, _val, 4); - _sum = vfma_laneq_f16(_sum, _w5, _val, 5); - _sum = vfma_laneq_f16(_sum, _w6, _val, 6); - _sum = vfma_laneq_f16(_sum, _w7, _val, 7); - - kptr += 32; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr + j * 4, _sum); - } - - outptr += outw * 4; - } - } + convolution_pack8to4_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -2026,160 +1467,20 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const activation->forward_inplace(top_blob, opt); } } - else - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x4_t _sum = vdup_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); - - float16x4_t _w0 = vld1_f16(kptr); - float16x4_t _w1 = vld1_f16(kptr + 4); - float16x4_t _w2 = vld1_f16(kptr + 8); - float16x4_t _w3 = vld1_f16(kptr + 12); - - _sum = vfma_lane_f16(_sum, _w0, _val, 0); - _sum = vfma_lane_f16(_sum, _w1, _val, 1); - _sum = vfma_lane_f16(_sum, _w2, _val, 2); - _sum = vfma_lane_f16(_sum, _w3, _val, 3); - - kptr += 16; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr + j * 4, _sum); - } - - outptr += outw * 4; - } - } - } - } - - if (elempack == 1 && out_elempack == 4) - { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float16x4_t _sum = vdup_n_f16((__fp16)0.f); - - if (bias_term) - { - _sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4); - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float16x4_t _val = vdup_n_f16(sptr[space_ofs[k]]); - float16x4_t _w = vld1_f16(kptr); - _sum = vfma_f16(_sum, _val, _w); - - kptr += 4; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_f16(outptr + j * 4, _sum); - } - - outptr += outw * 4; - } - } - } - } - - if (elempack == 4 && out_elempack == 1) - { - { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); - float16x4_t _w = vld1_f16(kptr); - float16x4_t _s4 = vmul_f16(_val, _w); - - sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot - - kptr += 4; - } - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = sum; - } - - outptr += outw; - } - } - } + else + { + convolution_pack4_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } + } + + if (elempack == 1 && out_elempack == 4) + { + convolution_pack1to4_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + } + + if (elempack == 4 && out_elempack == 1) + { + convolution_pack4to1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } if (elempack == 1 && out_elempack == 1) @@ -2195,49 +1496,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - __fp16* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const __fp16* kptr = weight_data_fp16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const __fp16* sptr = m.row<__fp16>(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - __fp16 val = sptr[space_ofs[k]]; - __fp16 w = kptr[k]; - sum += val * w; - } - - kptr += maxk; - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = sum; - } - - outptr += outw; - } - } + convolution_fp16s(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -2259,127 +1518,27 @@ int Convolution_arm::create_pipeline_bf16s(const Option& opt) { if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_pack4_bf16, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s1_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_pack4_bf16, num_input, num_output); + convolution_im2col_sgemm_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4_bf16, num_input, num_output); + conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_bf16, num_input, num_output); conv3x3s1_winograd42_transform_kernel_pack4_neon(weight_data, weight_3x3_winograd42_data_pack4, num_input, num_output); } else { - // src = kw-kh-inch-outch - // dst = 4b-4a-kw-kh-inch/4a-outch/4b - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4_bf16.create(maxk, num_input / 4, num_output / 4, (size_t)2 * 16, 16); - - for (int q = 0; q + 3 < num_output; q += 4) - { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack4_bf16.channel(q / 4); - - for (int p = 0; p + 3 < num_input; p += 4) - { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - const float* k10 = k1.row(p); - const float* k11 = k1.row(p + 1); - const float* k12 = k1.row(p + 2); - const float* k13 = k1.row(p + 3); - - const float* k20 = k2.row(p); - const float* k21 = k2.row(p + 1); - const float* k22 = k2.row(p + 2); - const float* k23 = k2.row(p + 3); - - const float* k30 = k3.row(p); - const float* k31 = k3.row(p + 1); - const float* k32 = k3.row(p + 2); - const float* k33 = k3.row(p + 3); - - unsigned short* g00 = g0.row(p / 4); - - for (int k = 0; k < maxk; k++) - { - g00[0] = float32_to_bfloat16(k00[k]); - g00[1] = float32_to_bfloat16(k10[k]); - g00[2] = float32_to_bfloat16(k20[k]); - g00[3] = float32_to_bfloat16(k30[k]); - - g00[4] = float32_to_bfloat16(k01[k]); - g00[5] = float32_to_bfloat16(k11[k]); - g00[6] = float32_to_bfloat16(k21[k]); - g00[7] = float32_to_bfloat16(k31[k]); - - g00[8] = float32_to_bfloat16(k02[k]); - g00[9] = float32_to_bfloat16(k12[k]); - g00[10] = float32_to_bfloat16(k22[k]); - g00[11] = float32_to_bfloat16(k32[k]); - - g00[12] = float32_to_bfloat16(k03[k]); - g00[13] = float32_to_bfloat16(k13[k]); - g00[14] = float32_to_bfloat16(k23[k]); - g00[15] = float32_to_bfloat16(k33[k]); - - g00 += 16; - } - } - } + convolution_transform_kernel_pack4_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output, kernel_w, kernel_h); } } // pack1to4 if (elempack == 1 && out_elempack == 4) { - // src = kw-kh-inch-outch - // dst = 4b-kw-kh-inch-outch/4b - { - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack1to4_bf16.create(maxk, num_input, num_output / 4, (size_t)2 * 4, 4); - - for (int q = 0; q + 3 < num_output; q += 4) - { - const Mat k0 = weight_data_r2.channel(q); - const Mat k1 = weight_data_r2.channel(q + 1); - const Mat k2 = weight_data_r2.channel(q + 2); - const Mat k3 = weight_data_r2.channel(q + 3); - - Mat g0 = weight_data_pack1to4_bf16.channel(q / 4); - - for (int p = 0; p < num_input; p++) - { - const float* k00 = k0.row(p); - const float* k10 = k1.row(p); - const float* k20 = k2.row(p); - const float* k30 = k3.row(p); - - unsigned short* g00 = g0.row(p); - - for (int k = 0; k < maxk; k++) - { - g00[0] = float32_to_bfloat16(k00[k]); - g00[1] = float32_to_bfloat16(k10[k]); - g00[2] = float32_to_bfloat16(k20[k]); - g00[3] = float32_to_bfloat16(k30[k]); - - g00 += 4; - } - } - } - } + convolution_transform_kernel_pack1to4_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output, kernel_w, kernel_h); } // pack4to1 @@ -2387,49 +1546,19 @@ int Convolution_arm::create_pipeline_bf16s(const Option& opt) { if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output); + conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output); } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output); + conv1x1s1_sgemm_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1_bf16, num_input, num_output); + conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_bf16, num_input, num_output); } else { - // src = kw-kh-inch-outch - // dst = 4a-kw-kh-inch/4a-outch - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_pack4to1_bf16.create(maxk, num_input / 4, num_output, (size_t)2 * 4, 4); - - for (int q = 0; q < num_output; q++) - { - const Mat k0 = weight_data_r2.channel(q); - Mat g0 = weight_data_pack4to1_bf16.channel(q); - - for (int p = 0; p + 3 < num_input; p += 4) - { - const float* k00 = k0.row(p); - const float* k01 = k0.row(p + 1); - const float* k02 = k0.row(p + 2); - const float* k03 = k0.row(p + 3); - - unsigned short* g00 = g0.row(p / 4); - - for (int k = 0; k < maxk; k++) - { - g00[0] = float32_to_bfloat16(k00[k]); - g00[1] = float32_to_bfloat16(k01[k]); - g00[2] = float32_to_bfloat16(k02[k]); - g00[3] = float32_to_bfloat16(k03[k]); - - g00 += 4; - } - } - } + convolution_transform_kernel_pack4to1_bf16s_neon(weight_data, weight_data_bf16, num_input, num_output, kernel_w, kernel_h); } } #endif // __ARM_NEON @@ -2486,33 +1615,12 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const // return forwardDilation_arm(bottom_blob_bordered, top_blob, opt); // } - const int maxk = kernel_w * kernel_h; - - // kernel offsets - std::vector _space_ofs(maxk); - int* space_ofs = &_space_ofs[0]; - { - int p1 = 0; - int p2 = 0; - int gap = w * dilation_h - kernel_w * dilation_w; - for (int i = 0; i < kernel_h; i++) - { - for (int j = 0; j < kernel_w; j++) - { - space_ofs[p1] = p2; - p1++; - p2 += dilation_w; - } - p2 += gap; - } - } - #if __ARM_NEON if (elempack == 4 && out_elempack == 4) { if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv1x1s1_sgemm_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2521,7 +1629,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv1x1s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2537,7 +1645,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else { - conv3x3s1_winograd64_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv3x3s1_winograd64_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); } if (activation) @@ -2547,7 +1655,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2556,7 +1664,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2565,7 +1673,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + conv5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2574,64 +1682,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - unsigned short* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32(((const float*)bias_data) + p * 4); - } - - const unsigned short* kptr = weight_data_pack4_bf16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const unsigned short* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4)); - - float32x4_t _w0 = vcvt_f32_bf16(vld1_u16(kptr)); - float32x4_t _w1 = vcvt_f32_bf16(vld1_u16(kptr + 4)); - float32x4_t _w2 = vcvt_f32_bf16(vld1_u16(kptr + 8)); - float32x4_t _w3 = vcvt_f32_bf16(vld1_u16(kptr + 12)); - -#if __aarch64__ - _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0); - _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1); - _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2); - _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3); -#else - _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0); - _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1); - _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0); - _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1); -#endif - - kptr += 16; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum)); - } - - outptr += outw * 4; - } - } + convolution_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -2639,7 +1690,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt); + conv3x3s1_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2648,7 +1699,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt); + conv3x3s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2657,7 +1708,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv7x7s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4_bf16, bias_data, opt); + conv7x7s2_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2666,49 +1717,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) - { - unsigned short* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float32x4_t _sum = vdupq_n_f32(0.f); - - if (bias_term) - { - _sum = vld1q_f32(((const float*)bias_data) + p * 4); - } - - const unsigned short* kptr = weight_data_pack1to4_bf16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const unsigned short* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vdupq_n_f32(bfloat16_to_float32(sptr[space_ofs[k]])); - float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr)); - _sum = vmlaq_f32(_sum, _val, _w); - - kptr += 4; - } - } - - _sum = activation_ps(_sum, activation_type, activation_params); - - vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum)); - } - - outptr += outw * 4; - } - } + convolution_pack1to4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -2716,7 +1725,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const { if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv1x1s1_sgemm_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt); + conv1x1s1_sgemm_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2725,7 +1734,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv1x1s2_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt); + conv1x1s2_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2735,9 +1744,9 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { // TODO more proper condition - conv3x3s1_winograd64_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt); + conv3x3s1_winograd64_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); - // conv3x3s1_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1_bf16, bias_data, opt); + // conv3x3s1_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); if (activation) { @@ -2746,56 +1755,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - unsigned short* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const unsigned short* kptr = weight_data_pack4to1_bf16.channel(p); - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const unsigned short* sptr = m.row(i * stride_h) + j * stride_w * 4; - - for (int k = 0; k < maxk; k++) - { - float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4)); - float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr)); - float32x4_t _s4 = vmulq_f32(_val, _w); -#if __aarch64__ - sum += vaddvq_f32(_s4); // dot -#else - float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4)); - _ss = vpadd_f32(_ss, _ss); - sum += vget_lane_f32(_ss, 0); -#endif - - kptr += 4; - } - } - - sum = activation_ss(sum, activation_type, activation_params); - - outptr[j] = float32_to_bfloat16(sum); - } - - outptr += outw; - } - } + convolution_pack4to1_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __ARM_NEON @@ -2813,74 +1773,7 @@ int Convolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const } else { - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - unsigned short* outptr = top_blob.channel(p); - - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - float sum = 0.f; - - if (bias_term) - { - sum = bias_data[p]; - } - - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * channels * p; - - // channels - for (int q = 0; q < channels; q++) - { - const Mat m = bottom_blob_bordered.channel(q); - const unsigned short* sptr = m.row(i * stride_h) + j * stride_w; - - for (int k = 0; k < maxk; k++) - { - float val = bfloat16_to_float32(sptr[space_ofs[k]]); - float wt = bfloat16_to_float32(kptr[k]); - sum += val * wt; - } - - kptr += maxk; - } - - if (activation_type == 1) - { - sum = std::max(sum, 0.f); - } - else if (activation_type == 2) - { - float slope = activation_params[0]; - sum = sum > 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; - } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } - else if (activation_type == 5) - { - sum = static_cast(sum * tanh(log(exp(sum) + 1.f))); - } - - outptr[j] = float32_to_bfloat16(sum); - } - - outptr += outw; - } - } + convolution_bf16s(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } diff --git a/src/layer/arm/convolution_arm.h b/src/layer/arm/convolution_arm.h index fa38366d3..bf505c217 100644 --- a/src/layer/arm/convolution_arm.h +++ b/src/layer/arm/convolution_arm.h @@ -46,7 +46,6 @@ public: bool use_winograd3x3; bool use_sgemm1x1; Mat weight_3x3_winograd64_data; - Mat weight_1x1_sgemm_data; Mat weight_3x3s2_data; Mat weight_sgemm_data; @@ -59,15 +58,13 @@ public: Mat weight_data_pack4to1; Mat weight_3x3_winograd42_data_pack4; + Mat weight_sgemm_data_pack4; // fp16 Mat weight_data_fp16; Mat bias_data_fp16; // bf16 - Mat weight_data_pack4_bf16; - Mat weight_data_pack1to4_bf16; - Mat weight_data_pack4to1_bf16; Mat weight_data_bf16; // int8 diff --git a/src/layer/arm/convolution_bf16s.h b/src/layer/arm/convolution_bf16s.h new file mode 100644 index 000000000..c2d2eb7e6 --- /dev/null +++ b/src/layer/arm/convolution_bf16s.h @@ -0,0 +1,115 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_bf16s(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + unsigned short* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * channels * p; + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const unsigned short* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float val = bfloat16_to_float32(sptr[space_ofs[k]]); + float wt = bfloat16_to_float32(kptr[k]); + sum += val * wt; + } + + kptr += maxk; + } + + if (activation_type == 1) + { + sum = std::max(sum, 0.f); + } + else if (activation_type == 2) + { + float slope = activation_params[0]; + sum = sum > 0.f ? sum : sum * slope; + } + else if (activation_type == 3) + { + float min = activation_params[0]; + float max = activation_params[1]; + if (sum < min) + sum = min; + if (sum > max) + sum = max; + } + else if (activation_type == 4) + { + sum = static_cast(1.f / (1.f + exp(-sum))); + } + else if (activation_type == 5) + { + sum = static_cast(sum * tanh(log(exp(sum) + 1.f))); + } + + outptr[j] = float32_to_bfloat16(sum); + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_fp16s.h b/src/layer/arm/convolution_fp16s.h new file mode 100644 index 000000000..7b08a9852 --- /dev/null +++ b/src/layer/arm/convolution_fp16s.h @@ -0,0 +1,90 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_fp16s(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row<__fp16>(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float val = (float)sptr[space_ofs[k]]; + float w = (float)kptr[k]; + sum += val * w; + } + + kptr += maxk; + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = (__fp16)sum; + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_pack1to4.h b/src/layer/arm/convolution_pack1to4.h new file mode 100644 index 000000000..edc556b55 --- /dev/null +++ b/src/layer/arm/convolution_pack1to4.h @@ -0,0 +1,131 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack1to4_neon(const Mat& weight_data, Mat& weight_data_pack1to4, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4b-kw-kh-inch-outch/4b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_pack1to4.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4); + + for (int q = 0; q + 3 < num_output; q += 4) + { + const Mat k0 = weight_data_r2.channel(q); + const Mat k1 = weight_data_r2.channel(q + 1); + const Mat k2 = weight_data_r2.channel(q + 2); + const Mat k3 = weight_data_r2.channel(q + 3); + + Mat g0 = weight_data_pack1to4.channel(q / 4); + + for (int p = 0; p < num_input; p++) + { + const float* k00 = k0.row(p); + const float* k10 = k1.row(p); + const float* k20 = k2.row(p); + const float* k30 = k3.row(p); + + float* g00 = g0.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + + g00 += 4; + } + } + } +} + +static void convolution_pack1to4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack1to4, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4; + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const float* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) // 29.23 + { + float32x4_t _val = vdupq_n_f32(sptr[space_ofs[k]]); + float32x4_t _w = vld1q_f32(kptr); + _sum = vmlaq_f32(_sum, _val, _w); + + kptr += 4; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f32(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack1to4_bf16s.h b/src/layer/arm/convolution_pack1to4_bf16s.h new file mode 100644 index 000000000..5128c8d1c --- /dev/null +++ b/src/layer/arm/convolution_pack1to4_bf16s.h @@ -0,0 +1,131 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack1to4_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4b-kw-kh-inch-outch/4b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_bf16.create(maxk, num_input, num_output / 4, (size_t)2 * 4, 4); + + for (int q = 0; q + 3 < num_output; q += 4) + { + const Mat k0 = weight_data_r2.channel(q); + const Mat k1 = weight_data_r2.channel(q + 1); + const Mat k2 = weight_data_r2.channel(q + 2); + const Mat k3 = weight_data_r2.channel(q + 3); + + Mat g0 = weight_data_bf16.channel(q / 4); + + for (int p = 0; p < num_input; p++) + { + const float* k00 = k0.row(p); + const float* k10 = k1.row(p); + const float* k20 = k2.row(p); + const float* k30 = k3.row(p); + + unsigned short* g00 = g0.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = float32_to_bfloat16(k00[k]); + g00[1] = float32_to_bfloat16(k10[k]); + g00[2] = float32_to_bfloat16(k20[k]); + g00[3] = float32_to_bfloat16(k30[k]); + + g00 += 4; + } + } + } +} + +static void convolution_pack1to4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + unsigned short* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const unsigned short* kptr = weight_data_bf16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const unsigned short* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vdupq_n_f32(bfloat16_to_float32(sptr[space_ofs[k]])); + float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr)); + _sum = vmlaq_f32(_sum, _val, _w); + + kptr += 4; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum)); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack1to4_fp16s.h b/src/layer/arm/convolution_pack1to4_fp16s.h new file mode 100644 index 000000000..983f8b9ef --- /dev/null +++ b/src/layer/arm/convolution_pack1to4_fp16s.h @@ -0,0 +1,167 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack1to4_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_f16(vdup_n_f16(sptr[space_ofs[k]])); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); + _sum = vfmaq_f32(_sum, _val, _w); + + kptr += 4; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum)); + } + + outptr += outw * 4; + } + } +} + +static void convolution_pack1to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x4_t _sum = vdup_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1_f16(bias_data_ptr + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vdup_n_f16(sptr[space_ofs[k]]); + float16x4_t _w = vld1_f16(kptr); + _sum = vfma_f16(_sum, _val, _w); + + kptr += 4; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack1to8_fp16s.h b/src/layer/arm/convolution_pack1to8_fp16s.h new file mode 100644 index 000000000..6f6b0be51 --- /dev/null +++ b/src/layer/arm/convolution_pack1to8_fp16s.h @@ -0,0 +1,90 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f16(bias_data_ptr + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vdupq_n_f16(sptr[space_ofs[k]]); + float16x8_t _w = vld1q_f16(kptr); + _sum = vfmaq_f16(_sum, _val, _w); + + kptr += 8; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } +} diff --git a/src/layer/arm/convolution_pack4.h b/src/layer/arm/convolution_pack4.h new file mode 100644 index 000000000..88f238305 --- /dev/null +++ b/src/layer/arm/convolution_pack4.h @@ -0,0 +1,175 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack4_neon(const Mat& weight_data, Mat& weight_data_pack4, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4b-4a-kw-kh-inch/4a-outch/4b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_pack4.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16); + + for (int q = 0; q + 3 < num_output; q += 4) + { + const Mat k0 = weight_data_r2.channel(q); + const Mat k1 = weight_data_r2.channel(q + 1); + const Mat k2 = weight_data_r2.channel(q + 2); + const Mat k3 = weight_data_r2.channel(q + 3); + + Mat g0 = weight_data_pack4.channel(q / 4); + + for (int p = 0; p + 3 < num_input; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + float* g00 = g0.row(p / 4); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + + g00[4] = k01[k]; + g00[5] = k11[k]; + g00[6] = k21[k]; + g00[7] = k31[k]; + + g00[8] = k02[k]; + g00[9] = k12[k]; + g00[10] = k22[k]; + g00[11] = k32[k]; + + g00[12] = k03[k]; + g00[13] = k13[k]; + g00[14] = k23[k]; + g00[15] = k33[k]; + + g00 += 16; + } + } + } +} + +static void convolution_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack4, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16; + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const float* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) // 29.23 + { + float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4); + + float32x4_t _w0 = vld1q_f32(kptr); + float32x4_t _w1 = vld1q_f32(kptr + 4); + float32x4_t _w2 = vld1q_f32(kptr + 8); + float32x4_t _w3 = vld1q_f32(kptr + 12); + +#if __aarch64__ + _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0); + _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1); + _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2); + _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3); +#else + _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1); + _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1); +#endif + + kptr += 16; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f32(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack4_bf16s.h b/src/layer/arm/convolution_pack4_bf16s.h new file mode 100644 index 000000000..2a741df36 --- /dev/null +++ b/src/layer/arm/convolution_pack4_bf16s.h @@ -0,0 +1,176 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack4_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4b-4a-kw-kh-inch/4a-outch/4b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_bf16.create(maxk, num_input / 4, num_output / 4, (size_t)2 * 16, 16); + + for (int q = 0; q + 3 < num_output; q += 4) + { + const Mat k0 = weight_data_r2.channel(q); + const Mat k1 = weight_data_r2.channel(q + 1); + const Mat k2 = weight_data_r2.channel(q + 2); + const Mat k3 = weight_data_r2.channel(q + 3); + + Mat g0 = weight_data_bf16.channel(q / 4); + + for (int p = 0; p + 3 < num_input; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + unsigned short* g00 = g0.row(p / 4); + + for (int k = 0; k < maxk; k++) + { + g00[0] = float32_to_bfloat16(k00[k]); + g00[1] = float32_to_bfloat16(k10[k]); + g00[2] = float32_to_bfloat16(k20[k]); + g00[3] = float32_to_bfloat16(k30[k]); + + g00[4] = float32_to_bfloat16(k01[k]); + g00[5] = float32_to_bfloat16(k11[k]); + g00[6] = float32_to_bfloat16(k21[k]); + g00[7] = float32_to_bfloat16(k31[k]); + + g00[8] = float32_to_bfloat16(k02[k]); + g00[9] = float32_to_bfloat16(k12[k]); + g00[10] = float32_to_bfloat16(k22[k]); + g00[11] = float32_to_bfloat16(k32[k]); + + g00[12] = float32_to_bfloat16(k03[k]); + g00[13] = float32_to_bfloat16(k13[k]); + g00[14] = float32_to_bfloat16(k23[k]); + g00[15] = float32_to_bfloat16(k33[k]); + + g00 += 16; + } + } + } +} + +static void convolution_pack4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + unsigned short* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const unsigned short* kptr = weight_data_bf16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const unsigned short* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4)); + + float32x4_t _w0 = vcvt_f32_bf16(vld1_u16(kptr)); + float32x4_t _w1 = vcvt_f32_bf16(vld1_u16(kptr + 4)); + float32x4_t _w2 = vcvt_f32_bf16(vld1_u16(kptr + 8)); + float32x4_t _w3 = vcvt_f32_bf16(vld1_u16(kptr + 12)); + +#if __aarch64__ + _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0); + _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1); + _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2); + _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3); +#else + _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1); + _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1); +#endif + + kptr += 16; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum)); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack4_fp16s.h b/src/layer/arm/convolution_pack4_fp16s.h new file mode 100644 index 000000000..fddb05eca --- /dev/null +++ b/src/layer/arm/convolution_pack4_fp16s.h @@ -0,0 +1,183 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack4_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f32(bias_data_ptr + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4)); + + float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr)); + float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4)); + float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8)); + float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12)); + + _sum = vfmaq_laneq_f32(_sum, _w0, _val, 0); + _sum = vfmaq_laneq_f32(_sum, _w1, _val, 1); + _sum = vfmaq_laneq_f32(_sum, _w2, _val, 2); + _sum = vfmaq_laneq_f32(_sum, _w3, _val, 3); + + kptr += 16; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum)); + } + + outptr += outw * 4; + } + } +} + +static void convolution_pack4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x4_t _sum = vdup_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1_f16(bias_data_ptr + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); + + float16x4_t _w0 = vld1_f16(kptr); + float16x4_t _w1 = vld1_f16(kptr + 4); + float16x4_t _w2 = vld1_f16(kptr + 8); + float16x4_t _w3 = vld1_f16(kptr + 12); + + _sum = vfma_lane_f16(_sum, _w0, _val, 0); + _sum = vfma_lane_f16(_sum, _w1, _val, 1); + _sum = vfma_lane_f16(_sum, _w2, _val, 2); + _sum = vfma_lane_f16(_sum, _w3, _val, 3); + + kptr += 16; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_pack4to1.h b/src/layer/arm/convolution_pack4to1.h new file mode 100644 index 000000000..874be014d --- /dev/null +++ b/src/layer/arm/convolution_pack4to1.h @@ -0,0 +1,134 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack4to1_neon(const Mat& weight_data, Mat& weight_data_pack4to1, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4a-kw-kh-inch/4a-outch + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_pack4to1.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4); + + for (int q = 0; q < num_output; q++) + { + const Mat k0 = weight_data_r2.channel(q); + Mat g0 = weight_data_pack4to1.channel(q); + + for (int p = 0; p + 3 < num_input; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + float* g00 = g0.row(p / 4); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k01[k]; + g00[2] = k02[k]; + g00[3] = k03[k]; + + g00 += 4; + } + } + } +} + +static void convolution_pack4to1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack4to1, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4; + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const float* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) // 29.23 + { + float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4); + float32x4_t _w = vld1q_f32(kptr); + float32x4_t _s4 = vmulq_f32(_val, _w); +#if __aarch64__ + sum += vaddvq_f32(_s4); // dot +#else + float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4)); + _ss = vpadd_f32(_ss, _ss); + sum += vget_lane_f32(_ss, 0); +#endif + + kptr += 4; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = sum; + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_pack4to1_bf16s.h b/src/layer/arm/convolution_pack4to1_bf16s.h new file mode 100644 index 000000000..f8909aaf9 --- /dev/null +++ b/src/layer/arm/convolution_pack4to1_bf16s.h @@ -0,0 +1,134 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_transform_kernel_pack4to1_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // src = kw-kh-inch-outch + // dst = 4a-kw-kh-inch/4a-outch + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_bf16.create(maxk, num_input / 4, num_output, (size_t)2 * 4, 4); + + for (int q = 0; q < num_output; q++) + { + const Mat k0 = weight_data_r2.channel(q); + Mat g0 = weight_data_bf16.channel(q); + + for (int p = 0; p + 3 < num_input; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + unsigned short* g00 = g0.row(p / 4); + + for (int k = 0; k < maxk; k++) + { + g00[0] = float32_to_bfloat16(k00[k]); + g00[1] = float32_to_bfloat16(k01[k]); + g00[2] = float32_to_bfloat16(k02[k]); + g00[3] = float32_to_bfloat16(k03[k]); + + g00 += 4; + } + } + } +} + +static void convolution_pack4to1_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + unsigned short* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const unsigned short* kptr = weight_data_bf16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const unsigned short* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4)); + float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr)); + float32x4_t _s4 = vmulq_f32(_val, _w); +#if __aarch64__ + sum += vaddvq_f32(_s4); // dot +#else + float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4)); + _ss = vpadd_f32(_ss, _ss); + sum += vget_lane_f32(_ss, 0); +#endif + + kptr += 4; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = float32_to_bfloat16(sum); + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_pack4to1_fp16s.h b/src/layer/arm/convolution_pack4to1_fp16s.h new file mode 100644 index 000000000..f0f07f794 --- /dev/null +++ b/src/layer/arm/convolution_pack4to1_fp16s.h @@ -0,0 +1,171 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack4to1_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4)); + float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr)); + float32x4_t _s4 = vmulq_f32(_val, _w); + + sum += vaddvq_f32(_s4); // dot + + kptr += 4; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = (__fp16)sum; + } + + outptr += outw; + } + } +} + +static void convolution_pack4to1_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); + float16x4_t _w = vld1_f16(kptr); + float16x4_t _s4 = vmul_f16(_val, _w); + + sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot + + kptr += 4; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = sum; + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_pack4to8_fp16s.h b/src/layer/arm/convolution_pack4to8_fp16s.h new file mode 100644 index 000000000..41e7dc5ec --- /dev/null +++ b/src/layer/arm/convolution_pack4to8_fp16s.h @@ -0,0 +1,98 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack4to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f16(bias_data_ptr + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 4; + + for (int k = 0; k < maxk; k++) + { + float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4); + + float16x8_t _w0 = vld1q_f16(kptr); + float16x8_t _w1 = vld1q_f16(kptr + 8); + float16x8_t _w2 = vld1q_f16(kptr + 16); + float16x8_t _w3 = vld1q_f16(kptr + 24); + + _sum = vfmaq_lane_f16(_sum, _w0, _val, 0); + _sum = vfmaq_lane_f16(_sum, _w1, _val, 1); + _sum = vfmaq_lane_f16(_sum, _w2, _val, 2); + _sum = vfmaq_lane_f16(_sum, _w3, _val, 3); + + kptr += 32; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } +} diff --git a/src/layer/arm/convolution_pack8_fp16s.h b/src/layer/arm/convolution_pack8_fp16s.h new file mode 100644 index 000000000..e977eb4f9 --- /dev/null +++ b/src/layer/arm/convolution_pack8_fp16s.h @@ -0,0 +1,106 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x8_t _sum = vdupq_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1q_f16(bias_data_ptr + p * 8); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + + float16x8_t _w0 = vld1q_f16(kptr); + float16x8_t _w1 = vld1q_f16(kptr + 8); + float16x8_t _w2 = vld1q_f16(kptr + 16); + float16x8_t _w3 = vld1q_f16(kptr + 24); + float16x8_t _w4 = vld1q_f16(kptr + 32); + float16x8_t _w5 = vld1q_f16(kptr + 40); + float16x8_t _w6 = vld1q_f16(kptr + 48); + float16x8_t _w7 = vld1q_f16(kptr + 56); + + _sum = vfmaq_laneq_f16(_sum, _w0, _val, 0); + _sum = vfmaq_laneq_f16(_sum, _w1, _val, 1); + _sum = vfmaq_laneq_f16(_sum, _w2, _val, 2); + _sum = vfmaq_laneq_f16(_sum, _w3, _val, 3); + _sum = vfmaq_laneq_f16(_sum, _w4, _val, 4); + _sum = vfmaq_laneq_f16(_sum, _w5, _val, 5); + _sum = vfmaq_laneq_f16(_sum, _w6, _val, 6); + _sum = vfmaq_laneq_f16(_sum, _w7, _val, 7); + + kptr += 64; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f16(outptr + j * 8, _sum); + } + + outptr += outw * 8; + } + } +} diff --git a/src/layer/arm/convolution_pack8to1_fp16s.h b/src/layer/arm/convolution_pack8to1_fp16s.h new file mode 100644 index 000000000..a5aa21ddd --- /dev/null +++ b/src/layer/arm/convolution_pack8to1_fp16s.h @@ -0,0 +1,93 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack8to1_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const float* bias_data_ptr = bias_data; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; + + if (bias_data_ptr) + { + sum = bias_data_ptr[p]; + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + float16x8_t _w = vld1q_f16(kptr); + float16x8_t _s8 = vmulq_f16(_val, _w); + + float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8)); + sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot + + kptr += 8; + } + } + + sum = activation_ss(sum, activation_type, activation_params); + + outptr[j] = sum; + } + + outptr += outw; + } + } +} diff --git a/src/layer/arm/convolution_pack8to4_fp16s.h b/src/layer/arm/convolution_pack8to4_fp16s.h new file mode 100644 index 000000000..8e3c00536 --- /dev/null +++ b/src/layer/arm/convolution_pack8to4_fp16s.h @@ -0,0 +1,106 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_pack8to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + const __fp16* bias_data_ptr = bias_data_fp16; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr = top_blob.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float16x4_t _sum = vdup_n_f16((__fp16)0.f); + + if (bias_data_ptr) + { + _sum = vld1_f16(bias_data_ptr + p * 4); + } + + const __fp16* kptr = weight_data_fp16.channel(p); + + // channels + for (int q = 0; q < channels; q++) + { + const Mat m = bottom_blob.channel(q); + const __fp16* sptr = m.row(i * stride_h) + j * stride_w * 8; + + for (int k = 0; k < maxk; k++) + { + float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8); + + float16x4_t _w0 = vld1_f16(kptr); + float16x4_t _w1 = vld1_f16(kptr + 4); + float16x4_t _w2 = vld1_f16(kptr + 8); + float16x4_t _w3 = vld1_f16(kptr + 12); + float16x4_t _w4 = vld1_f16(kptr + 16); + float16x4_t _w5 = vld1_f16(kptr + 20); + float16x4_t _w6 = vld1_f16(kptr + 24); + float16x4_t _w7 = vld1_f16(kptr + 28); + + _sum = vfma_laneq_f16(_sum, _w0, _val, 0); + _sum = vfma_laneq_f16(_sum, _w1, _val, 1); + _sum = vfma_laneq_f16(_sum, _w2, _val, 2); + _sum = vfma_laneq_f16(_sum, _w3, _val, 3); + _sum = vfma_laneq_f16(_sum, _w4, _val, 4); + _sum = vfma_laneq_f16(_sum, _w5, _val, 5); + _sum = vfma_laneq_f16(_sum, _w6, _val, 6); + _sum = vfma_laneq_f16(_sum, _w7, _val, 7); + + kptr += 32; + } + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1_f16(outptr + j * 4, _sum); + } + + outptr += outw * 4; + } + } +} diff --git a/src/layer/arm/convolution_sgemm.h b/src/layer/arm/convolution_sgemm.h index b72969eb7..517c6cbad 100644 --- a/src/layer/arm/convolution_sgemm.h +++ b/src/layer/arm/convolution_sgemm.h @@ -1,6 +1,6 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. +// Tencent is pleased to support the open source community by making ncnn available. // -// Copyright (C) 2019 BUG1989. All rights reserved. +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. // // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -12,1511 +12,1822 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -static void conv_im2col_sgemm_transform_kernel_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size) +static void im2col_sgemm_neon(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) { - const float* kernel = _kernel; + // Mat bottom_im2col(size, maxk, inch, 4u, 1, opt.workspace_allocator); -#if __ARM_NEON && __aarch64__ - // kernel memory packed 8 x 8 - kernel_tm.create(8 * kernel_size, inch, outch / 8 + (outch % 8) / 4 + outch % 4); -#else - // kernel memory packed 4 x 8 - kernel_tm.create(4 * kernel_size, inch, outch / 4 + outch % 4); -#endif + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const float* bias = _bias; + + // permute + Mat tmp; +#if __ARM_NEON + if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + size % 4, 4u, 1, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + size % 4, 4u, 1, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 4u, 1, opt.workspace_allocator); + { + int nn_size = size >> 3; + int remain_size_start = 0; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + + float* tmpptr = tmp.channel(i / 8); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i; + + for (int k = 0; k < maxk; k++) + { + vst1q_f32(tmpptr, vld1q_f32(img0)); + vst1q_f32(tmpptr + 4, vld1q_f32(img0 + 4)); + img0 += size; + tmpptr += 8; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i; + + for (int k = 0; k < maxk; k++) + { + vst1q_f32(tmpptr, vld1q_f32(img0)); + img0 += size; + tmpptr += 4; + } + } + } + + remain_size_start += nn_size << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i; + + for (int k = 0; k < maxk; k++) + { + tmpptr[0] = img0[0]; + img0 += size; + tmpptr += 1; + } + } + } + } +#else // __ARM_NEON + tmp.create(maxk, inch, size, 4u, 1, opt.workspace_allocator); + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < size; i++) + { + float* tmpptr = tmp.channel(i); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i; + for (int k = 0; k < maxk; k++) + { + tmpptr[0] = img0[0]; + img0 += size; + tmpptr += 1; + } + } + } + } +#endif // __ARM_NEON + +#if __ARM_NEON int nn_outch = 0; int remain_outch_start = 0; -#if __ARM_NEON && __aarch64__ +#if __aarch64__ nn_outch = outch >> 3; remain_outch_start = nn_outch << 3; + #pragma omp parallel for num_threads(opt.num_threads) for (int pp = 0; pp < nn_outch; pp++) { int p = pp * 8; - const float* k0 = kernel + (p + 0) * inch * kernel_size; - const float* k1 = kernel + (p + 1) * inch * kernel_size; - const float* k2 = kernel + (p + 2) * inch * kernel_size; - const float* k3 = kernel + (p + 3) * inch * kernel_size; - const float* k4 = kernel + (p + 4) * inch * kernel_size; - const float* k5 = kernel + (p + 5) * inch * kernel_size; - const float* k6 = kernel + (p + 6) * inch * kernel_size; - const float* k7 = kernel + (p + 7) * inch * kernel_size; + float* outptr0 = top_blob.channel(p); + float* outptr1 = top_blob.channel(p + 1); + float* outptr2 = top_blob.channel(p + 2); + float* outptr3 = top_blob.channel(p + 3); + float* outptr4 = top_blob.channel(p + 4); + float* outptr5 = top_blob.channel(p + 5); + float* outptr6 = top_blob.channel(p + 6); + float* outptr7 = top_blob.channel(p + 7); - float* ktmp = kernel_tm.channel(p / 8); + const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p : zeros; - for (int q = 0; q < inch * kernel_size; q++) + int i = 0; + for (; i + 7 < size; i += 8) + { + const float* tmpptr = tmp.channel(i / 8); + const float* kptr = kernel.channel(p / 8); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%20] \n" + "dup v16.4s, v0.s[0] \n" + "dup v17.4s, v0.s[0] \n" + "dup v18.4s, v0.s[1] \n" + "dup v19.4s, v0.s[1] \n" + "dup v20.4s, v0.s[2] \n" + "dup v21.4s, v0.s[2] \n" + "dup v22.4s, v0.s[3] \n" + "dup v23.4s, v0.s[3] \n" + "dup v24.4s, v1.s[0] \n" + "dup v25.4s, v1.s[0] \n" + "dup v26.4s, v1.s[1] \n" + "dup v27.4s, v1.s[1] \n" + "dup v28.4s, v1.s[2] \n" + "dup v29.4s, v1.s[2] \n" + "dup v30.4s, v1.s[3] \n" + "dup v31.4s, v1.s[3] \n" + + // inch loop + "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n" + + "prfm pldl1keep, [%8, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64 \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v18.4s, v8.4s, v0.s[1] \n" + "fmla v20.4s, v8.4s, v0.s[2] \n" + "fmla v22.4s, v8.4s, v0.s[3] \n" + + "fmla v17.4s, v9.4s, v0.s[0] \n" + "fmla v19.4s, v9.4s, v0.s[1] \n" + "fmla v21.4s, v9.4s, v0.s[2] \n" + "fmla v23.4s, v9.4s, v0.s[3] \n" + + "fmla v24.4s, v8.4s, v1.s[0] \n" + "fmla v26.4s, v8.4s, v1.s[1] \n" + "fmla v28.4s, v8.4s, v1.s[2] \n" + "fmla v30.4s, v8.4s, v1.s[3] \n" + + "fmla v25.4s, v9.4s, v1.s[0] \n" + "fmla v27.4s, v9.4s, v1.s[1] \n" + "fmla v29.4s, v9.4s, v1.s[2] \n" + "fmla v31.4s, v9.4s, v1.s[3] \n" + + "prfm pldl1keep, [%8, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%8], #64 \n" + + "fmla v16.4s, v10.4s, v2.s[0] \n" + "fmla v18.4s, v10.4s, v2.s[1] \n" + "fmla v20.4s, v10.4s, v2.s[2] \n" + "fmla v22.4s, v10.4s, v2.s[3] \n" + + "fmla v17.4s, v11.4s, v2.s[0] \n" + "fmla v19.4s, v11.4s, v2.s[1] \n" + "fmla v21.4s, v11.4s, v2.s[2] \n" + "fmla v23.4s, v11.4s, v2.s[3] \n" + + "fmla v24.4s, v10.4s, v3.s[0] \n" + "fmla v26.4s, v10.4s, v3.s[1] \n" + "fmla v28.4s, v10.4s, v3.s[2] \n" + "fmla v30.4s, v10.4s, v3.s[3] \n" + + "fmla v25.4s, v11.4s, v3.s[0] \n" + "fmla v27.4s, v11.4s, v3.s[1] \n" + "fmla v29.4s, v11.4s, v3.s[2] \n" + "fmla v31.4s, v11.4s, v3.s[3] \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" + + "fmla v16.4s, v12.4s, v4.s[0] \n" + "fmla v18.4s, v12.4s, v4.s[1] \n" + "fmla v20.4s, v12.4s, v4.s[2] \n" + "fmla v22.4s, v12.4s, v4.s[3] \n" + + "fmla v17.4s, v13.4s, v4.s[0] \n" + "fmla v19.4s, v13.4s, v4.s[1] \n" + "fmla v21.4s, v13.4s, v4.s[2] \n" + "fmla v23.4s, v13.4s, v4.s[3] \n" + + "fmla v24.4s, v12.4s, v5.s[0] \n" + "fmla v26.4s, v12.4s, v5.s[1] \n" + "fmla v28.4s, v12.4s, v5.s[2] \n" + "fmla v30.4s, v12.4s, v5.s[3] \n" + + "fmla v25.4s, v13.4s, v5.s[0] \n" + "fmla v27.4s, v13.4s, v5.s[1] \n" + "fmla v29.4s, v13.4s, v5.s[2] \n" + "fmla v31.4s, v13.4s, v5.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v16.4s, v14.4s, v6.s[0] \n" + "fmla v18.4s, v14.4s, v6.s[1] \n" + "fmla v20.4s, v14.4s, v6.s[2] \n" + "fmla v22.4s, v14.4s, v6.s[3] \n" + + "fmla v17.4s, v15.4s, v6.s[0] \n" + "fmla v19.4s, v15.4s, v6.s[1] \n" + "fmla v21.4s, v15.4s, v6.s[2] \n" + "fmla v23.4s, v15.4s, v6.s[3] \n" + + "fmla v24.4s, v14.4s, v7.s[0] \n" + "fmla v26.4s, v14.4s, v7.s[1] \n" + "fmla v28.4s, v14.4s, v7.s[2] \n" + "fmla v30.4s, v14.4s, v7.s[3] \n" + + "fmla v25.4s, v15.4s, v7.s[0] \n" + "fmla v27.4s, v15.4s, v7.s[1] \n" + "fmla v29.4s, v15.4s, v7.s[2] \n" + "fmla v31.4s, v15.4s, v7.s[3] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and w4, %w21, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%8, #256] \n" + "ld1 {v8.4s, v9.4s}, [%8], #32 \n" + + "prfm pldl1keep, [%9, #256] \n" + "ld1 {v0.4s, v1.4s}, [%9], #32 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v18.4s, v8.4s, v0.s[1] \n" + "fmla v20.4s, v8.4s, v0.s[2] \n" + "fmla v22.4s, v8.4s, v0.s[3] \n" + + "fmla v17.4s, v9.4s, v0.s[0] \n" + "fmla v19.4s, v9.4s, v0.s[1] \n" + "fmla v21.4s, v9.4s, v0.s[2] \n" + "fmla v23.4s, v9.4s, v0.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v24.4s, v8.4s, v1.s[0] \n" + "fmla v26.4s, v8.4s, v1.s[1] \n" + "fmla v28.4s, v8.4s, v1.s[2] \n" + "fmla v30.4s, v8.4s, v1.s[3] \n" + + "fmla v25.4s, v9.4s, v1.s[0] \n" + "fmla v27.4s, v9.4s, v1.s[1] \n" + "fmla v29.4s, v9.4s, v1.s[2] \n" + "fmla v31.4s, v9.4s, v1.s[3] \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v16.4s, v17.4s}, [%0], #32 \n" + "st1 {v18.4s, v19.4s}, [%1], #32 \n" + "st1 {v20.4s, v21.4s}, [%2], #32 \n" + "st1 {v22.4s, v23.4s}, [%3], #32 \n" + "st1 {v24.4s, v25.4s}, [%4], #32 \n" + "st1 {v26.4s, v27.4s}, [%5], #32 \n" + "st1 {v28.4s, v29.4s}, [%6], #32 \n" + "st1 {v30.4s, v31.4s}, [%7], #32 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(outptr4), // %4 + "=r"(outptr5), // %5 + "=r"(outptr6), // %6 + "=r"(outptr7), // %7 + "=r"(tmpptr), // %8 + "=r"(kptr) // %9 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(outptr4), + "5"(outptr5), + "6"(outptr6), + "7"(outptr7), + "8"(tmpptr), + "9"(kptr), + "r"(biasptr), // %20 + "r"(nn) // %21 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + const float* kptr = kernel.channel(p / 8); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%20] \n" + "dup v16.4s, v0.s[0] \n" + "dup v17.4s, v0.s[1] \n" + "dup v18.4s, v0.s[2] \n" + "dup v19.4s, v0.s[3] \n" + "dup v20.4s, v1.s[0] \n" + "dup v21.4s, v1.s[1] \n" + "dup v22.4s, v1.s[2] \n" + "dup v23.4s, v1.s[3] \n" + + // inch loop + "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n" + + "prfm pldl1keep, [%8, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%8], #64 \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v0.s[1] \n" + "fmla v18.4s, v8.4s, v0.s[2] \n" + "fmla v19.4s, v8.4s, v0.s[3] \n" + "fmla v20.4s, v8.4s, v1.s[0] \n" + "fmla v21.4s, v8.4s, v1.s[1] \n" + "fmla v22.4s, v8.4s, v1.s[2] \n" + "fmla v23.4s, v8.4s, v1.s[3] \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" + + "fmla v16.4s, v9.4s, v2.s[0] \n" + "fmla v17.4s, v9.4s, v2.s[1] \n" + "fmla v18.4s, v9.4s, v2.s[2] \n" + "fmla v19.4s, v9.4s, v2.s[3] \n" + "fmla v20.4s, v9.4s, v3.s[0] \n" + "fmla v21.4s, v9.4s, v3.s[1] \n" + "fmla v22.4s, v9.4s, v3.s[2] \n" + "fmla v23.4s, v9.4s, v3.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v16.4s, v10.4s, v4.s[0] \n" + "fmla v17.4s, v10.4s, v4.s[1] \n" + "fmla v18.4s, v10.4s, v4.s[2] \n" + "fmla v19.4s, v10.4s, v4.s[3] \n" + "fmla v20.4s, v10.4s, v5.s[0] \n" + "fmla v21.4s, v10.4s, v5.s[1] \n" + "fmla v22.4s, v10.4s, v5.s[2] \n" + "fmla v23.4s, v10.4s, v5.s[3] \n" + + "fmla v16.4s, v11.4s, v6.s[0] \n" + "fmla v17.4s, v11.4s, v6.s[1] \n" + "fmla v18.4s, v11.4s, v6.s[2] \n" + "fmla v19.4s, v11.4s, v6.s[3] \n" + "fmla v20.4s, v11.4s, v7.s[0] \n" + "fmla v21.4s, v11.4s, v7.s[1] \n" + "fmla v22.4s, v11.4s, v7.s[2] \n" + "fmla v23.4s, v11.4s, v7.s[3] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and w4, %w21, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%8, #128] \n" + "ld1 {v8.4s}, [%8], #16 \n" + + "prfm pldl1keep, [%9, #256] \n" + "ld1 {v0.4s, v1.4s}, [%9], #32 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v0.s[1] \n" + "fmla v18.4s, v8.4s, v0.s[2] \n" + "fmla v19.4s, v8.4s, v0.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v20.4s, v8.4s, v1.s[0] \n" + "fmla v21.4s, v8.4s, v1.s[1] \n" + "fmla v22.4s, v8.4s, v1.s[2] \n" + "fmla v23.4s, v8.4s, v1.s[3] \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v16.4s}, [%0], #16 \n" + "st1 {v17.4s}, [%1], #16 \n" + "st1 {v18.4s}, [%2], #16 \n" + "st1 {v19.4s}, [%3], #16 \n" + "st1 {v20.4s}, [%4], #16 \n" + "st1 {v21.4s}, [%5], #16 \n" + "st1 {v22.4s}, [%6], #16 \n" + "st1 {v23.4s}, [%7], #16 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(outptr4), // %4 + "=r"(outptr5), // %5 + "=r"(outptr6), // %6 + "=r"(outptr7), // %7 + "=r"(tmpptr), // %8 + "=r"(kptr) // %9 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(outptr4), + "5"(outptr5), + "6"(outptr6), + "7"(outptr7), + "8"(tmpptr), + "9"(kptr), + "r"(biasptr), // %20 + "r"(nn) // %21 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + } + for (; i < size; i++) { - ktmp[0] = k0[0]; - ktmp[1] = k1[0]; - ktmp[2] = k2[0]; - ktmp[3] = k3[0]; - ktmp[4] = k4[0]; - ktmp[5] = k5[0]; - ktmp[6] = k6[0]; - ktmp[7] = k7[0]; - ktmp += 8; - - k0 += 1; - k1 += 1; - k2 += 1; - k3 += 1; - k4 += 1; - k5 += 1; - k6 += 1; - k7 += 1; + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); + const float* kptr = kernel.channel(p / 8); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v24.4s, v25.4s}, [%20] \n" + + // inch loop + "lsr w4, %w21, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "eor v16.16b, v16.16b, v16.16b \n" + "eor v17.16b, v17.16b, v17.16b \n" + "eor v18.16b, v18.16b, v18.16b \n" + "eor v19.16b, v19.16b, v19.16b \n" + "eor v20.16b, v20.16b, v20.16b \n" + "eor v21.16b, v21.16b, v21.16b \n" + "eor v22.16b, v22.16b, v22.16b \n" + "eor v23.16b, v23.16b, v23.16b \n" + + "0: \n" + + "prfm pldl1keep, [%8, #128] \n" + "ld1 {v8.4s}, [%8], #16 \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%9], #64 \n" + + "fmla v16.4s, v0.4s, v8.s[0] \n" + "fmla v17.4s, v1.4s, v8.s[0] \n" + "fmla v18.4s, v2.4s, v8.s[1] \n" + "fmla v19.4s, v3.4s, v8.s[1] \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%9], #64 \n" + + "subs w4, w4, #1 \n" + + "fmla v20.4s, v4.4s, v8.s[2] \n" + "fmla v21.4s, v5.4s, v8.s[2] \n" + "fmla v22.4s, v6.4s, v8.s[3] \n" + "fmla v23.4s, v7.4s, v8.s[3] \n" + + "bne 0b \n" + + "fadd v16.4s, v16.4s, v18.4s \n" + "fadd v17.4s, v17.4s, v19.4s \n" + "fadd v20.4s, v20.4s, v22.4s \n" + "fadd v21.4s, v21.4s, v23.4s \n" + "fadd v16.4s, v16.4s, v20.4s \n" + "fadd v17.4s, v17.4s, v21.4s \n" + "fadd v24.4s, v24.4s, v16.4s \n" + "fadd v25.4s, v25.4s, v17.4s \n" + + "1: \n" + + // remain loop + "and w4, %w21, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%8, #32] \n" + "ld1r {v8.4s}, [%8], #4 \n" + + "prfm pldl1keep, [%9, #256] \n" + "ld1 {v0.4s, v1.4s}, [%9], #32 \n" + + "subs w4, w4, #1 \n" + + "fmla v24.4s, v8.4s, v0.4s \n" + "fmla v25.4s, v8.4s, v1.4s \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v24.s}[0],[%0], #4 \n" + "st1 {v24.s}[1],[%1], #4 \n" + "st1 {v24.s}[2],[%2], #4 \n" + "st1 {v24.s}[3],[%3], #4 \n" + "st1 {v25.s}[0],[%4], #4 \n" + "st1 {v25.s}[1],[%5], #4 \n" + "st1 {v25.s}[2],[%6], #4 \n" + "st1 {v25.s}[3],[%7], #4 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(outptr4), // %4 + "=r"(outptr5), // %5 + "=r"(outptr6), // %6 + "=r"(outptr7), // %7 + "=r"(tmpptr), // %8 + "=r"(kptr) // %9 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(outptr4), + "5"(outptr5), + "6"(outptr6), + "7"(outptr7), + "8"(tmpptr), + "9"(kptr), + "r"(biasptr), // %20 + "r"(nn) // %21 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25"); } } -#endif +#endif // __aarch64__ nn_outch = (outch - remain_outch_start) >> 2; + #pragma omp parallel for num_threads(opt.num_threads) for (int pp = 0; pp < nn_outch; pp++) { int p = remain_outch_start + pp * 4; - const float* k0 = kernel + (p + 0) * inch * kernel_size; - const float* k1 = kernel + (p + 1) * inch * kernel_size; - const float* k2 = kernel + (p + 2) * inch * kernel_size; - const float* k3 = kernel + (p + 3) * inch * kernel_size; + float* outptr0 = top_blob.channel(p); + float* outptr1 = top_blob.channel(p + 1); + float* outptr2 = top_blob.channel(p + 2); + float* outptr3 = top_blob.channel(p + 3); + + const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p : zeros; + + int i = 0; + for (; i + 7 < size; i += 8) + { + const float* tmpptr = tmp.channel(i / 8); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); +#else + const float* kptr = kernel.channel(p / 4); +#endif + + int nn = inch * maxk; // inch always > 0 -#if __ARM_NEON && __aarch64__ - float* ktmp = kernel_tm.channel(p / 8 + (p % 8) / 4); +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%12] \n" + "dup v8.4s, v0.s[0] \n" + "dup v9.4s, v0.s[0] \n" + "dup v10.4s, v0.s[1] \n" + "dup v11.4s, v0.s[1] \n" + "dup v12.4s, v0.s[2] \n" + "dup v13.4s, v0.s[2] \n" + "dup v14.4s, v0.s[3] \n" + "dup v15.4s, v0.s[3] \n" + + // inch loop + "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" + + "prfm pldl1keep, [%5, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v10.4s, v4.4s, v0.s[1] \n" + "fmla v12.4s, v4.4s, v0.s[2] \n" + "fmla v14.4s, v4.4s, v0.s[3] \n" + + "fmla v9.4s, v5.4s, v0.s[0] \n" + "fmla v11.4s, v5.4s, v0.s[1] \n" + "fmla v13.4s, v5.4s, v0.s[2] \n" + "fmla v15.4s, v5.4s, v0.s[3] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%4], #64 \n" + + "fmla v8.4s, v6.4s, v1.s[0] \n" + "fmla v10.4s, v6.4s, v1.s[1] \n" + "fmla v12.4s, v6.4s, v1.s[2] \n" + "fmla v14.4s, v6.4s, v1.s[3] \n" + + "fmla v9.4s, v7.4s, v1.s[0] \n" + "fmla v11.4s, v7.4s, v1.s[1] \n" + "fmla v13.4s, v7.4s, v1.s[2] \n" + "fmla v15.4s, v7.4s, v1.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v16.4s, v2.s[0] \n" + "fmla v10.4s, v16.4s, v2.s[1] \n" + "fmla v12.4s, v16.4s, v2.s[2] \n" + "fmla v14.4s, v16.4s, v2.s[3] \n" + + "fmla v9.4s, v17.4s, v2.s[0] \n" + "fmla v11.4s, v17.4s, v2.s[1] \n" + "fmla v13.4s, v17.4s, v2.s[2] \n" + "fmla v15.4s, v17.4s, v2.s[3] \n" + + "fmla v8.4s, v18.4s, v3.s[0] \n" + "fmla v10.4s, v18.4s, v3.s[1] \n" + "fmla v12.4s, v18.4s, v3.s[2] \n" + "fmla v14.4s, v18.4s, v3.s[3] \n" + + "fmla v9.4s, v19.4s, v3.s[0] \n" + "fmla v11.4s, v19.4s, v3.s[1] \n" + "fmla v13.4s, v19.4s, v3.s[2] \n" + "fmla v15.4s, v19.4s, v3.s[3] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and w4, %w13, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v4.4s, v5.4s}, [%4], #32 \n" + + "prfm pldl1keep, [%5, #128] \n" + "ld1 {v0.4s}, [%5], #16 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v10.4s, v4.4s, v0.s[1] \n" + "fmla v12.4s, v4.4s, v0.s[2] \n" + "fmla v14.4s, v4.4s, v0.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v9.4s, v5.4s, v0.s[0] \n" + "fmla v11.4s, v5.4s, v0.s[1] \n" + "fmla v13.4s, v5.4s, v0.s[2] \n" + "fmla v15.4s, v5.4s, v0.s[3] \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v8.4s, v9.4s}, [%0], #32 \n" + "st1 {v10.4s, v11.4s}, [%1], #32 \n" + "st1 {v12.4s, v13.4s}, [%2], #32 \n" + "st1 {v14.4s, v15.4s}, [%3], #32 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); +#else // __aarch64__ + asm volatile( + "vld1.f32 {d0-d1}, [%12] \n" + "vdup.f32 q8, d0[0] \n" + "vdup.f32 q9, d0[0] \n" + "vdup.f32 q10, d0[1] \n" + "vdup.f32 q11, d0[1] \n" + "vdup.f32 q12, d1[0] \n" + "vdup.f32 q13, d1[0] \n" + "vdup.f32 q14, d1[1] \n" + "vdup.f32 q15, d1[1] \n" + + // inch loop + "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 + "cmp r4, #0 \n" + "beq 1f \n" + + "0: \n" + + "pld [%4, #512] \n" + "vldm %4!, {d8-d15} \n" + // "vld1.f32 {d8-d11}, [%4 :128]! \n" + // "vld1.f32 {d12-d15}, [%4 :128]! \n" + + "pld [%5, #512] \n" + "vldm %5!, {d0-d7} \n" + // "vld1.f32 {d0-d3}, [%5 :128]! \n" + // "vld1.f32 {d4-d7}, [%5 :128]! \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q10, q4, d0[1] \n" + "vmla.f32 q12, q4, d1[0] \n" + "vmla.f32 q14, q4, d1[1] \n" + + "vmla.f32 q9, q5, d0[0] \n" + "vmla.f32 q11, q5, d0[1] \n" + "vmla.f32 q13, q5, d1[0] \n" + "vmla.f32 q15, q5, d1[1] \n" + + "vmla.f32 q8, q6, d2[0] \n" + "vmla.f32 q10, q6, d2[1] \n" + "vmla.f32 q12, q6, d3[0] \n" + "vmla.f32 q14, q6, d3[1] \n" + + "vmla.f32 q9, q7, d2[0] \n" + "vmla.f32 q11, q7, d2[1] \n" + "vmla.f32 q13, q7, d3[0] \n" + "vmla.f32 q15, q7, d3[1] \n" + + "pld [%4, #512] \n" + "vldm %4!, {d8-d15} \n" + // "vld1.f32 {d8-d11}, [%4 :128]! \n" + // "vld1.f32 {d12-d15}, [%4 :128]! \n" + + "vmla.f32 q8, q4, d4[0] \n" + "vmla.f32 q10, q4, d4[1] \n" + "vmla.f32 q12, q4, d5[0] \n" + "vmla.f32 q14, q4, d5[1] \n" + + "vmla.f32 q9, q5, d4[0] \n" + "vmla.f32 q11, q5, d4[1] \n" + "vmla.f32 q13, q5, d5[0] \n" + "vmla.f32 q15, q5, d5[1] \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q6, d6[0] \n" + "vmla.f32 q10, q6, d6[1] \n" + "vmla.f32 q12, q6, d7[0] \n" + "vmla.f32 q14, q6, d7[1] \n" + + "vmla.f32 q9, q7, d6[0] \n" + "vmla.f32 q11, q7, d6[1] \n" + "vmla.f32 q13, q7, d7[0] \n" + "vmla.f32 q15, q7, d7[1] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and r4, %13, #3 \n" // r4 = remain = inch & 3; + "cmp r4, #0 \n" + "beq 3f \n" + + "2: \n" + + "pld [%4, #256] \n" + "vld1.f32 {d8-d11}, [%4 :128]! \n" + + "pld [%5, #128] \n" + "vld1.f32 {d0-d1}, [%5 :128]! \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q10, q4, d0[1] \n" + "vmla.f32 q12, q4, d1[0] \n" + "vmla.f32 q14, q4, d1[1] \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q9, q5, d0[0] \n" + "vmla.f32 q11, q5, d0[1] \n" + "vmla.f32 q13, q5, d1[0] \n" + "vmla.f32 q15, q5, d1[1] \n" + + "bne 2b \n" + + "3: \n" + + "vst1.f32 {d16-d19}, [%0 :128]! \n" + "vst1.f32 {d20-d23}, [%1 :128]! \n" + "vst1.f32 {d24-d27}, [%2 :128]! \n" + "vst1.f32 {d28-d31}, [%3 :128]! \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); +#endif // __aarch64__ + } + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); #else - float* ktmp = kernel_tm.channel(p / 4); -#endif // __ARM_NEON && __aarch64__ + const float* kptr = kernel.channel(p / 4); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%12] \n" + "dup v8.4s, v0.s[0] \n" + "dup v9.4s, v0.s[1] \n" + "dup v10.4s, v0.s[2] \n" + "dup v11.4s, v0.s[3] \n" + + // inch loop + "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" + + "prfm pldl1keep, [%5, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" + + "fmla v8.4s, v5.4s, v1.s[0] \n" + "fmla v9.4s, v5.4s, v1.s[1] \n" + "fmla v10.4s, v5.4s, v1.s[2] \n" + "fmla v11.4s, v5.4s, v1.s[3] \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v6.4s, v2.s[0] \n" + "fmla v9.4s, v6.4s, v2.s[1] \n" + "fmla v10.4s, v6.4s, v2.s[2] \n" + "fmla v11.4s, v6.4s, v2.s[3] \n" + + "fmla v8.4s, v7.4s, v3.s[0] \n" + "fmla v9.4s, v7.4s, v3.s[1] \n" + "fmla v10.4s, v7.4s, v3.s[2] \n" + "fmla v11.4s, v7.4s, v3.s[3] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and w4, %w13, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%4, #128] \n" + "ld1 {v4.4s}, [%4], #16 \n" + + "prfm pldl1keep, [%5, #128] \n" + "ld1 {v0.4s}, [%5], #16 \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" - for (int q = 0; q < inch * kernel_size; q++) + "bne 2b \n" + + "3: \n" + + "st1 {v8.4s}, [%0], #16 \n" + "st1 {v9.4s}, [%1], #16 \n" + "st1 {v10.4s}, [%2], #16 \n" + "st1 {v11.4s}, [%3], #16 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"); +#else // __aarch64__ + asm volatile( + "vld1.f32 {d0-d1}, [%12] \n" + "vdup.f32 q8, d0[0] \n" + "vdup.f32 q9, d0[1] \n" + "vdup.f32 q10, d1[0] \n" + "vdup.f32 q11, d1[1] \n" + + // inch loop + "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 + "cmp r4, #0 \n" + "beq 1f \n" + + "0: \n" + + "pld [%4, #512] \n" + "vldm %4!, {d8-d15} \n" + // "vld1.f32 {d8-d11}, [%4 :128]! \n" + // "vld1.f32 {d12-d15}, [%4 :128]! \n" + + "pld [%5, #512] \n" + "vldm %5!, {d0-d7} \n" + // "vld1.f32 {d0-d3}, [%5 :128]! \n" + // "vld1.f32 {d4-d7}, [%5 :128]! \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d0[1] \n" + "vmla.f32 q10, q4, d1[0] \n" + "vmla.f32 q11, q4, d1[1] \n" + + "vmla.f32 q8, q5, d2[0] \n" + "vmla.f32 q9, q5, d2[1] \n" + "vmla.f32 q10, q5, d3[0] \n" + "vmla.f32 q11, q5, d3[1] \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q6, d4[0] \n" + "vmla.f32 q9, q6, d4[1] \n" + "vmla.f32 q10, q6, d5[0] \n" + "vmla.f32 q11, q6, d5[1] \n" + + "vmla.f32 q8, q7, d6[0] \n" + "vmla.f32 q9, q7, d6[1] \n" + "vmla.f32 q10, q7, d7[0] \n" + "vmla.f32 q11, q7, d7[1] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and r4, %13, #3 \n" // r4 = remain = inch & 3; + "cmp r4, #0 \n" + "beq 3f \n" + + "2: \n" + + "pld [%4, #128] \n" + "vld1.f32 {d8-d9}, [%4 :128]! \n" + + "pld [%5, #128] \n" + "vld1.f32 {d0-d1}, [%5 :128]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d0[1] \n" + "vmla.f32 q10, q4, d1[0] \n" + "vmla.f32 q11, q4, d1[1] \n" + + "bne 2b \n" + + "3: \n" + + "vst1.f32 {d16-d17}, [%0 :128]! \n" + "vst1.f32 {d18-d19}, [%1 :128]! \n" + "vst1.f32 {d20-d21}, [%2 :128]! \n" + "vst1.f32 {d22-d23}, [%3 :128]! \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); +#endif // __aarch64__ + } + for (; i < size; i++) { - ktmp[0] = k0[0]; - ktmp[1] = k1[0]; - ktmp[2] = k2[0]; - ktmp[3] = k3[0]; - ktmp += 4; - - k0 += 1; - k1 += 1; - k2 += 1; - k3 += 1; + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4); +#else + const float* kptr = kernel.channel(p / 4); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v12.4s}, [%12] \n" + + // inch loop + "lsr w4, %w13, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "eor v8.16b, v8.16b, v8.16b \n" + "eor v9.16b, v9.16b, v9.16b \n" + "eor v10.16b, v10.16b, v10.16b \n" + "eor v11.16b, v11.16b, v11.16b \n" + + "0: \n" + + "prfm pldl1keep, [%4, #128] \n" + "ld1 {v4.4s}, [%4], #16 \n" + + "prfm pldl1keep, [%5, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%5], #64 \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v0.4s, v4.s[0] \n" + "fmla v9.4s, v1.4s, v4.s[1] \n" + "fmla v10.4s, v2.4s, v4.s[2] \n" + "fmla v11.4s, v3.4s, v4.s[3] \n" + + "bne 0b \n" + + "fadd v8.4s, v8.4s, v9.4s \n" + "fadd v10.4s, v10.4s, v11.4s \n" + "fadd v8.4s, v8.4s, v10.4s \n" + "fadd v12.4s, v12.4s, v8.4s \n" + + "1: \n" + + // remain loop + "and w4, %w13, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%4, #32] \n" + "ld1r {v4.4s}, [%4], #4 \n" + + "prfm pldl1keep, [%5, #128] \n" + "ld1 {v0.4s}, [%5], #16 \n" + + "subs w4, w4, #1 \n" + + "fmla v12.4s, v4.4s, v0.4s \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v12.s}[0], [%0], #4 \n" + "st1 {v12.s}[1], [%1], #4 \n" + "st1 {v12.s}[2], [%2], #4 \n" + "st1 {v12.s}[3], [%3], #4 \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v8", "v9", "v10", "v11", "v12"); +#else // __aarch64__ + asm volatile( + "vld1.f32 {d24-d25}, [%12] \n" + + // inch loop + "lsr r4, %13, #2 \n" // r4 = nn = inch >> 2 + "cmp r4, #0 \n" + "beq 1f \n" + + "veor q8, q8, q8 \n" + "veor q9, q9, q9 \n" + "veor q10, q10, q10 \n" + "veor q11, q11, q11 \n" + + "0: \n" + + "pld [%4, #128] \n" + "vld1.f32 {d8-d9}, [%4 :128]! \n" + + "pld [%5, #512] \n" + "vldm %5!, {d0-d7} \n" + // "vld1.f32 {d0-d3}, [%5 :128]! \n" + // "vld1.f32 {d4-d7}, [%5 :128]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q0, d8[0] \n" + "vmla.f32 q9, q1, d8[1] \n" + "vmla.f32 q10, q2, d9[0] \n" + "vmla.f32 q11, q3, d9[1] \n" + + "bne 0b \n" + + "vadd.f32 q8, q8, q9 \n" + "vadd.f32 q10, q10, q11 \n" + "vadd.f32 q8, q8, q10 \n" + "vadd.f32 q12, q12, q8 \n" + + "1: \n" + + // remain loop + "and r4, %13, #3 \n" // r4 = remain = inch & 3; + "cmp r4, #0 \n" + "beq 3f \n" + + "2: \n" + + "pld [%4, #32] \n" + "vld1.f32 {d8[],d9[]}, [%4]! \n" + + "pld [%5, #128] \n" + "vld1.f32 {d0-d1}, [%5 :128]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q12, q4, q0 \n" + + "bne 2b \n" + + "3: \n" + + "vst1.f32 {d24[0]}, [%0]! \n" + "vst1.f32 {d24[1]}, [%1]! \n" + "vst1.f32 {d25[0]}, [%2]! \n" + "vst1.f32 {d25[1]}, [%3]! \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(biasptr), // %12 + "r"(nn) // %13 + : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11", "q12"); +#endif // __aarch64__ } } remain_outch_start += nn_outch << 2; + #pragma omp parallel for num_threads(opt.num_threads) for (int p = remain_outch_start; p < outch; p++) { - const float* k0 = kernel + (p + 0) * inch * kernel_size; + float* outptr0 = top_blob.channel(p); -#if __ARM_NEON && __aarch64__ - float* ktmp = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - float* ktmp = kernel_tm.channel(p / 4 + p % 4); -#endif // __ARM_NEON && __aarch64__ + const float bias0 = bias ? bias[p] : 0.f; - for (int q = 0; q < inch * kernel_size; q++) + int i = 0; + for (; i + 7 < size; i += 8) { - ktmp[0] = k0[0]; - ktmp++; - k0++; - } - } -} + const float* tmpptr = tmp.channel(i / 8); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const float* kptr = kernel.channel(p / 4 + p % 4); +#endif -static void conv_im2col_sgemm_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias, - const int kernel_w, const int kernel_h, const int stride_w, const int stride_h, const Option& opt) -{ - int w = bottom_blob.w; - int inch = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; + int nn = inch * maxk; // inch always > 0 - int outw = top_blob.w; - int outh = top_blob.h; - int outch = top_blob.c; +#if __aarch64__ + asm volatile( + "dup v8.4s, %w6 \n" + "dup v9.4s, %w6 \n" - const float* bias = _bias; + // inch loop + "lsr w4, %w7, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" - // im2col - Mat bottom_im2col(outw * outh, kernel_h * kernel_w * inch, elemsize, opt.workspace_allocator); - { - const int stride = kernel_h * kernel_w * outw * outh; - float* ret = (float*)bottom_im2col; + "0: \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < inch; p++) - { - const float* input = bottom_blob.channel(p); - int retID = stride * p; - for (int u = 0; u < kernel_h; u++) - { - for (int v = 0; v < kernel_w; v++) - { - for (int i = 0; i < outh; i++) - { - for (int j = 0; j < outw; j++) - { - int row = u + i * stride_h; - int col = v + j * stride_w; - int index = row * w + col; - ret[retID] = input[index]; - retID++; - } - } - } - } - } - } + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" - int kernel_size = kernel_w * kernel_h; - int out_size = outw * outh; + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v0.4s}, [%2], #16 \n" - // bottom_im2col memory packed 8 x 8 - Mat bottom_tm(8 * kernel_size, inch, out_size / 8 + out_size % 8, elemsize, opt.workspace_allocator); - { - int nn_size = out_size >> 3; - int remain_size_start = nn_size << 3; + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v5.4s, v0.s[0] \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int ii = 0; ii < nn_size; ii++) - { - int i = ii * 8; + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - const float* img0 = bottom_im2col.channel(0); - img0 += i; + "fmla v8.4s, v6.4s, v0.s[1] \n" + "fmla v9.4s, v7.4s, v0.s[1] \n" - float* tmpptr = bottom_tm.channel(i / 8); + "subs w4, w4, #1 \n" - for (int q = 0; q < inch * kernel_size; q++) - { -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #256] \n" - "ld1 {v0.4s, v1.4s}, [%0] \n" - "st1 {v0.4s, v1.4s}, [%1] \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "cc", "memory", "v0", "v1"); -#else - asm volatile( - "pld [%0, #256] \n" - "vld1.f32 {d0-d3}, [%0] \n" - "vst1.f32 {d0-d3}, [%1] \n" - : "=r"(img0), // %0 - "=r"(tmpptr) // %1 - : "0"(img0), - "1"(tmpptr) - : "memory", "q0", "q1"); -#endif // __aarch64__ -#else - tmpptr[0] = img0[0]; - tmpptr[1] = img0[1]; - tmpptr[2] = img0[2]; - tmpptr[3] = img0[3]; - tmpptr[4] = img0[4]; - tmpptr[5] = img0[5]; - tmpptr[6] = img0[6]; - tmpptr[7] = img0[7]; -#endif // __ARM_NEON - tmpptr += 8; - img0 += out_size; - } - } + "fmla v8.4s, v12.4s, v0.s[2] \n" + "fmla v9.4s, v13.4s, v0.s[2] \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_size_start; i < out_size; i++) - { - const float* img0 = bottom_im2col.channel(0); - img0 += i; + "fmla v8.4s, v14.4s, v0.s[3] \n" + "fmla v9.4s, v15.4s, v0.s[3] \n" - float* tmpptr = bottom_tm.channel(i / 8 + i % 8); + "bne 0b \n" - for (int q = 0; q < inch * kernel_size; q++) - { - tmpptr[0] = img0[0]; + "1: \n" - tmpptr += 1; - img0 += out_size; - } - } - } + // remain loop + "and w4, %w7, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" - // sgemm(int M, int N, int L, float* A, float* B, float* C) - { - //int M = outch; // outch - int N = outw * outh; // outsize or out stride - int L = kernel_w * kernel_h * inch; // ksize * inch + "2: \n" - int nn_outch = 0; - int remain_outch_start = 0; + "prfm pldl1keep, [%1, #256] \n" + "ld1 {v4.4s, v5.4s}, [%1], #32 \n" -#if __aarch64__ - nn_outch = outch >> 3; - remain_outch_start = nn_outch << 3; + "prfm pldl1keep, [%2, #32] \n" + "ld1r {v0.4s}, [%2], #4 \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int i = pp * 8; - - float* output0 = top_blob.channel(i); - float* output1 = top_blob.channel(i + 1); - float* output2 = top_blob.channel(i + 2); - float* output3 = top_blob.channel(i + 3); - float* output4 = top_blob.channel(i + 4); - float* output5 = top_blob.channel(i + 5); - float* output6 = top_blob.channel(i + 6); - float* output7 = top_blob.channel(i + 7); - - const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + i : zeros; - - int j = 0; - for (; j + 7 < N; j = j + 8) - { - const float* vb = bottom_tm.channel(j / 8); - const float* va = kernel_tm.channel(i / 8); -#if __ARM_NEON - asm volatile( - "ld1 {v0.4s, v1.4s}, [%21] \n" - "dup v16.4s, v0.s[0] \n" // sum0 - "dup v17.4s, v0.s[0] \n" - "dup v18.4s, v0.s[1] \n" // sum1 - "dup v19.4s, v0.s[1] \n" - "dup v20.4s, v0.s[2] \n" // sum2 - "dup v21.4s, v0.s[2] \n" - "dup v22.4s, v0.s[3] \n" // sum3 - "dup v23.4s, v0.s[3] \n" - "dup v24.4s, v1.s[0] \n" // sum4 - "dup v25.4s, v1.s[0] \n" - "dup v26.4s, v1.s[1] \n" // sum5 - "dup v27.4s, v1.s[1] \n" - "dup v28.4s, v1.s[2] \n" // sum6 - "dup v29.4s, v1.s[2] \n" - "dup v30.4s, v1.s[3] \n" // sum7 - "dup v31.4s, v1.s[3] \n" - - "lsr w4, %w20, #2 \n" // r4 = nn = L >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" // for (; k+3> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" // for (; k+3> 2 + "cmp r4, #0 \n" + "beq 1f \n" - output0[0] = sum0; - output1[0] = sum1; - output2[0] = sum2; - output3[0] = sum3; - output4[0] = sum4; - output5[0] = sum5; - output6[0] = sum6; - output7[0] = sum7; -#endif // __ARM_NEON - output0++; - output1++; - output2++; - output3++; - output4++; - output5++; - output6++; - output7++; - } - } -#endif // __aarch64__ + "0: \n" - nn_outch = (outch - remain_outch_start) >> 2; + "pld [%1, #512] \n" + "vldm %1!, {d8-d15} \n" + // "vld1.f32 {d8-d11}, [%1 :128]! \n" + // "vld1.f32 {d12-d15}, [%1 :128]! \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int i = remain_outch_start + pp * 4; + "pld [%2, #128] \n" + "vld1.f32 {d0-d1}, [%2 :128]! \n" - float* output0 = top_blob.channel(i); - float* output1 = top_blob.channel(i + 1); - float* output2 = top_blob.channel(i + 2); - float* output3 = top_blob.channel(i + 3); + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q5, d0[0] \n" - const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; - const float* biasptr = bias ? bias + i : zeros; + "pld [%1, #512] \n" + "vldm %1!, {d24-d31} \n" + // "vld1.f32 {d24-d27}, [%1 :128]! \n" + // "vld1.f32 {d28-d31}, [%1 :128]! \n" - int j = 0; - for (; j + 7 < N; j = j + 8) - { - const float* vb = bottom_tm.channel(j / 8); -#if __ARM_NEON && __aarch64__ - const float* va = kernel_tm.channel(i / 8 + (i % 8) / 4); -#else - const float* va = kernel_tm.channel(i / 4); -#endif // __ARM_NEON && __aarch64__ + "vmla.f32 q8, q6, d0[1] \n" + "vmla.f32 q9, q7, d0[1] \n" -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "ld1 {v0.4s}, [%13] \n" - "dup v16.4s, v0.s[0] \n" // sum0 - "dup v17.4s, v0.s[0] \n" - "dup v18.4s, v0.s[1] \n" // sum1 - "dup v19.4s, v0.s[1] \n" - "dup v20.4s, v0.s[2] \n" // sum2 - "dup v21.4s, v0.s[2] \n" - "dup v22.4s, v0.s[3] \n" // sum3 - "dup v23.4s, v0.s[3] \n" - - "lsr w4, %w12, #2 \n" // r4 = nn = L >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" // for (; k+3> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" // for(; nn != 0; nn--) - "pld [%5, #512] \n" - "vldm %5!, {d0-d7} \n" // kernel - "pld [%4, #512] \n" - "vldm %4!, {d8-d15} \n" // data - - "vmla.f32 q8, q4, d0[0] \n" // sum0 = (a00-a07) * k00 - "vmla.f32 q9, q5, d0[0] \n" - "vmla.f32 q10, q4, d0[1] \n" // sum1 = (a00-a07) * k10 - "vmla.f32 q11, q5, d0[1] \n" - "vmla.f32 q12, q4, d1[0] \n" // sum2 = (a00-a07) * k20 - "vmla.f32 q13, q5, d1[0] \n" - "vmla.f32 q14, q4, d1[1] \n" // sum3 = (a00-a07) * k30 - "vmla.f32 q15, q5, d1[1] \n" - - "vmla.f32 q8, q6, d2[0] \n" // sum0 += (a10-a17) * k01 - "vmla.f32 q9, q7, d2[0] \n" - "vmla.f32 q10, q6, d2[1] \n" // sum1 += (a10-a17) * k11 - "vmla.f32 q11, q7, d2[1] \n" - "vmla.f32 q12, q6, d3[0] \n" // sum2 += (a10-a17) * k21 - "vmla.f32 q13, q7, d3[0] \n" - "vmla.f32 q14, q6, d3[1] \n" // sum3 += (a10-a17) * k31 - "vmla.f32 q15, q7, d3[1] \n" - - "pld [%4, #512] \n" - "vldm %4!, {d8-d15} \n" // data - - "vmla.f32 q8, q4, d4[0] \n" // sum0 += (a20-a27) * k02 - "vmla.f32 q9, q5, d4[0] \n" - "vmla.f32 q10, q4, d4[1] \n" // sum1 += (a20-a27) * k12 - "vmla.f32 q11, q5, d4[1] \n" - "vmla.f32 q12, q4, d5[0] \n" // sum2 += (a20-a27) * k22 - "vmla.f32 q13, q5, d5[0] \n" - "vmla.f32 q14, q4, d5[1] \n" // sum3 += (a20-a27) * k32 - "vmla.f32 q15, q5, d5[1] \n" - - "vmla.f32 q8, q6, d6[0] \n" // sum0 += (a30-a37) * k03 - "vmla.f32 q9, q7, d6[0] \n" - "vmla.f32 q10, q6, d6[1] \n" // sum1 += (a30-a37) * k13 - "vmla.f32 q11, q7, d6[1] \n" - "vmla.f32 q12, q6, d7[0] \n" // sum2 += (a30-a37) * k23 - "vmla.f32 q13, q7, d7[0] \n" - "vmla.f32 q14, q6, d7[1] \n" // sum3 += (a30-a37) * k33 - "vmla.f32 q15, q7, d7[1] \n" - - "subs r4, r4, #1 \n" - "bne 0b \n" // end for - - "1: \n" - // remain loop - "and r4, %12, #3 \n" // r4 = remain = inch & 3 - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" // for(; remain != 0; remain--) - - "pld [%5, #128] \n" - "vld1.f32 {d0-d1}, [%5]! \n" - "pld [%4, #256] \n" - "vld1.f32 {d8-d11}, [%4]! \n" - - "vmla.f32 q8, q4, d0[0] \n" // sum0 += (a00-a70) * k00 - "vmla.f32 q9, q5, d0[0] \n" - "vmla.f32 q10, q4, d0[1] \n" // sum1 += (a00-a70) * k10 - "vmla.f32 q11, q5, d0[1] \n" - "vmla.f32 q12, q4, d1[0] \n" // sum2 += (a00-a70) * k20 - "vmla.f32 q13, q5, d1[0] \n" - "vmla.f32 q14, q4, d1[1] \n" // sum3 += (a00-a70) * k30 - "vmla.f32 q15, q5, d1[1] \n" - - "subs r4, r4, #1 \n" - "bne 2b \n" - - "3: \n" // store the result to memory - "vst1.f32 {d16-d19}, [%0] \n" - "vst1.f32 {d20-d23}, [%1] \n" - "vst1.f32 {d24-d27}, [%2] \n" - "vst1.f32 {d28-d31}, [%3] \n" - - : "=r"(output0), // %0 - "=r"(output1), // %1 - "=r"(output2), // %2 - "=r"(output3), // %3 - "=r"(vb), // %4 - "=r"(va) // %5 - : "0"(output0), - "1"(output1), - "2"(output2), - "3"(output3), - "4"(vb), - "5"(va), - "r"(L), // %12 - "r"(biasptr) // %13 - : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif // __aarch64__ -#else - float sum0[8] = {0}; - float sum1[8] = {0}; - float sum2[8] = {0}; - float sum3[8] = {0}; + "subs r4, r4, #1 \n" - int k = 0; - for (; k + 7 < L; k = k + 8) - { - for (int n = 0; n < 8; n++) - { - sum0[n] += va[0] * vb[n]; - sum1[n] += va[1] * vb[n]; - sum2[n] += va[2] * vb[n]; - sum3[n] += va[3] * vb[n]; - va += 4; - - sum0[n] += va[0] * vb[n + 8]; - sum1[n] += va[1] * vb[n + 8]; - sum2[n] += va[2] * vb[n + 8]; - sum3[n] += va[3] * vb[n + 8]; - va += 4; - - sum0[n] += va[0] * vb[n + 16]; - sum1[n] += va[1] * vb[n + 16]; - sum2[n] += va[2] * vb[n + 16]; - sum3[n] += va[3] * vb[n + 16]; - va += 4; - - sum0[n] += va[0] * vb[n + 24]; - sum1[n] += va[1] * vb[n + 24]; - sum2[n] += va[2] * vb[n + 24]; - sum3[n] += va[3] * vb[n + 24]; - va += 4; - - sum0[n] += va[0] * vb[n + 32]; - sum1[n] += va[1] * vb[n + 32]; - sum2[n] += va[2] * vb[n + 32]; - sum3[n] += va[3] * vb[n + 32]; - va += 4; - - sum0[n] += va[0] * vb[n + 40]; - sum1[n] += va[1] * vb[n + 40]; - sum2[n] += va[2] * vb[n + 40]; - sum3[n] += va[3] * vb[n + 40]; - va += 4; - - sum0[n] += va[0] * vb[n + 48]; - sum1[n] += va[1] * vb[n + 48]; - sum2[n] += va[2] * vb[n + 48]; - sum3[n] += va[3] * vb[n + 48]; - va += 4; - - sum0[n] += va[0] * vb[n + 56]; - sum1[n] += va[1] * vb[n + 56]; - sum2[n] += va[2] * vb[n + 56]; - sum3[n] += va[3] * vb[n + 56]; - va -= 28; - } + "vmla.f32 q8, q12, d1[0] \n" + "vmla.f32 q9, q13, d1[0] \n" - va += 32; - vb += 64; - } + "vmla.f32 q8, q14, d1[1] \n" + "vmla.f32 q9, q15, d1[1] \n" - for (; k < L; k++) - { - for (int n = 0; n < 8; n++) - { - sum0[n] += va[0] * vb[n]; - sum1[n] += va[1] * vb[n]; - sum2[n] += va[2] * vb[n]; - sum3[n] += va[3] * vb[n]; - } + "bne 0b \n" - va += 4; - vb += 8; - } + "1: \n" - for (int n = 0; n < 8; n++) - { - output0[n] = sum0[n] + biasptr[0]; - output1[n] = sum1[n] + biasptr[1]; - output2[n] = sum2[n] + biasptr[2]; - output3[n] = sum3[n] + biasptr[3]; - } -#endif // __ARM_NEON - output0 += 8; - output1 += 8; - output2 += 8; - output3 += 8; - } + // remain loop + "and r4, %7, #3 \n" // r4 = remain = inch & 3; + "cmp r4, #0 \n" + "beq 3f \n" - for (; j < N; j++) - { - float* vb = bottom_tm.channel(j / 8 + j % 8); -#if __ARM_NEON && __aarch64__ - const float* va = kernel_tm.channel(i / 8 + (i % 8) / 4); -#else - const float* va = kernel_tm.channel(i / 4); -#endif // __ARM_NEON && __aarch64__ + "2: \n" -#if __ARM_NEON -#if __aarch64__ - asm volatile( - "ld1 {v14.4s}, [%13] \n" // sum0_3 inital with bias - - "lsr w4, %w12, #2 \n" // r4 = nn = L >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "eor v16.16b, v16.16b, v16.16b \n" // sum0 - "eor v17.16b, v17.16b, v17.16b \n" // sum1 - "eor v18.16b, v18.16b, v18.16b \n" // sum2 - "eor v19.16b, v19.16b, v19.16b \n" // sum3 - - "0: \n" // for (; k+3> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "veor q8, q8, q8 \n" - "veor q9, q9, q9 \n" - "veor q10, q10, q10 \n" - "veor q11, q11, q11 \n" - - "0: \n" // for(; nn != 0; nn--) - "pld [%5, #512] \n" - "vldm %5!, {d0-d7} \n" // kernel - "pld [%4, #128] \n" - "vld1.f32 {d8-d9}, [%4]! \n" // data - - "vmla.f32 q8, q0, d8[0] \n" // (k00-k30) * a00 - "vmla.f32 q9, q1, d8[1] \n" // (k01-k31) * a01 - "vmla.f32 q10, q2, d9[0] \n" // (k02-k32) * a02 - "vmla.f32 q11, q3, d9[1] \n" // (k03-k33) * a03 - - "subs r4, r4, #1 \n" - "bne 0b \n" // end for - - "vadd.f32 q8, q8, q9 \n" - "vadd.f32 q10, q10, q11 \n" - "vadd.f32 q8, q8, q10 \n" - "vadd.f32 q12, q12, q8 \n" - - "1: \n" - // remain loop - "and r4, %12, #3 \n" // r4 = remain = inch & 3 - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" // for(; remain != 0; remain--) - "pld [%5, #128] \n" - "vld1.f32 {d0-d1}, [%5]! \n" - "pld [%4, #32] \n" - "vld1.f32 {d8[],d9[]}, [%4]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q12, q0, q4 \n" - "bne 2b \n" - - "3: \n" // store the result to memory - "vst1.f32 {d24[0]}, [%0] \n" - "vst1.f32 {d24[1]}, [%1] \n" - "vst1.f32 {d25[0]}, [%2] \n" - "vst1.f32 {d25[1]}, [%3] \n" - - : "=r"(output0), // %0 - "=r"(output1), // %1 - "=r"(output2), // %2 - "=r"(output3), // %3 - "=r"(vb), // %4 - "=r"(va) // %5 - : "0"(output0), - "1"(output1), - "2"(output2), - "3"(output3), - "4"(vb), - "5"(va), - "r"(L), // %12 - "r"(biasptr) // %13 - : "cc", "memory", "r4", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12"); + "pld [%1, #256] \n" + "vld1.f32 {d8-d11}, [%1 :128]! \n" + + "pld [%2, #32] \n" + "vld1.f32 {d0[],d1[]}, [%2]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q4, q0 \n" + "vmla.f32 q9, q5, q0 \n" + + "bne 2b \n" + + "3: \n" + + "vst1.f32 {d16-d19}, [%0 :128]! \n" + + : "=r"(outptr0), // %0 + "=r"(tmpptr), // %1 + "=r"(kptr) // %2 + : "0"(outptr0), + "1"(tmpptr), + "2"(kptr), + "r"(bias0), // %6 + "r"(nn) // %7 + : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8", "q9", "q12", "q13", "q14", "q15"); #endif // __aarch64__ + } + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); #else - float sum0 = biasptr[0]; - float sum1 = biasptr[1]; - float sum2 = biasptr[2]; - float sum3 = biasptr[3]; + const float* kptr = kernel.channel(p / 4 + p % 4); +#endif - for (int k = 0; k < L; k++) - { - sum0 += va[0] * vb[0]; - sum1 += va[1] * vb[0]; - sum2 += va[2] * vb[0]; - sum3 += va[3] * vb[0]; + int nn = inch * maxk; // inch always > 0 - va += 4; - vb += 1; - } +#if __aarch64__ + asm volatile( + "dup v8.4s, %w6 \n" - output0[0] = sum0; - output1[0] = sum1; - output2[0] = sum2; - output3[0] = sum3; -#endif // __ARM_NEON - output0++; - output1++; - output2++; - output3++; - } - } + // inch loop + "lsr w4, %w7, #2 \n" // w4 = nn = inch >> 2 + "cmp w4, #0 \n" + "beq 1f \n" - remain_outch_start += nn_outch << 2; + "0: \n" - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = remain_outch_start; i < outch; i++) + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v0.4s}, [%2], #16 \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v8.4s, v5.4s, v0.s[1] \n" + "fmla v8.4s, v6.4s, v0.s[2] \n" + "fmla v8.4s, v7.4s, v0.s[3] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and w4, %w7, #3 \n" // w4 = remain = inch & 3; + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n" + + "prfm pldl1keep, [%1, #128] \n" + "ld1 {v4.4s}, [%1], #16 \n" + + "prfm pldl1keep, [%2, #32] \n" + "ld1r {v0.4s}, [%2], #4 \n" + + "subs w4, w4, #1 \n" + + "fmla v8.4s, v4.4s, v0.4s \n" + + "bne 2b \n" + + "3: \n" + + "st1 {v8.4s}, [%0], #16 \n" + + : "=r"(outptr0), // %0 + "=r"(tmpptr), // %1 + "=r"(kptr) // %2 + : "0"(outptr0), + "1"(tmpptr), + "2"(kptr), + "r"(bias0), // %6 + "r"(nn) // %7 + : "cc", "memory", "x4", "v0", "v4", "v5", "v6", "v7", "v8"); +#else // __aarch64__ + asm volatile( + "vdup.f32 q8, %6 \n" + + // inch loop + "lsr r4, %7, #2 \n" // r4 = nn = inch >> 2 + "cmp r4, #0 \n" + "beq 1f \n" + + "0: \n" + + "pld [%1, #512] \n" + "vldm %1!, {d8-d15} \n" + // "vld1.f32 {d8-d11}, [%1 :128]! \n" + // "vld1.f32 {d12-d15}, [%1 :128]! \n" + + "pld [%2, #128] \n" + "vld1.f32 {d0-d1}, [%2]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q8, q5, d0[1] \n" + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q8, q7, d1[1] \n" + + "bne 0b \n" + + "1: \n" + + // remain loop + "and r4, %7, #3 \n" // r4 = remain = inch & 3; + "cmp r4, #0 \n" + "beq 3f \n" + + "2: \n" + + "pld [%1, #128] \n" + "vld1.f32 {d8-d9}, [%1 :128]! \n" + + "pld [%2, #32] \n" + "vld1.f32 {d0[],d1[]}, [%2]! \n" + + "subs r4, r4, #1 \n" + + "vmla.f32 q8, q4, q0 \n" + + "bne 2b \n" + + "3: \n" + + "vst1.f32 {d16-d17}, [%0 :128]! \n" + + : "=r"(outptr0), // %0 + "=r"(tmpptr), // %1 + "=r"(kptr) // %2 + : "0"(outptr0), + "1"(tmpptr), + "2"(kptr), + "r"(bias0), // %6 + "r"(nn) // %7 + : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8"); +#endif // __aarch64__ + } + for (; i < size; i++) { - float* output = top_blob.channel(i); + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + i % 4); +#if __aarch64__ + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const float* kptr = kernel.channel(p / 4 + p % 4); +#endif + + int nn = inch * maxk; // inch always > 0 - const float bias0 = bias ? bias[i] : 0.f; + float32x4_t _sum0 = vdupq_n_f32(0.f); - int j = 0; - for (; j + 7 < N; j = j + 8) + int q = 0; + for (; q + 3 < nn; q += 4) { - const float* vb = bottom_tm.channel(j / 8); -#if __ARM_NEON && __aarch64__ - const float* va = kernel_tm.channel(i / 8 + (i % 8) / 4 + i % 4); -#else - const float* va = kernel_tm.channel(i / 4 + i % 4); -#endif // __ARM_NEON && __aarch64__ + float32x4_t _p0 = vld1q_f32(tmpptr); + tmpptr += 4; + + float32x4_t _k0 = vld1q_f32(kptr); + kptr += 4; -#if __ARM_NEON #if __aarch64__ - asm volatile( - "dup v16.4s, %w7 \n" // sum0 - "dup v17.4s, %w7 \n" // sum0n - - "lsr w4, %w6, #2 \n" // r4 = nn = L >> 2 - "cmp w4, #0 \n" - "beq 1f \n" - - "0: \n" // for (; k+3> 2 - "cmp r4, #0 \n" - "beq 1f \n" - - "0: \n" - - "pld [%1, #512] \n" - "vldm %1!, {d8-d15} \n" - "pld [%2, #128] \n" - "vld1.f32 {d0-d1}, [%2]! \n" - - "vmla.f32 q8, q4, d0[0] \n" - "vmla.f32 q9, q5, d0[0] \n" - - "pld [%1, #512] \n" - "vldm %1!, {d24-d31} \n" - - "vmla.f32 q8, q6, d0[1] \n" - "vmla.f32 q9, q7, d0[1] \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q12, d1[0] \n" - "vmla.f32 q9, q13, d1[0] \n" - "vmla.f32 q8, q14, d1[1] \n" - "vmla.f32 q9, q15, d1[1] \n" - - "bne 0b \n" - - "1: \n" - // remain loop - "and r4, %6, #3 \n" // r4 = remain = inch & 3; - "cmp r4, #0 \n" - "beq 3f \n" - - "2: \n" - "pld [%1, #256] \n" - "vld1.f32 {d8-d11}, [%1]! \n" - "pld [%2, #32] \n" - "vld1.f32 {d0[],d1[]}, [%2]! \n" - - "subs r4, r4, #1 \n" - - "vmla.f32 q8, q4, q0 \n" - "vmla.f32 q9, q5, q0 \n" - "bne 2b \n" - - "3: \n" - "vst1.f32 {d16-d19}, [%0] \n" - - : "=r"(output), // %0 - "=r"(vb), // %1 - "=r"(va) // %2 - : "0"(output), - "1"(vb), - "2"(va), - "r"(L), // %6 - "r"(bias0) // %7 - : "cc", "memory", "r4", "q0", "q4", "q5", "q6", "q7", "q8", "q9", "q12", "q13", "q14", "q15"); -#endif // __aarch64__ + _sum0 = vmlaq_f32(_sum0, _p0, _k0); +#endif + } + +#if __aarch64__ + float sum0 = bias0 + vaddvq_f32(_sum0); #else - float sum[8] = {0}; + float32x2_t _ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); + float sum0 = bias0 + vget_lane_f32(vpadd_f32(_ss, _ss), 0); +#endif - int k = 0; - for (; k + 7 < L; k = k + 8) - { - for (int n = 0; n < 8; n++) - { - sum[n] += va[0] * vb[n]; - sum[n] += va[1] * vb[n + 8]; - sum[n] += va[2] * vb[n + 16]; - sum[n] += va[3] * vb[n + 24]; - sum[n] += va[4] * vb[n + 32]; - sum[n] += va[5] * vb[n + 40]; - sum[n] += va[6] * vb[n + 48]; - sum[n] += va[7] * vb[n + 56]; - } + for (; q < nn; q++) + { + sum0 += tmpptr[0] * kptr[0]; + tmpptr++; + kptr++; + } - va += 8; - vb += 64; - } + outptr0[0] = sum0; - for (; k < L; k++) - { - for (int n = 0; n < 8; n++) - { - sum[n] += va[0] * vb[n]; - } + outptr0++; + } + } +#else // __ARM_NEON + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* outptr0 = top_blob.channel(p); - va += 1; - vb += 8; - } + const float bias0 = bias ? bias[p] : 0.f; - for (int n = 0; n < 8; n++) - { - output[n] = sum[n] + bias0; - } -#endif // __ARM_NEON - output += 8; - } + for (int i = 0; i < size; i++) + { + const float* tmpptr = tmp.channel(i); + const float* kptr = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 - for (; j < N; j++) + float sum0 = bias0; + + for (int q = 0; q < nn; q++) { - const float* vb = bottom_tm.channel(j / 8 + j % 8); -#if __ARM_NEON && __aarch64__ - const float* va = kernel_tm.channel(i / 8 + (i % 8) / 4 + i % 4); -#else - const float* va = kernel_tm.channel(i / 4 + i % 4); -#endif // __ARM_NEON && __aarch64__ + sum0 += tmpptr[0] * kptr[0]; + tmpptr++; + kptr++; + } + + outptr0[0] = sum0; + + outptr0++; + } + } +#endif // __ARM_NEON +} - int k = 0; +static void convolution_im2col_sgemm_transform_kernel_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // interleave + // src = maxk-inch-outch + // dst = 4b-4a-maxk-inch/4a-outch/4b + Mat kernel = _kernel.reshape(maxk, inch, outch); #if __ARM_NEON - float32x4_t _sum0 = vdupq_n_f32(0.f); +#if __aarch64__ + kernel_tm.create(32 * maxk, inch / 4 + inch % 4, outch / 8 + (outch % 8) / 4 + outch % 4); +#else + kernel_tm.create(16 * maxk, inch / 4 + inch % 4, outch / 4 + outch % 4); +#endif - for (; k + 3 < L; k += 4) - { - float32x4_t _p0 = vld1q_f32(vb); - vb += 4; + int q = 0; +#if __aarch64__ + for (; q + 7 < outch; q += 8) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); + const Mat k4 = kernel.channel(q + 4); + const Mat k5 = kernel.channel(q + 5); + const Mat k6 = kernel.channel(q + 6); + const Mat k7 = kernel.channel(q + 7); - float32x4_t _k0 = vld1q_f32(va); - va += 4; + float* g00 = kernel_tm.channel(q / 8); + + for (int p = 0; p < inch; p++) + { + const float* k00 = k0.row(p); + const float* k10 = k1.row(p); + const float* k20 = k2.row(p); + const float* k30 = k3.row(p); + const float* k40 = k4.row(p); + const float* k50 = k5.row(p); + const float* k60 = k6.row(p); + const float* k70 = k7.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + g00[4] = k40[k]; + g00[5] = k50[k]; + g00[6] = k60[k]; + g00[7] = k70[k]; + + g00 += 8; + } + } + } +#endif // __aarch64__ + for (; q + 3 < outch; q += 4) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); #if __aarch64__ - _sum0 = vfmaq_f32(_sum0, _p0, _k0); + float* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4); #else - _sum0 = vmlaq_f32(_sum0, _p0, _k0); + float* g00 = kernel_tm.channel(q / 4); #endif - } + + for (int p = 0; p < inch; p++) + { + const float* k00 = k0.row(p); + const float* k10 = k1.row(p); + const float* k20 = k2.row(p); + const float* k30 = k3.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + + g00 += 4; + } + } + } + for (; q < outch; q++) + { + const Mat k0 = kernel.channel(q); #if __aarch64__ - float sum0 = bias0 + vaddvq_f32(_sum0); + float* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4 + q % 4); #else - float32x2_t _ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); - float sum0 = bias0 + vget_lane_f32(vpadd_f32(_ss, _ss), 0); + float* g00 = kernel_tm.channel(q / 4 + q % 4); #endif + + for (int p = 0; p < inch; p++) + { + const float* k00 = k0.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + + g00 += 1; + } + } + } #else - float sum0 = bias0; + kernel_tm = kernel; #endif // __ARM_NEON - for (; k < L; k++) +} + +static void convolution_im2col_sgemm_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 4u, 1, opt.workspace_allocator); + { + const int gap = w * stride_h - outw * stride_w; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + float* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) { - sum0 += va[0] * vb[0]; + const float* sptr = img.row(dilation_h * u) + dilation_w * v; - va += 1; - vb += 1; - } - output[0] = sum0; + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j + 3 < outw; j += 4) + { + ptr[0] = sptr[0]; + ptr[1] = sptr[stride_w]; + ptr[2] = sptr[stride_w * 2]; + ptr[3] = sptr[stride_w * 3]; + + sptr += stride_w * 4; + ptr += 4; + } + for (; j + 1 < outw; j += 2) + { + ptr[0] = sptr[0]; + ptr[1] = sptr[stride_w]; + + sptr += stride_w * 2; + ptr += 2; + } + for (; j < outw; j++) + { + ptr[0] = sptr[0]; + + sptr += stride_w; + ptr += 1; + } - output++; + sptr += gap; + } + } } } } + + im2col_sgemm_neon(bottom_im2col, top_blob, kernel, _bias, opt); } diff --git a/src/layer/arm/convolution_sgemm_pack4.h b/src/layer/arm/convolution_sgemm_pack4.h new file mode 100644 index 000000000..b1417d37b --- /dev/null +++ b/src/layer/arm/convolution_sgemm_pack4.h @@ -0,0 +1,1688 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void im2col_sgemm_pack4_neon(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + // Mat bottom_im2col(size, maxk, inch, 16u, 4, opt.workspace_allocator); + + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const float* bias = _bias; + + // permute + Mat tmp; +#if __aarch64__ + if (size >= 12) + tmp.create(12 * maxk, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, 16u, 4, opt.workspace_allocator); + else if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 16u, 4, opt.workspace_allocator); +#else + if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 16u, 4, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 16u, 4, opt.workspace_allocator); +#endif + { +#if __aarch64__ + int nn_size = size / 12; + int remain_size_start = 0; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 12; + + float* tmpptr = tmp.channel(i / 12); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n" + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v4.4s, v5.4s, v6.4s, v7.4s}, [%0], #64 \n" + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v8.4s, v9.4s, v10.4s, v11.4s}, [%0] \n" + "st1 {v0.4s}, [%1], #16 \n" + "st1 {v4.4s}, [%1], #16 \n" + "st1 {v8.4s}, [%1], #16 \n" + "sub %0, %0, #128 \n" + "st1 {v1.4s}, [%1], #16 \n" + "st1 {v5.4s}, [%1], #16 \n" + "st1 {v9.4s}, [%1], #16 \n" + "st1 {v2.4s}, [%1], #16 \n" + "st1 {v6.4s}, [%1], #16 \n" + "st1 {v10.4s}, [%1], #16 \n" + "st1 {v3.4s}, [%1], #16 \n" + "st1 {v7.4s}, [%1], #16 \n" + "st1 {v11.4s}, [%1], #16 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"); + img0 += size * 4; + } + } + } + + remain_size_start += nn_size * 12; + nn_size = (size - remain_size_start) >> 3; +#else + int nn_size = size >> 3; + int remain_size_start = 0; +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + +#if __aarch64__ + float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); +#else + float* tmpptr = tmp.channel(i / 8); +#endif + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%0], #64 \n" + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%0] \n" + "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n" + "sub %0, %0, #64 \n" + "st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); +#else + asm volatile( + "pld [%0, #512] \n" + "vldm %0!, {d0-d7} \n" + "pld [%0, #512] \n" + "vldm %0, {d16-d23} \n" + + // transpose 8x4 + "vtrn.32 q0, q1 \n" + "vtrn.32 q2, q3 \n" + "vtrn.32 q8, q9 \n" + "vtrn.32 q10, q11 \n" + "vswp d1, d4 \n" + "vswp d3, d6 \n" + "vswp d17, d20 \n" + "vswp d19, d22 \n" + "vswp q1, q8 \n" + "vswp q3, q10 \n" + + "vst1.f32 {d0-d3}, [%1 :128]! \n" + "vst1.f32 {d16-d19}, [%1 :128]! \n" + "sub %0, %0, #64 \n" + "vst1.f32 {d4-d7}, [%1 :128]! \n" + "vst1.f32 {d20-d23}, [%1 :128]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + +#if __aarch64__ + float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); +#else + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); +#endif + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%0] \n" + "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3"); +#else + asm volatile( + "pld [%0, #512] \n" + "vldm %0, {d0-d7} \n" + "vstm %1!, {d0-d7} \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0", "q1", "q2", "q3"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 2; + nn_size = (size - remain_size_start) >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 2; + +#if __aarch64__ + float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); +#else + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); +#endif + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.4s, v1.4s}, [%0] \n" + "st1 {v0.4s, v1.4s}, [%1], #32 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1"); +#else + asm volatile( + "pld [%0, #256] \n" + "vld1.f32 {d0-d3}, [%0 :128] \n" + "vst1.f32 {d0-d3}, [%1 :128]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0", "q1"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { +#if __aarch64__ + float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); +#else + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); +#endif + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.4s}, [%0] \n" + "st1 {v0.4s}, [%1], #16 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0"); +#else + asm volatile( + "pld [%0, #128] \n" + "vld1.f32 {d0-d1}, [%0 :128] \n" + "vst1.f32 {d0-d1}, [%1 :128]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + } + + int remain_outch_start = 0; + +#if __aarch64__ + int nn_outch = outch >> 1; + remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + float* outptr0 = top_blob.channel(p); + float* outptr1 = top_blob.channel(p + 1); + + const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p * 4 : zeros; + + int i = 0; + for (; i + 11 < size; i += 12) + { + const float* tmpptr = tmp.channel(i / 12); + const float* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v8.16b, v0.16b \n" + "mov v9.16b, v0.16b \n" + "mov v10.16b, v0.16b \n" + "mov v11.16b, v0.16b \n" + "mov v12.16b, v0.16b \n" + "mov v13.16b, v0.16b \n" + "mov v14.16b, v0.16b \n" + "mov v15.16b, v0.16b \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v1.16b \n" + "mov v21.16b, v1.16b \n" + "mov v22.16b, v1.16b \n" + "mov v23.16b, v1.16b \n" + "mov v24.16b, v1.16b \n" + "mov v25.16b, v1.16b \n" + "mov v26.16b, v1.16b \n" + "mov v27.16b, v1.16b \n" + "mov v28.16b, v1.16b \n" + "mov v29.16b, v1.16b \n" + "mov v30.16b, v1.16b \n" + "mov v31.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" // w0011_01 + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" + "fmla v12.4s, v4.4s, v1.s[0] \n" + "fmla v13.4s, v4.4s, v1.s[1] \n" + "fmla v14.4s, v4.4s, v1.s[2] \n" + "fmla v15.4s, v4.4s, v1.s[3] \n" + "fmla v16.4s, v4.4s, v2.s[0] \n" + "fmla v17.4s, v4.4s, v2.s[1] \n" + "fmla v18.4s, v4.4s, v2.s[2] \n" + "fmla v19.4s, v4.4s, v2.s[3] \n" + + "fmla v20.4s, v5.4s, v0.s[0] \n" + "fmla v21.4s, v5.4s, v0.s[1] \n" + "fmla v22.4s, v5.4s, v0.s[2] \n" + "fmla v23.4s, v5.4s, v0.s[3] \n" + "fmla v24.4s, v5.4s, v1.s[0] \n" + "fmla v25.4s, v5.4s, v1.s[1] \n" + "fmla v26.4s, v5.4s, v1.s[2] \n" + "fmla v27.4s, v5.4s, v1.s[3] \n" + "fmla v28.4s, v5.4s, v2.s[0] \n" + "fmla v29.4s, v5.4s, v2.s[1] \n" + "fmla v30.4s, v5.4s, v2.s[2] \n" + "fmla v31.4s, v5.4s, v2.s[3] \n" + + "fmla v8.4s, v6.4s, v3.s[0] \n" + "fmla v9.4s, v6.4s, v3.s[1] \n" + "fmla v10.4s, v6.4s, v3.s[2] \n" + "fmla v11.4s, v6.4s, v3.s[3] \n" + + "fmla v20.4s, v7.4s, v3.s[0] \n" + "fmla v21.4s, v7.4s, v3.s[1] \n" + "fmla v22.4s, v7.4s, v3.s[2] \n" + "fmla v23.4s, v7.4s, v3.s[3] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" + + "fmla v12.4s, v6.4s, v0.s[0] \n" + "fmla v13.4s, v6.4s, v0.s[1] \n" + "fmla v14.4s, v6.4s, v0.s[2] \n" + "fmla v15.4s, v6.4s, v0.s[3] \n" + "fmla v16.4s, v6.4s, v1.s[0] \n" + "fmla v17.4s, v6.4s, v1.s[1] \n" + "fmla v18.4s, v6.4s, v1.s[2] \n" + "fmla v19.4s, v6.4s, v1.s[3] \n" + + "fmla v24.4s, v7.4s, v0.s[0] \n" + "fmla v25.4s, v7.4s, v0.s[1] \n" + "fmla v26.4s, v7.4s, v0.s[2] \n" + "fmla v27.4s, v7.4s, v0.s[3] \n" + "fmla v28.4s, v7.4s, v1.s[0] \n" + "fmla v29.4s, v7.4s, v1.s[1] \n" + "fmla v30.4s, v7.4s, v1.s[2] \n" + "fmla v31.4s, v7.4s, v1.s[3] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%4], #64 \n" // w2233_01 + + "fmla v8.4s, v4.4s, v2.s[0] \n" + "fmla v9.4s, v4.4s, v2.s[1] \n" + "fmla v10.4s, v4.4s, v2.s[2] \n" + "fmla v11.4s, v4.4s, v2.s[3] \n" + "fmla v12.4s, v4.4s, v3.s[0] \n" + "fmla v13.4s, v4.4s, v3.s[1] \n" + "fmla v14.4s, v4.4s, v3.s[2] \n" + "fmla v15.4s, v4.4s, v3.s[3] \n" + + "fmla v20.4s, v5.4s, v2.s[0] \n" + "fmla v21.4s, v5.4s, v2.s[1] \n" + "fmla v22.4s, v5.4s, v2.s[2] \n" + "fmla v23.4s, v5.4s, v2.s[3] \n" + "fmla v24.4s, v5.4s, v3.s[0] \n" + "fmla v25.4s, v5.4s, v3.s[1] \n" + "fmla v26.4s, v5.4s, v3.s[2] \n" + "fmla v27.4s, v5.4s, v3.s[3] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" + + "fmla v16.4s, v4.4s, v0.s[0] \n" + "fmla v17.4s, v4.4s, v0.s[1] \n" + "fmla v18.4s, v4.4s, v0.s[2] \n" + "fmla v19.4s, v4.4s, v0.s[3] \n" + + "fmla v28.4s, v5.4s, v0.s[0] \n" + "fmla v29.4s, v5.4s, v0.s[1] \n" + "fmla v30.4s, v5.4s, v0.s[2] \n" + "fmla v31.4s, v5.4s, v0.s[3] \n" + + "fmla v8.4s, v6.4s, v1.s[0] \n" + "fmla v9.4s, v6.4s, v1.s[1] \n" + "fmla v10.4s, v6.4s, v1.s[2] \n" + "fmla v11.4s, v6.4s, v1.s[3] \n" + "fmla v12.4s, v6.4s, v2.s[0] \n" + "fmla v13.4s, v6.4s, v2.s[1] \n" + "fmla v14.4s, v6.4s, v2.s[2] \n" + "fmla v15.4s, v6.4s, v2.s[3] \n" + "fmla v16.4s, v6.4s, v3.s[0] \n" + "fmla v17.4s, v6.4s, v3.s[1] \n" + "fmla v18.4s, v6.4s, v3.s[2] \n" + "fmla v19.4s, v6.4s, v3.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v20.4s, v7.4s, v1.s[0] \n" + "fmla v21.4s, v7.4s, v1.s[1] \n" + "fmla v22.4s, v7.4s, v1.s[2] \n" + "fmla v23.4s, v7.4s, v1.s[3] \n" + "fmla v24.4s, v7.4s, v2.s[0] \n" + "fmla v25.4s, v7.4s, v2.s[1] \n" + "fmla v26.4s, v7.4s, v2.s[2] \n" + "fmla v27.4s, v7.4s, v2.s[3] \n" + "fmla v28.4s, v7.4s, v3.s[0] \n" + "fmla v29.4s, v7.4s, v3.s[1] \n" + "fmla v30.4s, v7.4s, v3.s[2] \n" + "fmla v31.4s, v7.4s, v3.s[3] \n" + + "bne 0b \n" + + "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" + "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" + "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" + "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 7 < size; i += 8) + { + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + const float* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v0.16b \n" + "mov v21.16b, v0.16b \n" + "mov v22.16b, v0.16b \n" + "mov v23.16b, v0.16b \n" + "mov v24.16b, v1.16b \n" + "mov v25.16b, v1.16b \n" + "mov v26.16b, v1.16b \n" + "mov v27.16b, v1.16b \n" + "mov v28.16b, v1.16b \n" + "mov v29.16b, v1.16b \n" + "mov v30.16b, v1.16b \n" + "mov v31.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // r4 r5 r6 r7 + + "fmla v20.4s, v8.4s, v4.s[0] \n" + "fmla v21.4s, v8.4s, v5.s[0] \n" + "fmla v22.4s, v8.4s, v6.s[0] \n" + "fmla v23.4s, v8.4s, v7.s[0] \n" + + "fmla v24.4s, v9.4s, v0.s[0] \n" + "fmla v25.4s, v9.4s, v1.s[0] \n" + "fmla v26.4s, v9.4s, v2.s[0] \n" + "fmla v27.4s, v9.4s, v3.s[0] \n" + "fmla v28.4s, v9.4s, v4.s[0] \n" + "fmla v29.4s, v9.4s, v5.s[0] \n" + "fmla v30.4s, v9.4s, v6.s[0] \n" + "fmla v31.4s, v9.4s, v7.s[0] \n" + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v10.4s, v2.s[1] \n" + "fmla v19.4s, v10.4s, v3.s[1] \n" + "fmla v20.4s, v10.4s, v4.s[1] \n" + "fmla v21.4s, v10.4s, v5.s[1] \n" + "fmla v22.4s, v10.4s, v6.s[1] \n" + "fmla v23.4s, v10.4s, v7.s[1] \n" + + "fmla v24.4s, v11.4s, v0.s[1] \n" + "fmla v25.4s, v11.4s, v1.s[1] \n" + "fmla v26.4s, v11.4s, v2.s[1] \n" + "fmla v27.4s, v11.4s, v3.s[1] \n" + "fmla v28.4s, v11.4s, v4.s[1] \n" + "fmla v29.4s, v11.4s, v5.s[1] \n" + "fmla v30.4s, v11.4s, v6.s[1] \n" + "fmla v31.4s, v11.4s, v7.s[1] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v12.4s, v2.s[2] \n" + "fmla v19.4s, v12.4s, v3.s[2] \n" + "fmla v20.4s, v12.4s, v4.s[2] \n" + "fmla v21.4s, v12.4s, v5.s[2] \n" + "fmla v22.4s, v12.4s, v6.s[2] \n" + "fmla v23.4s, v12.4s, v7.s[2] \n" + + "fmla v24.4s, v13.4s, v0.s[2] \n" + "fmla v25.4s, v13.4s, v1.s[2] \n" + "fmla v26.4s, v13.4s, v2.s[2] \n" + "fmla v27.4s, v13.4s, v3.s[2] \n" + "fmla v28.4s, v13.4s, v4.s[2] \n" + "fmla v29.4s, v13.4s, v5.s[2] \n" + "fmla v30.4s, v13.4s, v6.s[2] \n" + "fmla v31.4s, v13.4s, v7.s[2] \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v14.4s, v2.s[3] \n" + "fmla v19.4s, v14.4s, v3.s[3] \n" + "fmla v20.4s, v14.4s, v4.s[3] \n" + "fmla v21.4s, v14.4s, v5.s[3] \n" + "fmla v22.4s, v14.4s, v6.s[3] \n" + "fmla v23.4s, v14.4s, v7.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v24.4s, v15.4s, v0.s[3] \n" + "fmla v25.4s, v15.4s, v1.s[3] \n" + "fmla v26.4s, v15.4s, v2.s[3] \n" + "fmla v27.4s, v15.4s, v3.s[3] \n" + "fmla v28.4s, v15.4s, v4.s[3] \n" + "fmla v29.4s, v15.4s, v5.s[3] \n" + "fmla v30.4s, v15.4s, v6.s[3] \n" + "fmla v31.4s, v15.4s, v7.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" + "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n" + "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const float* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v1.16b \n" + "mov v21.16b, v1.16b \n" + "mov v22.16b, v1.16b \n" + "mov v23.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%3], #64 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "fmla v20.4s, v9.4s, v0.s[0] \n" + "fmla v21.4s, v9.4s, v1.s[0] \n" + "fmla v22.4s, v9.4s, v2.s[0] \n" + "fmla v23.4s, v9.4s, v3.s[0] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v10.4s, v2.s[1] \n" + "fmla v19.4s, v10.4s, v3.s[1] \n" + + "fmla v20.4s, v11.4s, v0.s[1] \n" + "fmla v21.4s, v11.4s, v1.s[1] \n" + "fmla v22.4s, v11.4s, v2.s[1] \n" + "fmla v23.4s, v11.4s, v3.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v12.4s, v2.s[2] \n" + "fmla v19.4s, v12.4s, v3.s[2] \n" + + "fmla v20.4s, v13.4s, v0.s[2] \n" + "fmla v21.4s, v13.4s, v1.s[2] \n" + "fmla v22.4s, v13.4s, v2.s[2] \n" + "fmla v23.4s, v13.4s, v3.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v14.4s, v2.s[3] \n" + "fmla v19.4s, v14.4s, v3.s[3] \n" + + "fmla v20.4s, v15.4s, v0.s[3] \n" + "fmla v21.4s, v15.4s, v1.s[3] \n" + "fmla v22.4s, v15.4s, v2.s[3] \n" + "fmla v23.4s, v15.4s, v3.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + } + for (; i + 1 < size; i += 2) + { + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const float* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v1.16b \n" + "mov v19.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4s, v1.4s}, [%3], #32 \n" // r0 r1 + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v9.4s, v0.s[0] \n" + "fmla v19.4s, v9.4s, v1.s[0] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v11.4s, v0.s[1] \n" + "fmla v19.4s, v11.4s, v1.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v13.4s, v0.s[2] \n" + "fmla v19.4s, v13.4s, v1.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v15.4s, v0.s[3] \n" + "fmla v19.4s, v15.4s, v1.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s}, [%1], #32 \n" + "st1 {v18.4s, v19.4s}, [%2], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } + for (; i < size; i++) + { + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const float* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.4s, v17.4s}, [%10] \n" + + "0: \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v0.4s}, [%3], #16 \n" // r0 + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%4], #64 \n" // w0011_01 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v9.4s, v0.s[0] \n" + + "prfm pldl1keep, [%4, #512] \n" + "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%4], #64 \n" // w2233_01 + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v11.4s, v0.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v13.4s, v0.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v15.4s, v0.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s}, [%1], #16 \n" + "st1 {v17.4s}, [%2], #16 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); + } + } +#endif // __aarch64__ + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + float* outptr0 = top_blob.channel(p); + + const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p * 4 : zeros; + + int i = 0; +#if __aarch64__ + for (; i + 11 < size; i += 12) + { + const float* tmpptr = tmp.channel(i / 12); + const float* kptr0 = kernel.channel(p / 2 + p % 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v8.16b, v0.16b \n" + "mov v9.16b, v0.16b \n" + "mov v10.16b, v0.16b \n" + "mov v11.16b, v0.16b \n" + "mov v12.16b, v0.16b \n" + "mov v13.16b, v0.16b \n" + "mov v14.16b, v0.16b \n" + "mov v15.16b, v0.16b \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%3], #64 \n" // w0123_0 + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" + "fmla v12.4s, v4.4s, v1.s[0] \n" + "fmla v13.4s, v4.4s, v1.s[1] \n" + "fmla v14.4s, v4.4s, v1.s[2] \n" + "fmla v15.4s, v4.4s, v1.s[3] \n" + "fmla v16.4s, v4.4s, v2.s[0] \n" + "fmla v17.4s, v4.4s, v2.s[1] \n" + "fmla v18.4s, v4.4s, v2.s[2] \n" + "fmla v19.4s, v4.4s, v2.s[3] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" + + "fmla v8.4s, v5.4s, v3.s[0] \n" + "fmla v9.4s, v5.4s, v3.s[1] \n" + "fmla v10.4s, v5.4s, v3.s[2] \n" + "fmla v11.4s, v5.4s, v3.s[3] \n" + "fmla v12.4s, v5.4s, v20.s[0] \n" + "fmla v13.4s, v5.4s, v20.s[1] \n" + "fmla v14.4s, v5.4s, v20.s[2] \n" + "fmla v15.4s, v5.4s, v20.s[3] \n" + "fmla v16.4s, v5.4s, v21.s[0] \n" + "fmla v17.4s, v5.4s, v21.s[1] \n" + "fmla v18.4s, v5.4s, v21.s[2] \n" + "fmla v19.4s, v5.4s, v21.s[3] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" + + "fmla v8.4s, v6.4s, v22.s[0] \n" + "fmla v9.4s, v6.4s, v22.s[1] \n" + "fmla v10.4s, v6.4s, v22.s[2] \n" + "fmla v11.4s, v6.4s, v22.s[3] \n" + "fmla v12.4s, v6.4s, v23.s[0] \n" + "fmla v13.4s, v6.4s, v23.s[1] \n" + "fmla v14.4s, v6.4s, v23.s[2] \n" + "fmla v15.4s, v6.4s, v23.s[3] \n" + "fmla v16.4s, v6.4s, v24.s[0] \n" + "fmla v17.4s, v6.4s, v24.s[1] \n" + "fmla v18.4s, v6.4s, v24.s[2] \n" + "fmla v19.4s, v6.4s, v24.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v8.4s, v7.4s, v25.s[0] \n" + "fmla v9.4s, v7.4s, v25.s[1] \n" + "fmla v10.4s, v7.4s, v25.s[2] \n" + "fmla v11.4s, v7.4s, v25.s[3] \n" + "fmla v12.4s, v7.4s, v26.s[0] \n" + "fmla v13.4s, v7.4s, v26.s[1] \n" + "fmla v14.4s, v7.4s, v26.s[2] \n" + "fmla v15.4s, v7.4s, v26.s[3] \n" + "fmla v16.4s, v7.4s, v27.s[0] \n" + "fmla v17.4s, v7.4s, v27.s[1] \n" + "fmla v18.4s, v7.4s, v27.s[2] \n" + "fmla v19.4s, v7.4s, v27.s[3] \n" + + "bne 0b \n" + + "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" + "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); + } +#endif // __aarch64__ + for (; i + 7 < size; i += 8) + { +#if __aarch64__ + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + const float* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const float* tmpptr = tmp.channel(i / 8); + const float* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v0.16b \n" + "mov v21.16b, v0.16b \n" + "mov v22.16b, v0.16b \n" + "mov v23.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%2], #64 \n" // r4 r5 r6 r7 + + "fmla v20.4s, v8.4s, v4.s[0] \n" + "fmla v21.4s, v8.4s, v5.s[0] \n" + "fmla v22.4s, v8.4s, v6.s[0] \n" + "fmla v23.4s, v8.4s, v7.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + "fmla v18.4s, v9.4s, v2.s[1] \n" + "fmla v19.4s, v9.4s, v3.s[1] \n" + "fmla v20.4s, v9.4s, v4.s[1] \n" + "fmla v21.4s, v9.4s, v5.s[1] \n" + "fmla v22.4s, v9.4s, v6.s[1] \n" + "fmla v23.4s, v9.4s, v7.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + "fmla v18.4s, v10.4s, v2.s[2] \n" + "fmla v19.4s, v10.4s, v3.s[2] \n" + "fmla v20.4s, v10.4s, v4.s[2] \n" + "fmla v21.4s, v10.4s, v5.s[2] \n" + "fmla v22.4s, v10.4s, v6.s[2] \n" + "fmla v23.4s, v10.4s, v7.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + "fmla v18.4s, v11.4s, v2.s[3] \n" + "fmla v19.4s, v11.4s, v3.s[3] \n" + "fmla v20.4s, v11.4s, v4.s[3] \n" + "fmla v21.4s, v11.4s, v5.s[3] \n" + "fmla v22.4s, v11.4s, v6.s[3] \n" + "fmla v23.4s, v11.4s, v7.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + "vmov q10, q0 \n" + "vmov q11, q0 \n" + "vmov q12, q0 \n" + "vmov q13, q0 \n" + "vmov q14, q0 \n" + "vmov q15, q0 \n" + + "0: \n" + + "pld [%2, #512] \n" + "vldm %2!, {d0-d7} \n" + + "pld [%3, #512] \n" + "vldm %3!, {d8-d15} \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d0[1] \n" + "vmla.f32 q10, q4, d1[0] \n" + "vmla.f32 q11, q4, d1[1] \n" + "vmla.f32 q12, q4, d2[0] \n" + "vmla.f32 q13, q4, d2[1] \n" + "vmla.f32 q14, q4, d3[0] \n" + "vmla.f32 q15, q4, d3[1] \n" + + "vmla.f32 q8, q5, d4[0] \n" + "vmla.f32 q9, q5, d4[1] \n" + "vmla.f32 q10, q5, d5[0] \n" + "vmla.f32 q11, q5, d5[1] \n" + "vmla.f32 q12, q5, d6[0] \n" + "vmla.f32 q13, q5, d6[1] \n" + "vmla.f32 q14, q5, d7[0] \n" + "vmla.f32 q15, q5, d7[1] \n" + + "pld [%2, #512] \n" + "vldm %2!, {d0-d7} \n" + + "vmla.f32 q8, q6, d0[0] \n" + "vmla.f32 q9, q6, d0[1] \n" + "vmla.f32 q10, q6, d1[0] \n" + "vmla.f32 q11, q6, d1[1] \n" + "vmla.f32 q12, q6, d2[0] \n" + "vmla.f32 q13, q6, d2[1] \n" + "vmla.f32 q14, q6, d3[0] \n" + "vmla.f32 q15, q6, d3[1] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d4[0] \n" + "vmla.f32 q9, q7, d4[1] \n" + "vmla.f32 q10, q7, d5[0] \n" + "vmla.f32 q11, q7, d5[1] \n" + "vmla.f32 q12, q7, d6[0] \n" + "vmla.f32 q13, q7, d6[1] \n" + "vmla.f32 q14, q7, d7[0] \n" + "vmla.f32 q15, q7, d7[1] \n" + + "bne 0b \n" + + "vstm %1!, {d16-d23} \n" + "vstm %1!, {d24-d31} \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); +#endif + } + for (; i + 3 < size; i += 4) + { +#if __aarch64__ + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const float* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + const float* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%2], #64 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + "fmla v18.4s, v9.4s, v2.s[1] \n" + "fmla v19.4s, v9.4s, v3.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + "fmla v18.4s, v10.4s, v2.s[2] \n" + "fmla v19.4s, v10.4s, v3.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + "fmla v18.4s, v11.4s, v2.s[3] \n" + "fmla v19.4s, v11.4s, v3.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + "vmov q10, q0 \n" + "vmov q11, q0 \n" + + "0: \n" + + "pld [%2, #512] \n" + "vldm %2!, {d0-d7} \n" + + "pld [%3, #512] \n" + "vldm %3!, {d8-d15} \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d2[0] \n" + "vmla.f32 q10, q4, d4[0] \n" + "vmla.f32 q11, q4, d6[0] \n" + + "vmla.f32 q8, q5, d0[1] \n" + "vmla.f32 q9, q5, d2[1] \n" + "vmla.f32 q10, q5, d4[1] \n" + "vmla.f32 q11, q5, d6[1] \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q9, q6, d3[0] \n" + "vmla.f32 q10, q6, d5[0] \n" + "vmla.f32 q11, q6, d7[0] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d1[1] \n" + "vmla.f32 q9, q7, d3[1] \n" + "vmla.f32 q10, q7, d5[1] \n" + "vmla.f32 q11, q7, d7[1] \n" + + "bne 0b \n" + + "vstm %1!, {d16-d23} \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); +#endif + } + for (; i + 1 < size; i += 2) + { +#if __aarch64__ + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const float* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + const float* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.4s, v1.4s}, [%2], #32 \n" // r0 r1 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s}, [%1], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v16", "v17"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + + "0: \n" + + "pld [%2, #256] \n" + "vld1.f32 {d0-d3}, [%2 :128]! \n" + + "pld [%3, #512] \n" + "vldm %3!, {d8-d15} \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d2[0] \n" + + "vmla.f32 q8, q5, d0[1] \n" + "vmla.f32 q9, q5, d2[1] \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q9, q6, d3[0] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d1[1] \n" + "vmla.f32 q9, q7, d3[1] \n" + + "bne 0b \n" + + "vst1.f32 {d16-d19}, [%1 :128]! \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q4", "q5", "q6", "q7", "q8", "q9"); +#endif + } + for (; i < size; i++) + { +#if __aarch64__ + const float* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const float* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + const float* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v16.4s}, [%8] \n" + + "0: \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v0.4s}, [%2], #16 \n" // r0 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%3], #64 \n" // w0123 + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v16.4s, v9.4s, v0.s[1] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v16.4s, v11.4s, v0.s[3] \n" + + "bne 0b \n" + + "st1 {v16.4s}, [%1], #16 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v16"); +#else + asm volatile( + "vld1.f32 {d16-d17}, [%8] \n" + + "0: \n" + + "pld [%2, #128] \n" + "vld1.f32 {d0-d1}, [%2 :128]! \n" + + "pld [%3, #512] \n" + "vldm %3!, {d8-d15} \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q8, q5, d0[1] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q8, q7, d1[1] \n" + + "bne 0b \n" + + "vst1.f32 {d16-d17}, [%1 :128]! \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q4", "q5", "q6", "q7", "q8"); +#endif + } + } +} + +static void convolution_im2col_sgemm_transform_kernel_pack4_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // interleave + // src = maxk-inch-outch + // dst = 4b-4a-maxk-inch/4a-outch/4b + Mat kernel = _kernel.reshape(maxk, inch, outch); +#if __aarch64__ + kernel_tm.create(32 * maxk, inch / 4, outch / 8 + (outch % 8) / 4); +#else + kernel_tm.create(16 * maxk, inch / 4, outch / 4); +#endif + + int q = 0; +#if __aarch64__ + for (; q + 7 < outch; q += 8) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); + const Mat k4 = kernel.channel(q + 4); + const Mat k5 = kernel.channel(q + 5); + const Mat k6 = kernel.channel(q + 6); + const Mat k7 = kernel.channel(q + 7); + + float* g00 = kernel_tm.channel(q / 8); + + for (int p = 0; p + 3 < inch; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + const float* k40 = k4.row(p); + const float* k41 = k4.row(p + 1); + const float* k42 = k4.row(p + 2); + const float* k43 = k4.row(p + 3); + + const float* k50 = k5.row(p); + const float* k51 = k5.row(p + 1); + const float* k52 = k5.row(p + 2); + const float* k53 = k5.row(p + 3); + + const float* k60 = k6.row(p); + const float* k61 = k6.row(p + 1); + const float* k62 = k6.row(p + 2); + const float* k63 = k6.row(p + 3); + + const float* k70 = k7.row(p); + const float* k71 = k7.row(p + 1); + const float* k72 = k7.row(p + 2); + const float* k73 = k7.row(p + 3); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + g00[4] = k40[k]; + g00[5] = k50[k]; + g00[6] = k60[k]; + g00[7] = k70[k]; + + g00[8] = k01[k]; + g00[9] = k11[k]; + g00[10] = k21[k]; + g00[11] = k31[k]; + g00[12] = k41[k]; + g00[13] = k51[k]; + g00[14] = k61[k]; + g00[15] = k71[k]; + + g00[16] = k02[k]; + g00[17] = k12[k]; + g00[18] = k22[k]; + g00[19] = k32[k]; + g00[20] = k42[k]; + g00[21] = k52[k]; + g00[22] = k62[k]; + g00[23] = k72[k]; + + g00[24] = k03[k]; + g00[25] = k13[k]; + g00[26] = k23[k]; + g00[27] = k33[k]; + g00[28] = k43[k]; + g00[29] = k53[k]; + g00[30] = k63[k]; + g00[31] = k73[k]; + + g00 += 32; + } + } + } +#endif // __aarch64__ + for (; q + 3 < outch; q += 4) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); + +#if __aarch64__ + float* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4); +#else + float* g00 = kernel_tm.channel(q / 4); +#endif + + for (int p = 0; p + 3 < inch; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + g00[2] = k20[k]; + g00[3] = k30[k]; + + g00[4] = k01[k]; + g00[5] = k11[k]; + g00[6] = k21[k]; + g00[7] = k31[k]; + + g00[8] = k02[k]; + g00[9] = k12[k]; + g00[10] = k22[k]; + g00[11] = k32[k]; + + g00[12] = k03[k]; + g00[13] = k13[k]; + g00[14] = k23[k]; + g00[15] = k33[k]; + + g00 += 16; + } + } + } +} + +static void convolution_im2col_sgemm_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 16u, 4, opt.workspace_allocator); + { + const int gap = (w * stride_h - outw * stride_w) * 4; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + float* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) + { + const float* sptr = img.row(dilation_h * u) + dilation_w * v * 4; + + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j + 3 < outw; j += 4) + { + float32x4_t _val0 = vld1q_f32(sptr); + float32x4_t _val1 = vld1q_f32(sptr + stride_w * 4); + float32x4_t _val2 = vld1q_f32(sptr + stride_w * 8); + float32x4_t _val3 = vld1q_f32(sptr + stride_w * 12); + vst1q_f32(ptr, _val0); + vst1q_f32(ptr + 4, _val1); + vst1q_f32(ptr + 8, _val2); + vst1q_f32(ptr + 12, _val3); + + sptr += stride_w * 16; + ptr += 16; + } + for (; j + 1 < outw; j += 2) + { + float32x4_t _val0 = vld1q_f32(sptr); + float32x4_t _val1 = vld1q_f32(sptr + stride_w * 4); + vst1q_f32(ptr, _val0); + vst1q_f32(ptr + 4, _val1); + + sptr += stride_w * 8; + ptr += 8; + } + for (; j < outw; j++) + { + float32x4_t _val = vld1q_f32(sptr); + vst1q_f32(ptr, _val); + + sptr += stride_w * 4; + ptr += 4; + } + + sptr += gap; + } + } + } + } + } + + im2col_sgemm_pack4_neon(bottom_im2col, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/arm/convolution_sgemm_pack4_bf16s.h b/src/layer/arm/convolution_sgemm_pack4_bf16s.h new file mode 100644 index 000000000..f6e0447c6 --- /dev/null +++ b/src/layer/arm/convolution_sgemm_pack4_bf16s.h @@ -0,0 +1,1975 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void im2col_sgemm_pack4_bf16s_neon(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + // Mat bottom_im2col(size, maxk, inch, 8u, 4, opt.workspace_allocator); + + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const float* bias = _bias; + + // permute + Mat tmp; +#if __aarch64__ + if (size >= 12) + tmp.create(12 * maxk, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, 8u, 4, opt.workspace_allocator); + else if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 8u, 4, opt.workspace_allocator); +#else + if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 8u, 4, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 8u, 4, opt.workspace_allocator); +#endif + { +#if __aarch64__ + int nn_size = size / 12; + int remain_size_start = 0; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 12; + + unsigned short* tmpptr = tmp.channel(i / 12); + + for (int q = 0; q < inch; q++) + { + const unsigned short* img0 = (const unsigned short*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.4h, v5.4h, v6.4h, v7.4h}, [%0] \n" + "st1 {v0.8h}, [%1], #16 \n" + "st1 {v4.4h}, [%1], #8 \n" + "st1 {v1.8h}, [%1], #16 \n" + "st1 {v5.4h}, [%1], #8 \n" + "sub %0, %0, #64 \n" + "st1 {v2.8h}, [%1], #16 \n" + "st1 {v6.4h}, [%1], #8 \n" + "st1 {v3.8h}, [%1], #16 \n" + "st1 {v7.4h}, [%1], #8 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); + img0 += size * 4; + } + } + } + + remain_size_start += nn_size * 12; + nn_size = (size - remain_size_start) >> 3; +#else + int nn_size = size >> 3; + int remain_size_start = 0; +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + +#if __aarch64__ + unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); +#else + unsigned short* tmpptr = tmp.channel(i / 8); +#endif + + for (int q = 0; q < inch; q++) + { + const unsigned short* img0 = (const unsigned short*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3"); +#else + asm volatile( + "pld [%0, #256] \n" + "vld4.u16 {d0-d3}, [%0]! \n" + "pld [%0, #256] \n" + "vld4.u16 {d4-d7}, [%0] \n" + "sub %0, %0, #32 \n" + "vst1.u16 {d0}, [%1 :64]! \n" + "vst1.u16 {d4}, [%1 :64]! \n" + "vst1.u16 {d1}, [%1 :64]! \n" + "vst1.u16 {d5}, [%1 :64]! \n" + "vst1.u16 {d2}, [%1 :64]! \n" + "vst1.u16 {d6}, [%1 :64]! \n" + "vst1.u16 {d3}, [%1 :64]! \n" + "vst1.u16 {d7}, [%1 :64]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0", "q1", "q2", "q3"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + +#if __aarch64__ + unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); +#else + unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); +#endif + + for (int q = 0; q < inch; q++) + { + const unsigned short* img0 = (const unsigned short*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.8h, v1.8h}, [%0] \n" + "st1 {v0.8h, v1.8h}, [%1], #32 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1"); +#else + asm volatile( + "pld [%0, #256] \n" + "vld1.u16 {d0-d3}, [%0 :128] \n" + "vst1.u16 {d0-d3}, [%1 :128]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0", "q1"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 2; + nn_size = (size - remain_size_start) >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 2; + +#if __aarch64__ + unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); +#else + unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); +#endif + + for (int q = 0; q < inch; q++) + { + const unsigned short* img0 = (const unsigned short*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.8h}, [%0] \n" + "st1 {v0.8h}, [%1], #16 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0"); +#else + asm volatile( + "pld [%0, #128] \n" + "vld1.u16 {d0-d1}, [%0 :128] \n" + "vst1.u16 {d0-d1}, [%1 :128]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + + remain_size_start += nn_size << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { +#if __aarch64__ + unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); +#else + unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); +#endif + + for (int q = 0; q < inch; q++) + { + const unsigned short* img0 = (const unsigned short*)bottom_im2col.channel(q) + i * 4; + + for (int k = 0; k < maxk; k++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #64] \n" + "ld1 {v0.4h}, [%0] \n" + "st1 {v0.4h}, [%1], #8 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0"); +#else + asm volatile( + "pld [%0, #64] \n" + "vld1.u16 {d0}, [%0 :64] \n" + "vst1.u16 {d0}, [%1 :64]! \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "q0"); +#endif // __aarch64__ + img0 += size * 4; + } + } + } + } + + int remain_outch_start = 0; + +#if __aarch64__ + int nn_outch = outch >> 1; + remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + unsigned short* outptr0 = top_blob.channel(p); + unsigned short* outptr1 = top_blob.channel(p + 1); + + const float zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p * 4 : zeros; + + int i = 0; + for (; i + 11 < size; i += 12) + { + const unsigned short* tmpptr = tmp.channel(i / 12); + const unsigned short* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v8.16b, v0.16b \n" + "mov v9.16b, v0.16b \n" + "mov v10.16b, v0.16b \n" + "mov v11.16b, v0.16b \n" + "mov v12.16b, v0.16b \n" + "mov v13.16b, v0.16b \n" + "mov v14.16b, v0.16b \n" + "mov v15.16b, v0.16b \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v1.16b \n" + "mov v21.16b, v1.16b \n" + "mov v22.16b, v1.16b \n" + "mov v23.16b, v1.16b \n" + "mov v24.16b, v1.16b \n" + "mov v25.16b, v1.16b \n" + "mov v26.16b, v1.16b \n" + "mov v27.16b, v1.16b \n" + "mov v28.16b, v1.16b \n" + "mov v29.16b, v1.16b \n" + "mov v30.16b, v1.16b \n" + "mov v31.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%4], #32 \n" // w0011_01 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v4.4s, v4.4h, #16 \n" + "shll v5.4s, v5.4h, #16 \n" + "shll v6.4s, v6.4h, #16 \n" + "shll v7.4s, v7.4h, #16 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" + "fmla v12.4s, v4.4s, v1.s[0] \n" + "fmla v13.4s, v4.4s, v1.s[1] \n" + "fmla v14.4s, v4.4s, v1.s[2] \n" + "fmla v15.4s, v4.4s, v1.s[3] \n" + "fmla v16.4s, v4.4s, v2.s[0] \n" + "fmla v17.4s, v4.4s, v2.s[1] \n" + "fmla v18.4s, v4.4s, v2.s[2] \n" + "fmla v19.4s, v4.4s, v2.s[3] \n" + + "fmla v20.4s, v5.4s, v0.s[0] \n" + "fmla v21.4s, v5.4s, v0.s[1] \n" + "fmla v22.4s, v5.4s, v0.s[2] \n" + "fmla v23.4s, v5.4s, v0.s[3] \n" + "fmla v24.4s, v5.4s, v1.s[0] \n" + "fmla v25.4s, v5.4s, v1.s[1] \n" + "fmla v26.4s, v5.4s, v1.s[2] \n" + "fmla v27.4s, v5.4s, v1.s[3] \n" + "fmla v28.4s, v5.4s, v2.s[0] \n" + "fmla v29.4s, v5.4s, v2.s[1] \n" + "fmla v30.4s, v5.4s, v2.s[2] \n" + "fmla v31.4s, v5.4s, v2.s[3] \n" + + "fmla v8.4s, v6.4s, v3.s[0] \n" + "fmla v9.4s, v6.4s, v3.s[1] \n" + "fmla v10.4s, v6.4s, v3.s[2] \n" + "fmla v11.4s, v6.4s, v3.s[3] \n" + + "fmla v20.4s, v7.4s, v3.s[0] \n" + "fmla v21.4s, v7.4s, v3.s[1] \n" + "fmla v22.4s, v7.4s, v3.s[2] \n" + "fmla v23.4s, v7.4s, v3.s[3] \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "fmla v12.4s, v6.4s, v0.s[0] \n" + "fmla v13.4s, v6.4s, v0.s[1] \n" + "fmla v14.4s, v6.4s, v0.s[2] \n" + "fmla v15.4s, v6.4s, v0.s[3] \n" + "fmla v16.4s, v6.4s, v1.s[0] \n" + "fmla v17.4s, v6.4s, v1.s[1] \n" + "fmla v18.4s, v6.4s, v1.s[2] \n" + "fmla v19.4s, v6.4s, v1.s[3] \n" + + "fmla v24.4s, v7.4s, v0.s[0] \n" + "fmla v25.4s, v7.4s, v0.s[1] \n" + "fmla v26.4s, v7.4s, v0.s[2] \n" + "fmla v27.4s, v7.4s, v0.s[3] \n" + "fmla v28.4s, v7.4s, v1.s[0] \n" + "fmla v29.4s, v7.4s, v1.s[1] \n" + "fmla v30.4s, v7.4s, v1.s[2] \n" + "fmla v31.4s, v7.4s, v1.s[3] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%4], #32 \n" // w2233_01 + + "shll v4.4s, v4.4h, #16 \n" + "shll v5.4s, v5.4h, #16 \n" + "shll v6.4s, v6.4h, #16 \n" + "shll v7.4s, v7.4h, #16 \n" + + "fmla v8.4s, v4.4s, v2.s[0] \n" + "fmla v9.4s, v4.4s, v2.s[1] \n" + "fmla v10.4s, v4.4s, v2.s[2] \n" + "fmla v11.4s, v4.4s, v2.s[3] \n" + "fmla v12.4s, v4.4s, v3.s[0] \n" + "fmla v13.4s, v4.4s, v3.s[1] \n" + "fmla v14.4s, v4.4s, v3.s[2] \n" + "fmla v15.4s, v4.4s, v3.s[3] \n" + + "fmla v20.4s, v5.4s, v2.s[0] \n" + "fmla v21.4s, v5.4s, v2.s[1] \n" + "fmla v22.4s, v5.4s, v2.s[2] \n" + "fmla v23.4s, v5.4s, v2.s[3] \n" + "fmla v24.4s, v5.4s, v3.s[0] \n" + "fmla v25.4s, v5.4s, v3.s[1] \n" + "fmla v26.4s, v5.4s, v3.s[2] \n" + "fmla v27.4s, v5.4s, v3.s[3] \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "fmla v16.4s, v4.4s, v0.s[0] \n" + "fmla v17.4s, v4.4s, v0.s[1] \n" + "fmla v18.4s, v4.4s, v0.s[2] \n" + "fmla v19.4s, v4.4s, v0.s[3] \n" + + "fmla v28.4s, v5.4s, v0.s[0] \n" + "fmla v29.4s, v5.4s, v0.s[1] \n" + "fmla v30.4s, v5.4s, v0.s[2] \n" + "fmla v31.4s, v5.4s, v0.s[3] \n" + + "fmla v8.4s, v6.4s, v1.s[0] \n" + "fmla v9.4s, v6.4s, v1.s[1] \n" + "fmla v10.4s, v6.4s, v1.s[2] \n" + "fmla v11.4s, v6.4s, v1.s[3] \n" + "fmla v12.4s, v6.4s, v2.s[0] \n" + "fmla v13.4s, v6.4s, v2.s[1] \n" + "fmla v14.4s, v6.4s, v2.s[2] \n" + "fmla v15.4s, v6.4s, v2.s[3] \n" + "fmla v16.4s, v6.4s, v3.s[0] \n" + "fmla v17.4s, v6.4s, v3.s[1] \n" + "fmla v18.4s, v6.4s, v3.s[2] \n" + "fmla v19.4s, v6.4s, v3.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v20.4s, v7.4s, v1.s[0] \n" + "fmla v21.4s, v7.4s, v1.s[1] \n" + "fmla v22.4s, v7.4s, v1.s[2] \n" + "fmla v23.4s, v7.4s, v1.s[3] \n" + "fmla v24.4s, v7.4s, v2.s[0] \n" + "fmla v25.4s, v7.4s, v2.s[1] \n" + "fmla v26.4s, v7.4s, v2.s[2] \n" + "fmla v27.4s, v7.4s, v2.s[3] \n" + "fmla v28.4s, v7.4s, v3.s[0] \n" + "fmla v29.4s, v7.4s, v3.s[1] \n" + "fmla v30.4s, v7.4s, v3.s[2] \n" + "fmla v31.4s, v7.4s, v3.s[3] \n" + + "bne 0b \n" + + "shrn v8.4h, v8.4s, #16 \n" + "shrn v9.4h, v9.4s, #16 \n" + "shrn v10.4h, v10.4s, #16 \n" + "shrn v11.4h, v11.4s, #16 \n" + + "shrn v12.4h, v12.4s, #16 \n" + "shrn v13.4h, v13.4s, #16 \n" + "shrn v14.4h, v14.4s, #16 \n" + "shrn v15.4h, v15.4s, #16 \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "shrn v20.4h, v20.4s, #16 \n" + "shrn v21.4h, v21.4s, #16 \n" + "shrn v22.4h, v22.4s, #16 \n" + "shrn v23.4h, v23.4s, #16 \n" + + "shrn v24.4h, v24.4s, #16 \n" + "shrn v25.4h, v25.4s, #16 \n" + "shrn v26.4h, v26.4s, #16 \n" + "shrn v27.4h, v27.4s, #16 \n" + + "shrn v28.4h, v28.4s, #16 \n" + "shrn v29.4h, v29.4s, #16 \n" + "shrn v30.4h, v30.4s, #16 \n" + "shrn v31.4h, v31.4s, #16 \n" + + "st1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%1], #32 \n" + "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" + "st1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%1], #32 \n" + "st1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%2], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 7 < size; i += 8) + { + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + const unsigned short* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v0.16b \n" + "mov v21.16b, v0.16b \n" + "mov v22.16b, v0.16b \n" + "mov v23.16b, v0.16b \n" + "mov v24.16b, v1.16b \n" + "mov v25.16b, v1.16b \n" + "mov v26.16b, v1.16b \n" + "mov v27.16b, v1.16b \n" + "mov v28.16b, v1.16b \n" + "mov v29.16b, v1.16b \n" + "mov v30.16b, v1.16b \n" + "mov v31.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%3], #32 \n" // r4 r5 r6 r7 + + "shll v4.4s, v4.4h, #16 \n" + "shll v5.4s, v5.4h, #16 \n" + "shll v6.4s, v6.4h, #16 \n" + "shll v7.4s, v7.4h, #16 \n" + + "fmla v20.4s, v8.4s, v4.s[0] \n" + "fmla v21.4s, v8.4s, v5.s[0] \n" + "fmla v22.4s, v8.4s, v6.s[0] \n" + "fmla v23.4s, v8.4s, v7.s[0] \n" + + "fmla v24.4s, v9.4s, v0.s[0] \n" + "fmla v25.4s, v9.4s, v1.s[0] \n" + "fmla v26.4s, v9.4s, v2.s[0] \n" + "fmla v27.4s, v9.4s, v3.s[0] \n" + "fmla v28.4s, v9.4s, v4.s[0] \n" + "fmla v29.4s, v9.4s, v5.s[0] \n" + "fmla v30.4s, v9.4s, v6.s[0] \n" + "fmla v31.4s, v9.4s, v7.s[0] \n" + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v10.4s, v2.s[1] \n" + "fmla v19.4s, v10.4s, v3.s[1] \n" + "fmla v20.4s, v10.4s, v4.s[1] \n" + "fmla v21.4s, v10.4s, v5.s[1] \n" + "fmla v22.4s, v10.4s, v6.s[1] \n" + "fmla v23.4s, v10.4s, v7.s[1] \n" + + "fmla v24.4s, v11.4s, v0.s[1] \n" + "fmla v25.4s, v11.4s, v1.s[1] \n" + "fmla v26.4s, v11.4s, v2.s[1] \n" + "fmla v27.4s, v11.4s, v3.s[1] \n" + "fmla v28.4s, v11.4s, v4.s[1] \n" + "fmla v29.4s, v11.4s, v5.s[1] \n" + "fmla v30.4s, v11.4s, v6.s[1] \n" + "fmla v31.4s, v11.4s, v7.s[1] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 + + "shll v12.4s, v12.4h, #16 \n" + "shll v13.4s, v13.4h, #16 \n" + "shll v14.4s, v14.4h, #16 \n" + "shll v15.4s, v15.4h, #16 \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v12.4s, v2.s[2] \n" + "fmla v19.4s, v12.4s, v3.s[2] \n" + "fmla v20.4s, v12.4s, v4.s[2] \n" + "fmla v21.4s, v12.4s, v5.s[2] \n" + "fmla v22.4s, v12.4s, v6.s[2] \n" + "fmla v23.4s, v12.4s, v7.s[2] \n" + + "fmla v24.4s, v13.4s, v0.s[2] \n" + "fmla v25.4s, v13.4s, v1.s[2] \n" + "fmla v26.4s, v13.4s, v2.s[2] \n" + "fmla v27.4s, v13.4s, v3.s[2] \n" + "fmla v28.4s, v13.4s, v4.s[2] \n" + "fmla v29.4s, v13.4s, v5.s[2] \n" + "fmla v30.4s, v13.4s, v6.s[2] \n" + "fmla v31.4s, v13.4s, v7.s[2] \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v14.4s, v2.s[3] \n" + "fmla v19.4s, v14.4s, v3.s[3] \n" + "fmla v20.4s, v14.4s, v4.s[3] \n" + "fmla v21.4s, v14.4s, v5.s[3] \n" + "fmla v22.4s, v14.4s, v6.s[3] \n" + "fmla v23.4s, v14.4s, v7.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v24.4s, v15.4s, v0.s[3] \n" + "fmla v25.4s, v15.4s, v1.s[3] \n" + "fmla v26.4s, v15.4s, v2.s[3] \n" + "fmla v27.4s, v15.4s, v3.s[3] \n" + "fmla v28.4s, v15.4s, v4.s[3] \n" + "fmla v29.4s, v15.4s, v5.s[3] \n" + "fmla v30.4s, v15.4s, v6.s[3] \n" + "fmla v31.4s, v15.4s, v7.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "shrn v20.4h, v20.4s, #16 \n" + "shrn v21.4h, v21.4s, #16 \n" + "shrn v22.4h, v22.4s, #16 \n" + "shrn v23.4h, v23.4s, #16 \n" + + "shrn v24.4h, v24.4s, #16 \n" + "shrn v25.4h, v25.4s, #16 \n" + "shrn v26.4h, v26.4s, #16 \n" + "shrn v27.4h, v27.4s, #16 \n" + + "shrn v28.4h, v28.4s, #16 \n" + "shrn v29.4h, v29.4s, #16 \n" + "shrn v30.4h, v30.4s, #16 \n" + "shrn v31.4h, v31.4s, #16 \n" + + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + "st1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" + "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%1], #32 \n" + "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%2], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 3 < size; i += 4) + { + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const unsigned short* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v1.16b \n" + "mov v21.16b, v1.16b \n" + "mov v22.16b, v1.16b \n" + "mov v23.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%3], #32 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "fmla v20.4s, v9.4s, v0.s[0] \n" + "fmla v21.4s, v9.4s, v1.s[0] \n" + "fmla v22.4s, v9.4s, v2.s[0] \n" + "fmla v23.4s, v9.4s, v3.s[0] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 + + "shll v12.4s, v12.4h, #16 \n" + "shll v13.4s, v13.4h, #16 \n" + "shll v14.4s, v14.4h, #16 \n" + "shll v15.4s, v15.4h, #16 \n" + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v10.4s, v2.s[1] \n" + "fmla v19.4s, v10.4s, v3.s[1] \n" + + "fmla v20.4s, v11.4s, v0.s[1] \n" + "fmla v21.4s, v11.4s, v1.s[1] \n" + "fmla v22.4s, v11.4s, v2.s[1] \n" + "fmla v23.4s, v11.4s, v3.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v12.4s, v2.s[2] \n" + "fmla v19.4s, v12.4s, v3.s[2] \n" + + "fmla v20.4s, v13.4s, v0.s[2] \n" + "fmla v21.4s, v13.4s, v1.s[2] \n" + "fmla v22.4s, v13.4s, v2.s[2] \n" + "fmla v23.4s, v13.4s, v3.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v14.4s, v2.s[3] \n" + "fmla v19.4s, v14.4s, v3.s[3] \n" + + "fmla v20.4s, v15.4s, v0.s[3] \n" + "fmla v21.4s, v15.4s, v1.s[3] \n" + "fmla v22.4s, v15.4s, v2.s[3] \n" + "fmla v23.4s, v15.4s, v3.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "shrn v20.4h, v20.4s, #16 \n" + "shrn v21.4h, v21.4s, #16 \n" + "shrn v22.4h, v22.4s, #16 \n" + "shrn v23.4h, v23.4s, #16 \n" + + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + } + for (; i + 1 < size; i += 2) + { + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const unsigned short* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s, v1.4s}, [%10] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v1.16b \n" + "mov v19.16b, v1.16b \n" + + "0: \n" + + "prfm pldl1keep, [%3, #128] \n" + "ld1 {v0.4h, v1.4h}, [%3], #16 \n" // r0 r1 + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v9.4s, v0.s[0] \n" + "fmla v19.4s, v9.4s, v1.s[0] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 + + "shll v12.4s, v12.4h, #16 \n" + "shll v13.4s, v13.4h, #16 \n" + "shll v14.4s, v14.4h, #16 \n" + "shll v15.4s, v15.4h, #16 \n" + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v10.4s, v1.s[1] \n" + "fmla v18.4s, v11.4s, v0.s[1] \n" + "fmla v19.4s, v11.4s, v1.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v12.4s, v1.s[2] \n" + "fmla v18.4s, v13.4s, v0.s[2] \n" + "fmla v19.4s, v13.4s, v1.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v14.4s, v1.s[3] \n" + "fmla v18.4s, v15.4s, v0.s[3] \n" + "fmla v19.4s, v15.4s, v1.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "st1 {v16.4h, v17.4h}, [%1], #16 \n" + "st1 {v18.4h, v19.4h}, [%2], #16 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } + for (; i < size; i++) + { + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const unsigned short* kptr0 = kernel.channel(p / 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.4s, v17.4s}, [%10] \n" + + "0: \n" + + "prfm pldl1keep, [%3, #64] \n" + "ld1 {v0.4h}, [%3], #8 \n" // r0 + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%4], #32 \n" // w0011_01 + + "shll v0.4s, v0.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v9.4s, v0.s[0] \n" + + "prfm pldl1keep, [%4, #256] \n" + "ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%4], #32 \n" // w2233_01 + + "shll v12.4s, v12.4h, #16 \n" + "shll v13.4s, v13.4h, #16 \n" + "shll v14.4s, v14.4h, #16 \n" + "shll v15.4s, v15.4h, #16 \n" + + "fmla v16.4s, v10.4s, v0.s[1] \n" + "fmla v17.4s, v11.4s, v0.s[1] \n" + + "fmla v16.4s, v12.4s, v0.s[2] \n" + "fmla v17.4s, v13.4s, v0.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v14.4s, v0.s[3] \n" + "fmla v17.4s, v15.4s, v0.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + + "st1 {v16.4h}, [%1], #8 \n" + "st1 {v17.4h}, [%2], #8 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(outptr1), // %2 + "=r"(tmpptr), // %3 + "=r"(kptr0) // %4 + : "0"(nn), + "1"(outptr0), + "2"(outptr1), + "3"(tmpptr), + "4"(kptr0), + "r"(biasptr) // %10 + : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); + } + } +#endif // __aarch64__ + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + unsigned short* outptr0 = top_blob.channel(p); + + const float zeros[4] = {0.f, 0.f, 0.f, 0.f}; + const float* biasptr = bias ? bias + p * 4 : zeros; + + int i = 0; +#if __aarch64__ + for (; i + 11 < size; i += 12) + { + const unsigned short* tmpptr = tmp.channel(i / 12); + const unsigned short* kptr0 = kernel.channel(p / 2 + p % 2); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v8.16b, v0.16b \n" + "mov v9.16b, v0.16b \n" + "mov v10.16b, v0.16b \n" + "mov v11.16b, v0.16b \n" + "mov v12.16b, v0.16b \n" + "mov v13.16b, v0.16b \n" + "mov v14.16b, v0.16b \n" + "mov v15.16b, v0.16b \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%3], #32 \n" // w0123_0 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v4.4s, v4.4h, #16 \n" + "shll v5.4s, v5.4h, #16 \n" + "shll v6.4s, v6.4h, #16 \n" + "shll v7.4s, v7.4h, #16 \n" + + "fmla v8.4s, v4.4s, v0.s[0] \n" + "fmla v9.4s, v4.4s, v0.s[1] \n" + "fmla v10.4s, v4.4s, v0.s[2] \n" + "fmla v11.4s, v4.4s, v0.s[3] \n" + "fmla v12.4s, v4.4s, v1.s[0] \n" + "fmla v13.4s, v4.4s, v1.s[1] \n" + "fmla v14.4s, v4.4s, v1.s[2] \n" + "fmla v15.4s, v4.4s, v1.s[3] \n" + "fmla v16.4s, v4.4s, v2.s[0] \n" + "fmla v17.4s, v4.4s, v2.s[1] \n" + "fmla v18.4s, v4.4s, v2.s[2] \n" + "fmla v19.4s, v4.4s, v2.s[3] \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%2], #32 \n" + + "shll v20.4s, v20.4h, #16 \n" + "shll v21.4s, v21.4h, #16 \n" + "shll v22.4s, v22.4h, #16 \n" + "shll v23.4s, v23.4h, #16 \n" + + "fmla v8.4s, v5.4s, v3.s[0] \n" + "fmla v9.4s, v5.4s, v3.s[1] \n" + "fmla v10.4s, v5.4s, v3.s[2] \n" + "fmla v11.4s, v5.4s, v3.s[3] \n" + "fmla v12.4s, v5.4s, v20.s[0] \n" + "fmla v13.4s, v5.4s, v20.s[1] \n" + "fmla v14.4s, v5.4s, v20.s[2] \n" + "fmla v15.4s, v5.4s, v20.s[3] \n" + "fmla v16.4s, v5.4s, v21.s[0] \n" + "fmla v17.4s, v5.4s, v21.s[1] \n" + "fmla v18.4s, v5.4s, v21.s[2] \n" + "fmla v19.4s, v5.4s, v21.s[3] \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%2], #32 \n" + + "shll v24.4s, v24.4h, #16 \n" + "shll v25.4s, v25.4h, #16 \n" + "shll v26.4s, v26.4h, #16 \n" + "shll v27.4s, v27.4h, #16 \n" + + "fmla v8.4s, v6.4s, v22.s[0] \n" + "fmla v9.4s, v6.4s, v22.s[1] \n" + "fmla v10.4s, v6.4s, v22.s[2] \n" + "fmla v11.4s, v6.4s, v22.s[3] \n" + "fmla v12.4s, v6.4s, v23.s[0] \n" + "fmla v13.4s, v6.4s, v23.s[1] \n" + "fmla v14.4s, v6.4s, v23.s[2] \n" + "fmla v15.4s, v6.4s, v23.s[3] \n" + "fmla v16.4s, v6.4s, v24.s[0] \n" + "fmla v17.4s, v6.4s, v24.s[1] \n" + "fmla v18.4s, v6.4s, v24.s[2] \n" + "fmla v19.4s, v6.4s, v24.s[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v8.4s, v7.4s, v25.s[0] \n" + "fmla v9.4s, v7.4s, v25.s[1] \n" + "fmla v10.4s, v7.4s, v25.s[2] \n" + "fmla v11.4s, v7.4s, v25.s[3] \n" + "fmla v12.4s, v7.4s, v26.s[0] \n" + "fmla v13.4s, v7.4s, v26.s[1] \n" + "fmla v14.4s, v7.4s, v26.s[2] \n" + "fmla v15.4s, v7.4s, v26.s[3] \n" + "fmla v16.4s, v7.4s, v27.s[0] \n" + "fmla v17.4s, v7.4s, v27.s[1] \n" + "fmla v18.4s, v7.4s, v27.s[2] \n" + "fmla v19.4s, v7.4s, v27.s[3] \n" + + "bne 0b \n" + + "shrn v8.4h, v8.4s, #16 \n" + "shrn v9.4h, v9.4s, #16 \n" + "shrn v10.4h, v10.4s, #16 \n" + "shrn v11.4h, v11.4s, #16 \n" + + "shrn v12.4h, v12.4s, #16 \n" + "shrn v13.4h, v13.4s, #16 \n" + "shrn v14.4h, v14.4s, #16 \n" + "shrn v15.4h, v15.4s, #16 \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "st1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%1], #32 \n" + "st1 {v12.4h, v13.4h, v14.4h, v15.4h}, [%1], #32 \n" + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); + } +#endif // __aarch64__ + for (; i + 7 < size; i += 8) + { +#if __aarch64__ + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + const unsigned short* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const unsigned short* tmpptr = tmp.channel(i / 8); + const unsigned short* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + "mov v20.16b, v0.16b \n" + "mov v21.16b, v0.16b \n" + "mov v22.16b, v0.16b \n" + "mov v23.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [%2], #32 \n" // r4 r5 r6 r7 + + "shll v4.4s, v4.4h, #16 \n" + "shll v5.4s, v5.4h, #16 \n" + "shll v6.4s, v6.4h, #16 \n" + "shll v7.4s, v7.4h, #16 \n" + + "fmla v20.4s, v8.4s, v4.s[0] \n" + "fmla v21.4s, v8.4s, v5.s[0] \n" + "fmla v22.4s, v8.4s, v6.s[0] \n" + "fmla v23.4s, v8.4s, v7.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + "fmla v18.4s, v9.4s, v2.s[1] \n" + "fmla v19.4s, v9.4s, v3.s[1] \n" + "fmla v20.4s, v9.4s, v4.s[1] \n" + "fmla v21.4s, v9.4s, v5.s[1] \n" + "fmla v22.4s, v9.4s, v6.s[1] \n" + "fmla v23.4s, v9.4s, v7.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + "fmla v18.4s, v10.4s, v2.s[2] \n" + "fmla v19.4s, v10.4s, v3.s[2] \n" + "fmla v20.4s, v10.4s, v4.s[2] \n" + "fmla v21.4s, v10.4s, v5.s[2] \n" + "fmla v22.4s, v10.4s, v6.s[2] \n" + "fmla v23.4s, v10.4s, v7.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + "fmla v18.4s, v11.4s, v2.s[3] \n" + "fmla v19.4s, v11.4s, v3.s[3] \n" + "fmla v20.4s, v11.4s, v4.s[3] \n" + "fmla v21.4s, v11.4s, v5.s[3] \n" + "fmla v22.4s, v11.4s, v6.s[3] \n" + "fmla v23.4s, v11.4s, v7.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "shrn v20.4h, v20.4s, #16 \n" + "shrn v21.4h, v21.4s, #16 \n" + "shrn v22.4h, v22.4s, #16 \n" + "shrn v23.4h, v23.4s, #16 \n" + + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + "st1 {v20.4h, v21.4h, v22.4h, v23.4h}, [%1], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + "vmov q10, q0 \n" + "vmov q11, q0 \n" + "vmov q12, q0 \n" + "vmov q13, q0 \n" + "vmov q14, q0 \n" + "vmov q15, q0 \n" + + "0: \n" + + "pld [%2, #256] \n" + "vld1.u16 {d4-d7}, [%2]! \n" + + "pld [%3, #256] \n" + "vld1.u16 {d12-d15}, [%3]! \n" + + "vshll.u16 q0, d4, #16 \n" + "vshll.u16 q1, d5, #16 \n" + "vshll.u16 q2, d6, #16 \n" + "vshll.u16 q3, d7, #16 \n" + + "vshll.u16 q4, d12, #16 \n" + "vshll.u16 q5, d13, #16 \n" + "vshll.u16 q6, d14, #16 \n" + "vshll.u16 q7, d15, #16 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d0[1] \n" + "vmla.f32 q10, q4, d1[0] \n" + "vmla.f32 q11, q4, d1[1] \n" + "vmla.f32 q12, q4, d2[0] \n" + "vmla.f32 q13, q4, d2[1] \n" + "vmla.f32 q14, q4, d3[0] \n" + "vmla.f32 q15, q4, d3[1] \n" + + "vmla.f32 q8, q5, d4[0] \n" + "vmla.f32 q9, q5, d4[1] \n" + "vmla.f32 q10, q5, d5[0] \n" + "vmla.f32 q11, q5, d5[1] \n" + "vmla.f32 q12, q5, d6[0] \n" + "vmla.f32 q13, q5, d6[1] \n" + "vmla.f32 q14, q5, d7[0] \n" + "vmla.f32 q15, q5, d7[1] \n" + + "pld [%2, #256] \n" + "vld1.u16 {d4-d7}, [%2]! \n" + + "vshll.u16 q0, d4, #16 \n" + "vshll.u16 q1, d5, #16 \n" + "vshll.u16 q2, d6, #16 \n" + "vshll.u16 q3, d7, #16 \n" + + "vmla.f32 q8, q6, d0[0] \n" + "vmla.f32 q9, q6, d0[1] \n" + "vmla.f32 q10, q6, d1[0] \n" + "vmla.f32 q11, q6, d1[1] \n" + "vmla.f32 q12, q6, d2[0] \n" + "vmla.f32 q13, q6, d2[1] \n" + "vmla.f32 q14, q6, d3[0] \n" + "vmla.f32 q15, q6, d3[1] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d4[0] \n" + "vmla.f32 q9, q7, d4[1] \n" + "vmla.f32 q10, q7, d5[0] \n" + "vmla.f32 q11, q7, d5[1] \n" + "vmla.f32 q12, q7, d6[0] \n" + "vmla.f32 q13, q7, d6[1] \n" + "vmla.f32 q14, q7, d7[0] \n" + "vmla.f32 q15, q7, d7[1] \n" + + "bne 0b \n" + + "vshrn.u32 d16, q8, #16 \n" + "vshrn.u32 d17, q9, #16 \n" + "vshrn.u32 d18, q10, #16 \n" + "vshrn.u32 d19, q11, #16 \n" + + "vshrn.u32 d20, q12, #16 \n" + "vshrn.u32 d21, q13, #16 \n" + "vshrn.u32 d22, q14, #16 \n" + "vshrn.u32 d23, q15, #16 \n" + + "vstm %1!, {d16-d23} \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); +#endif + } + for (; i + 3 < size; i += 4) + { +#if __aarch64__ + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const unsigned short* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + const unsigned short* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + "mov v18.16b, v0.16b \n" + "mov v19.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [%2], #32 \n" // r0 r1 r2 r3 + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + "shll v2.4s, v2.4h, #16 \n" + "shll v3.4s, v3.4h, #16 \n" + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + "fmla v18.4s, v8.4s, v2.s[0] \n" + "fmla v19.4s, v8.4s, v3.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + "fmla v18.4s, v9.4s, v2.s[1] \n" + "fmla v19.4s, v9.4s, v3.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + "fmla v18.4s, v10.4s, v2.s[2] \n" + "fmla v19.4s, v10.4s, v3.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + "fmla v18.4s, v11.4s, v2.s[3] \n" + "fmla v19.4s, v11.4s, v3.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + "shrn v18.4h, v18.4s, #16 \n" + "shrn v19.4h, v19.4s, #16 \n" + + "st1 {v16.4h, v17.4h, v18.4h, v19.4h}, [%1], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + "vmov q10, q0 \n" + "vmov q11, q0 \n" + + "0: \n" + + "pld [%2, #256] \n" + "vld1.u16 {d4-d7}, [%2]! \n" + + "pld [%3, #256] \n" + "vld1.u16 {d12-d15}, [%3]! \n" + + "vshll.u16 q0, d4, #16 \n" + "vshll.u16 q1, d5, #16 \n" + "vshll.u16 q2, d6, #16 \n" + "vshll.u16 q3, d7, #16 \n" + + "vshll.u16 q4, d12, #16 \n" + "vshll.u16 q5, d13, #16 \n" + "vshll.u16 q6, d14, #16 \n" + "vshll.u16 q7, d15, #16 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d2[0] \n" + "vmla.f32 q10, q4, d4[0] \n" + "vmla.f32 q11, q4, d6[0] \n" + + "vmla.f32 q8, q5, d0[1] \n" + "vmla.f32 q9, q5, d2[1] \n" + "vmla.f32 q10, q5, d4[1] \n" + "vmla.f32 q11, q5, d6[1] \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q9, q6, d3[0] \n" + "vmla.f32 q10, q6, d5[0] \n" + "vmla.f32 q11, q6, d7[0] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d1[1] \n" + "vmla.f32 q9, q7, d3[1] \n" + "vmla.f32 q10, q7, d5[1] \n" + "vmla.f32 q11, q7, d7[1] \n" + + "bne 0b \n" + + "vshrn.u32 d16, q8, #16 \n" + "vshrn.u32 d17, q9, #16 \n" + "vshrn.u32 d18, q10, #16 \n" + "vshrn.u32 d19, q11, #16 \n" + + "vst1.u16 {d16-d19}, [%1]! \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"); +#endif + } + for (; i + 1 < size; i += 2) + { +#if __aarch64__ + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const unsigned short* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + const unsigned short* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v0.4s}, [%8] \n" + "mov v16.16b, v0.16b \n" + "mov v17.16b, v0.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v0.4h, v1.4h}, [%2], #16 \n" // r0 r1 + + "shll v0.4s, v0.4h, #16 \n" + "shll v1.4s, v1.4h, #16 \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v17.4s, v8.4s, v1.s[0] \n" + + "fmla v16.4s, v9.4s, v0.s[1] \n" + "fmla v17.4s, v9.4s, v1.s[1] \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v17.4s, v10.4s, v1.s[2] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v11.4s, v0.s[3] \n" + "fmla v17.4s, v11.4s, v1.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + "shrn v17.4h, v17.4s, #16 \n" + + "st1 {v16.4h, v17.4h}, [%1], #16 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v16", "v17"); +#else + asm volatile( + "vld1.f32 {d0-d1}, [%8] \n" + "vmov q8, q0 \n" + "vmov q9, q0 \n" + + "0: \n" + + "pld [%2, #128] \n" + "vld1.u16 {d4-d5}, [%2 :128]! \n" + + "vshll.u16 q0, d4, #16 \n" + "vshll.u16 q1, d5, #16 \n" + + "pld [%3, #256] \n" + "vld1.u16 {d12-d15}, [%3]! \n" + + "vshll.u16 q4, d12, #16 \n" + "vshll.u16 q5, d13, #16 \n" + "vshll.u16 q6, d14, #16 \n" + "vshll.u16 q7, d15, #16 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q9, q4, d2[0] \n" + + "vmla.f32 q8, q5, d0[1] \n" + "vmla.f32 q9, q5, d2[1] \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q9, q6, d3[0] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q7, d1[1] \n" + "vmla.f32 q9, q7, d3[1] \n" + + "bne 0b \n" + + "vshrn.u32 d16, q8, #16 \n" + "vshrn.u32 d17, q9, #16 \n" + + "vst1.u16 {d16-d17}, [%1 :128]! \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q1", "q4", "q5", "q6", "q7", "q8", "q9"); +#endif + } + for (; i < size; i++) + { +#if __aarch64__ + const unsigned short* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const unsigned short* kptr0 = kernel.channel(p / 2 + p % 2); +#else + const unsigned short* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + const unsigned short* kptr0 = kernel.channel(p); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __aarch64__ + asm volatile( + "ld1 {v16.4s}, [%8] \n" + + "0: \n" + + "prfm pldl1keep, [%2, #64] \n" + "ld1 {v0.4h}, [%2], #8 \n" // r0 + + "shll v0.4s, v0.4h, #16 \n" + + "prfm pldl1keep, [%3, #256] \n" + "ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [%3], #32 \n" // w0123 + + "shll v8.4s, v8.4h, #16 \n" + "shll v9.4s, v9.4h, #16 \n" + "shll v10.4s, v10.4h, #16 \n" + "shll v11.4s, v11.4h, #16 \n" + + "fmla v16.4s, v8.4s, v0.s[0] \n" + "fmla v16.4s, v9.4s, v0.s[1] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.4s, v10.4s, v0.s[2] \n" + "fmla v16.4s, v11.4s, v0.s[3] \n" + + "bne 0b \n" + + "shrn v16.4h, v16.4s, #16 \n" + + "st1 {v16.4h}, [%1], #8 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v16"); +#else + asm volatile( + "vld1.f32 {d16-d17}, [%8] \n" + + "0: \n" + + "pld [%2, #64] \n" + "vld1.u16 {d1}, [%2 :64]! \n" + + "vshll.u16 q0, d1, #16 \n" + + "pld [%3, #256] \n" + "vld1.u16 {d12-d15}, [%3]! \n" + + "vshll.u16 q4, d12, #16 \n" + "vshll.u16 q5, d13, #16 \n" + "vshll.u16 q6, d14, #16 \n" + "vshll.u16 q7, d15, #16 \n" + + "vmla.f32 q8, q4, d0[0] \n" + "vmla.f32 q8, q5, d0[1] \n" + + "subs %0, %0, #1 \n" + + "vmla.f32 q8, q6, d1[0] \n" + "vmla.f32 q8, q7, d1[1] \n" + + "bne 0b \n" + + "vshrn.u32 d16, q8, #16 \n" + + "vst1.u16 {d16}, [%1 :64]! \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "q0", "q4", "q5", "q6", "q7", "q8"); +#endif + } + } +} + +static void convolution_im2col_sgemm_transform_kernel_pack4_bf16s_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // interleave + // src = maxk-inch-outch + // dst = 4b-4a-maxk-inch/4a-outch/4b + Mat kernel = _kernel.reshape(maxk, inch, outch); +#if __aarch64__ + kernel_tm.create(32 * maxk, inch / 4, outch / 8 + (outch % 8) / 4, 2u); +#else + kernel_tm.create(16 * maxk, inch / 4, outch / 4, 2u); +#endif + + int q = 0; +#if __aarch64__ + for (; q + 7 < outch; q += 8) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); + const Mat k4 = kernel.channel(q + 4); + const Mat k5 = kernel.channel(q + 5); + const Mat k6 = kernel.channel(q + 6); + const Mat k7 = kernel.channel(q + 7); + + unsigned short* g00 = kernel_tm.channel(q / 8); + + for (int p = 0; p + 3 < inch; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + const float* k40 = k4.row(p); + const float* k41 = k4.row(p + 1); + const float* k42 = k4.row(p + 2); + const float* k43 = k4.row(p + 3); + + const float* k50 = k5.row(p); + const float* k51 = k5.row(p + 1); + const float* k52 = k5.row(p + 2); + const float* k53 = k5.row(p + 3); + + const float* k60 = k6.row(p); + const float* k61 = k6.row(p + 1); + const float* k62 = k6.row(p + 2); + const float* k63 = k6.row(p + 3); + + const float* k70 = k7.row(p); + const float* k71 = k7.row(p + 1); + const float* k72 = k7.row(p + 2); + const float* k73 = k7.row(p + 3); + + for (int k = 0; k < maxk; k++) + { + g00[0] = float32_to_bfloat16(k00[k]); + g00[1] = float32_to_bfloat16(k10[k]); + g00[2] = float32_to_bfloat16(k20[k]); + g00[3] = float32_to_bfloat16(k30[k]); + g00[4] = float32_to_bfloat16(k40[k]); + g00[5] = float32_to_bfloat16(k50[k]); + g00[6] = float32_to_bfloat16(k60[k]); + g00[7] = float32_to_bfloat16(k70[k]); + + g00[8] = float32_to_bfloat16(k01[k]); + g00[9] = float32_to_bfloat16(k11[k]); + g00[10] = float32_to_bfloat16(k21[k]); + g00[11] = float32_to_bfloat16(k31[k]); + g00[12] = float32_to_bfloat16(k41[k]); + g00[13] = float32_to_bfloat16(k51[k]); + g00[14] = float32_to_bfloat16(k61[k]); + g00[15] = float32_to_bfloat16(k71[k]); + + g00[16] = float32_to_bfloat16(k02[k]); + g00[17] = float32_to_bfloat16(k12[k]); + g00[18] = float32_to_bfloat16(k22[k]); + g00[19] = float32_to_bfloat16(k32[k]); + g00[20] = float32_to_bfloat16(k42[k]); + g00[21] = float32_to_bfloat16(k52[k]); + g00[22] = float32_to_bfloat16(k62[k]); + g00[23] = float32_to_bfloat16(k72[k]); + + g00[24] = float32_to_bfloat16(k03[k]); + g00[25] = float32_to_bfloat16(k13[k]); + g00[26] = float32_to_bfloat16(k23[k]); + g00[27] = float32_to_bfloat16(k33[k]); + g00[28] = float32_to_bfloat16(k43[k]); + g00[29] = float32_to_bfloat16(k53[k]); + g00[30] = float32_to_bfloat16(k63[k]); + g00[31] = float32_to_bfloat16(k73[k]); + + g00 += 32; + } + } + } +#endif // __aarch64__ + for (; q + 3 < outch; q += 4) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + const Mat k2 = kernel.channel(q + 2); + const Mat k3 = kernel.channel(q + 3); + +#if __aarch64__ + unsigned short* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4); +#else + unsigned short* g00 = kernel_tm.channel(q / 4); +#endif + + for (int p = 0; p + 3 < inch; p += 4) + { + const float* k00 = k0.row(p); + const float* k01 = k0.row(p + 1); + const float* k02 = k0.row(p + 2); + const float* k03 = k0.row(p + 3); + + const float* k10 = k1.row(p); + const float* k11 = k1.row(p + 1); + const float* k12 = k1.row(p + 2); + const float* k13 = k1.row(p + 3); + + const float* k20 = k2.row(p); + const float* k21 = k2.row(p + 1); + const float* k22 = k2.row(p + 2); + const float* k23 = k2.row(p + 3); + + const float* k30 = k3.row(p); + const float* k31 = k3.row(p + 1); + const float* k32 = k3.row(p + 2); + const float* k33 = k3.row(p + 3); + + for (int k = 0; k < maxk; k++) + { + g00[0] = float32_to_bfloat16(k00[k]); + g00[1] = float32_to_bfloat16(k10[k]); + g00[2] = float32_to_bfloat16(k20[k]); + g00[3] = float32_to_bfloat16(k30[k]); + + g00[4] = float32_to_bfloat16(k01[k]); + g00[5] = float32_to_bfloat16(k11[k]); + g00[6] = float32_to_bfloat16(k21[k]); + g00[7] = float32_to_bfloat16(k31[k]); + + g00[8] = float32_to_bfloat16(k02[k]); + g00[9] = float32_to_bfloat16(k12[k]); + g00[10] = float32_to_bfloat16(k22[k]); + g00[11] = float32_to_bfloat16(k32[k]); + + g00[12] = float32_to_bfloat16(k03[k]); + g00[13] = float32_to_bfloat16(k13[k]); + g00[14] = float32_to_bfloat16(k23[k]); + g00[15] = float32_to_bfloat16(k33[k]); + + g00 += 16; + } + } + } +} + +static void convolution_im2col_sgemm_pack4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 8u, 4, opt.workspace_allocator); + { + const int gap = (w * stride_h - outw * stride_w) * 4; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + unsigned short* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) + { + const unsigned short* sptr = img.row(dilation_h * u) + dilation_w * v * 4; + + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j + 3 < outw; j += 4) + { + uint16x4_t _val0 = vld1_u16(sptr); + uint16x4_t _val1 = vld1_u16(sptr + stride_w * 4); + uint16x4_t _val2 = vld1_u16(sptr + stride_w * 8); + uint16x4_t _val3 = vld1_u16(sptr + stride_w * 12); + vst1_u16(ptr, _val0); + vst1_u16(ptr + 4, _val1); + vst1_u16(ptr + 8, _val2); + vst1_u16(ptr + 12, _val3); + + sptr += stride_w * 16; + ptr += 16; + } + for (; j + 1 < outw; j += 2) + { + uint16x4_t _val0 = vld1_u16(sptr); + uint16x4_t _val1 = vld1_u16(sptr + stride_w * 4); + vst1_u16(ptr, _val0); + vst1_u16(ptr + 4, _val1); + + sptr += stride_w * 8; + ptr += 8; + } + for (; j < outw; j++) + { + uint16x4_t _val = vld1_u16(sptr); + vst1_u16(ptr, _val); + + sptr += stride_w * 4; + ptr += 4; + } + + sptr += gap; + } + } + } + } + } + + im2col_sgemm_pack4_bf16s_neon(bottom_im2col, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/arm/convolution_sgemm_pack8_fp16s.h b/src/layer/arm/convolution_sgemm_pack8_fp16s.h new file mode 100644 index 000000000..b7b143014 --- /dev/null +++ b/src/layer/arm/convolution_sgemm_pack8_fp16s.h @@ -0,0 +1,819 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void im2col_sgemm_pack8_fp16sa_neon(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + // Mat bottom_im2col(size, maxk, inch, 16u, 8, opt.workspace_allocator); + + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const __fp16* bias = _bias; + + // permute + Mat tmp; + if (size >= 12) + tmp.create(12 * maxk, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, 16u, 8, opt.workspace_allocator); + else if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 16u, 8, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 16u, 8, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 16u, 8, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 16u, 8, opt.workspace_allocator); + { + int nn_size = size / 12; + int remain_size_start = 0; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 12; + + __fp16* tmpptr = tmp.channel(i / 12); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8; + + for (int k = 0; k < maxk; k++) + { + // transpose 12x8 + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n" + "ld4 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n" + + "sub %0, %0, #128 \n" + + "uzp1 v20.8h, v0.8h, v4.8h \n" // 0 + "uzp1 v21.8h, v16.8h, v1.8h \n" // 1 + "uzp1 v22.8h, v5.8h, v17.8h \n" // 2 + "uzp1 v23.8h, v2.8h, v6.8h \n" // 3 + "uzp1 v24.8h, v18.8h, v3.8h \n" // 4 + "uzp1 v25.8h, v7.8h, v19.8h \n" // 5 + "uzp2 v26.8h, v0.8h, v4.8h \n" // 6 + "uzp2 v27.8h, v16.8h, v1.8h \n" // 7 + "uzp2 v28.8h, v5.8h, v17.8h \n" // 8 + "uzp2 v29.8h, v2.8h, v6.8h \n" // 9 + "uzp2 v30.8h, v18.8h, v3.8h \n" // 10 + "uzp2 v31.8h, v7.8h, v19.8h \n" // 11 + + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + img0 += size * 8; + } + } + } + + remain_size_start += nn_size * 12; + nn_size = (size - remain_size_start) >> 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + + __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8; + + for (int k = 0; k < maxk; k++) + { + // transpose 8x8 + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" + "sub %0, %0, #64 \n" + + "uzp1 v16.8h, v0.8h, v4.8h \n" + "uzp2 v20.8h, v0.8h, v4.8h \n" + "uzp1 v17.8h, v1.8h, v5.8h \n" + "uzp2 v21.8h, v1.8h, v5.8h \n" + "uzp1 v18.8h, v2.8h, v6.8h \n" + "uzp2 v22.8h, v2.8h, v6.8h \n" + "uzp1 v19.8h, v3.8h, v7.8h \n" + "uzp2 v23.8h, v3.8h, v7.8h \n" + + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + img0 += size * 8; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + + __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8; + + for (int k = 0; k < maxk; k++) + { + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3"); + img0 += size * 8; + } + } + } + + remain_size_start += nn_size << 2; + nn_size = (size - remain_size_start) >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 2; + + __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8; + + for (int k = 0; k < maxk; k++) + { + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.8h, v1.8h}, [%0] \n" + "st1 {v0.8h, v1.8h}, [%1], #32 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0", "v1"); + img0 += size * 8; + } + } + } + + remain_size_start += nn_size << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { + __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8; + + for (int k = 0; k < maxk; k++) + { + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.8h}, [%0] \n" + "st1 {v0.8h}, [%1], #16 \n" + : "=r"(img0), // %0 + "=r"(tmpptr) // %1 + : "0"(img0), + "1"(tmpptr) + : "memory", "v0"); + img0 += size * 8; + } + } + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr0 = top_blob.channel(p); + + const __fp16 zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + const __fp16* biasptr = bias ? bias + p * 8 : zeros; + + int i = 0; + for (; i + 11 < size; i += 12) + { + const __fp16* tmpptr = tmp.channel(i / 12); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v20.8h}, [%8] \n" + "mov v21.16b, v20.16b \n" + "mov v22.16b, v20.16b \n" + "mov v23.16b, v20.16b \n" + "mov v24.16b, v20.16b \n" + "mov v25.16b, v20.16b \n" + "mov v26.16b, v20.16b \n" + "mov v27.16b, v20.16b \n" + "mov v28.16b, v20.16b \n" + "mov v29.16b, v20.16b \n" + "mov v30.16b, v20.16b \n" + "mov v31.16b, v20.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w0123 + + "fmla v20.8h, v12.8h, v0.h[0] \n" + "fmla v21.8h, v12.8h, v0.h[1] \n" + "fmla v22.8h, v12.8h, v0.h[2] \n" + "fmla v23.8h, v12.8h, v0.h[3] \n" + "fmla v24.8h, v12.8h, v0.h[4] \n" + "fmla v25.8h, v12.8h, v0.h[5] \n" + "fmla v26.8h, v12.8h, v0.h[6] \n" + "fmla v27.8h, v12.8h, v0.h[7] \n" + "fmla v28.8h, v12.8h, v1.h[0] \n" + "fmla v29.8h, v12.8h, v1.h[1] \n" + "fmla v30.8h, v12.8h, v1.h[2] \n" + "fmla v31.8h, v12.8h, v1.h[3] \n" + + "fmla v20.8h, v13.8h, v1.h[4] \n" + "fmla v21.8h, v13.8h, v1.h[5] \n" + "fmla v22.8h, v13.8h, v1.h[6] \n" + "fmla v23.8h, v13.8h, v1.h[7] \n" + "fmla v24.8h, v13.8h, v2.h[0] \n" + "fmla v25.8h, v13.8h, v2.h[1] \n" + "fmla v26.8h, v13.8h, v2.h[2] \n" + "fmla v27.8h, v13.8h, v2.h[3] \n" + "fmla v28.8h, v13.8h, v2.h[4] \n" + "fmla v29.8h, v13.8h, v2.h[5] \n" + "fmla v30.8h, v13.8h, v2.h[6] \n" + "fmla v31.8h, v13.8h, v2.h[7] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567 + + "fmla v20.8h, v14.8h, v3.h[0] \n" + "fmla v21.8h, v14.8h, v3.h[1] \n" + "fmla v22.8h, v14.8h, v3.h[2] \n" + "fmla v23.8h, v14.8h, v3.h[3] \n" + "fmla v24.8h, v14.8h, v3.h[4] \n" + "fmla v25.8h, v14.8h, v3.h[5] \n" + "fmla v26.8h, v14.8h, v3.h[6] \n" + "fmla v27.8h, v14.8h, v3.h[7] \n" + "fmla v28.8h, v14.8h, v4.h[0] \n" + "fmla v29.8h, v14.8h, v4.h[1] \n" + "fmla v30.8h, v14.8h, v4.h[2] \n" + "fmla v31.8h, v14.8h, v4.h[3] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%3], #64 \n" // w4567 + + "fmla v20.8h, v15.8h, v4.h[4] \n" + "fmla v21.8h, v15.8h, v4.h[5] \n" + "fmla v22.8h, v15.8h, v4.h[6] \n" + "fmla v23.8h, v15.8h, v4.h[7] \n" + "fmla v24.8h, v15.8h, v5.h[0] \n" + "fmla v25.8h, v15.8h, v5.h[1] \n" + "fmla v26.8h, v15.8h, v5.h[2] \n" + "fmla v27.8h, v15.8h, v5.h[3] \n" + "fmla v28.8h, v15.8h, v5.h[4] \n" + "fmla v29.8h, v15.8h, v5.h[5] \n" + "fmla v30.8h, v15.8h, v5.h[6] \n" + "fmla v31.8h, v15.8h, v5.h[7] \n" + + "fmla v20.8h, v16.8h, v6.h[0] \n" + "fmla v21.8h, v16.8h, v6.h[1] \n" + "fmla v22.8h, v16.8h, v6.h[2] \n" + "fmla v23.8h, v16.8h, v6.h[3] \n" + "fmla v24.8h, v16.8h, v6.h[4] \n" + "fmla v25.8h, v16.8h, v6.h[5] \n" + "fmla v26.8h, v16.8h, v6.h[6] \n" + "fmla v27.8h, v16.8h, v6.h[7] \n" + "fmla v28.8h, v16.8h, v7.h[0] \n" + "fmla v29.8h, v16.8h, v7.h[1] \n" + "fmla v30.8h, v16.8h, v7.h[2] \n" + "fmla v31.8h, v16.8h, v7.h[3] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%2], #64 \n" // r891011 + + "fmla v20.8h, v17.8h, v7.h[4] \n" + "fmla v21.8h, v17.8h, v7.h[5] \n" + "fmla v22.8h, v17.8h, v7.h[6] \n" + "fmla v23.8h, v17.8h, v7.h[7] \n" + "fmla v24.8h, v17.8h, v8.h[0] \n" + "fmla v25.8h, v17.8h, v8.h[1] \n" + "fmla v26.8h, v17.8h, v8.h[2] \n" + "fmla v27.8h, v17.8h, v8.h[3] \n" + "fmla v28.8h, v17.8h, v8.h[4] \n" + "fmla v29.8h, v17.8h, v8.h[5] \n" + "fmla v30.8h, v17.8h, v8.h[6] \n" + "fmla v31.8h, v17.8h, v8.h[7] \n" + + "fmla v20.8h, v18.8h, v9.h[0] \n" + "fmla v21.8h, v18.8h, v9.h[1] \n" + "fmla v22.8h, v18.8h, v9.h[2] \n" + "fmla v23.8h, v18.8h, v9.h[3] \n" + "fmla v24.8h, v18.8h, v9.h[4] \n" + "fmla v25.8h, v18.8h, v9.h[5] \n" + "fmla v26.8h, v18.8h, v9.h[6] \n" + "fmla v27.8h, v18.8h, v9.h[7] \n" + "fmla v28.8h, v18.8h, v10.h[0] \n" + "fmla v29.8h, v18.8h, v10.h[1] \n" + "fmla v30.8h, v18.8h, v10.h[2] \n" + "fmla v31.8h, v18.8h, v10.h[3] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v20.8h, v19.8h, v10.h[4] \n" + "fmla v21.8h, v19.8h, v10.h[5] \n" + "fmla v22.8h, v19.8h, v10.h[6] \n" + "fmla v23.8h, v19.8h, v10.h[7] \n" + "fmla v24.8h, v19.8h, v11.h[0] \n" + "fmla v25.8h, v19.8h, v11.h[1] \n" + "fmla v26.8h, v19.8h, v11.h[2] \n" + "fmla v27.8h, v19.8h, v11.h[3] \n" + "fmla v28.8h, v19.8h, v11.h[4] \n" + "fmla v29.8h, v19.8h, v11.h[5] \n" + "fmla v30.8h, v19.8h, v11.h[6] \n" + "fmla v31.8h, v19.8h, v11.h[7] \n" + + "bne 0b \n" + + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 7 < size; i += 8) + { + const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.8h}, [%8] \n" + "mov v17.16b, v16.16b \n" + "mov v18.16b, v16.16b \n" + "mov v19.16b, v16.16b \n" + "mov v20.16b, v16.16b \n" + "mov v21.16b, v16.16b \n" + "mov v22.16b, v16.16b \n" + "mov v23.16b, v16.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 + + "fmla v16.8h, v8.8h, v0.h[0] \n" + "fmla v17.8h, v8.8h, v0.h[1] \n" + "fmla v18.8h, v8.8h, v0.h[2] \n" + "fmla v19.8h, v8.8h, v0.h[3] \n" + "fmla v20.8h, v8.8h, v0.h[4] \n" + "fmla v21.8h, v8.8h, v0.h[5] \n" + "fmla v22.8h, v8.8h, v0.h[6] \n" + "fmla v23.8h, v8.8h, v0.h[7] \n" + + "fmla v16.8h, v9.8h, v1.h[0] \n" + "fmla v17.8h, v9.8h, v1.h[1] \n" + "fmla v18.8h, v9.8h, v1.h[2] \n" + "fmla v19.8h, v9.8h, v1.h[3] \n" + "fmla v20.8h, v9.8h, v1.h[4] \n" + "fmla v21.8h, v9.8h, v1.h[5] \n" + "fmla v22.8h, v9.8h, v1.h[6] \n" + "fmla v23.8h, v9.8h, v1.h[7] \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567 + + "fmla v16.8h, v10.8h, v2.h[0] \n" + "fmla v17.8h, v10.8h, v2.h[1] \n" + "fmla v18.8h, v10.8h, v2.h[2] \n" + "fmla v19.8h, v10.8h, v2.h[3] \n" + "fmla v20.8h, v10.8h, v2.h[4] \n" + "fmla v21.8h, v10.8h, v2.h[5] \n" + "fmla v22.8h, v10.8h, v2.h[6] \n" + "fmla v23.8h, v10.8h, v2.h[7] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 + + "fmla v16.8h, v11.8h, v3.h[0] \n" + "fmla v17.8h, v11.8h, v3.h[1] \n" + "fmla v18.8h, v11.8h, v3.h[2] \n" + "fmla v19.8h, v11.8h, v3.h[3] \n" + "fmla v20.8h, v11.8h, v3.h[4] \n" + "fmla v21.8h, v11.8h, v3.h[5] \n" + "fmla v22.8h, v11.8h, v3.h[6] \n" + "fmla v23.8h, v11.8h, v3.h[7] \n" + + "fmla v16.8h, v12.8h, v4.h[0] \n" + "fmla v17.8h, v12.8h, v4.h[1] \n" + "fmla v18.8h, v12.8h, v4.h[2] \n" + "fmla v19.8h, v12.8h, v4.h[3] \n" + "fmla v20.8h, v12.8h, v4.h[4] \n" + "fmla v21.8h, v12.8h, v4.h[5] \n" + "fmla v22.8h, v12.8h, v4.h[6] \n" + "fmla v23.8h, v12.8h, v4.h[7] \n" + + "fmla v16.8h, v13.8h, v5.h[0] \n" + "fmla v17.8h, v13.8h, v5.h[1] \n" + "fmla v18.8h, v13.8h, v5.h[2] \n" + "fmla v19.8h, v13.8h, v5.h[3] \n" + "fmla v20.8h, v13.8h, v5.h[4] \n" + "fmla v21.8h, v13.8h, v5.h[5] \n" + "fmla v22.8h, v13.8h, v5.h[6] \n" + "fmla v23.8h, v13.8h, v5.h[7] \n" + + "fmla v16.8h, v14.8h, v6.h[0] \n" + "fmla v17.8h, v14.8h, v6.h[1] \n" + "fmla v18.8h, v14.8h, v6.h[2] \n" + "fmla v19.8h, v14.8h, v6.h[3] \n" + "fmla v20.8h, v14.8h, v6.h[4] \n" + "fmla v21.8h, v14.8h, v6.h[5] \n" + "fmla v22.8h, v14.8h, v6.h[6] \n" + "fmla v23.8h, v14.8h, v6.h[7] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.8h, v15.8h, v7.h[0] \n" + "fmla v17.8h, v15.8h, v7.h[1] \n" + "fmla v18.8h, v15.8h, v7.h[2] \n" + "fmla v19.8h, v15.8h, v7.h[3] \n" + "fmla v20.8h, v15.8h, v7.h[4] \n" + "fmla v21.8h, v15.8h, v7.h[5] \n" + "fmla v22.8h, v15.8h, v7.h[6] \n" + "fmla v23.8h, v15.8h, v7.h[7] \n" + + "bne 0b \n" + + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + } + for (; i + 3 < size; i += 4) + { + const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.8h}, [%8] \n" + "mov v17.16b, v16.16b \n" + "mov v18.16b, v16.16b \n" + "mov v19.16b, v16.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 + + "fmla v16.8h, v8.8h, v0.h[0] \n" + "fmla v17.8h, v8.8h, v1.h[0] \n" + "fmla v18.8h, v8.8h, v2.h[0] \n" + "fmla v19.8h, v8.8h, v3.h[0] \n" + + "fmla v16.8h, v9.8h, v0.h[1] \n" + "fmla v17.8h, v9.8h, v1.h[1] \n" + "fmla v18.8h, v9.8h, v2.h[1] \n" + "fmla v19.8h, v9.8h, v3.h[1] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 + + "fmla v16.8h, v10.8h, v0.h[2] \n" + "fmla v17.8h, v10.8h, v1.h[2] \n" + "fmla v18.8h, v10.8h, v2.h[2] \n" + "fmla v19.8h, v10.8h, v3.h[2] \n" + + "fmla v16.8h, v11.8h, v0.h[3] \n" + "fmla v17.8h, v11.8h, v1.h[3] \n" + "fmla v18.8h, v11.8h, v2.h[3] \n" + "fmla v19.8h, v11.8h, v3.h[3] \n" + + "fmla v16.8h, v12.8h, v0.h[4] \n" + "fmla v17.8h, v12.8h, v1.h[4] \n" + "fmla v18.8h, v12.8h, v2.h[4] \n" + "fmla v19.8h, v12.8h, v3.h[4] \n" + + "fmla v16.8h, v13.8h, v0.h[5] \n" + "fmla v17.8h, v13.8h, v1.h[5] \n" + "fmla v18.8h, v13.8h, v2.h[5] \n" + "fmla v19.8h, v13.8h, v3.h[5] \n" + + "fmla v16.8h, v14.8h, v0.h[6] \n" + "fmla v17.8h, v14.8h, v1.h[6] \n" + "fmla v18.8h, v14.8h, v2.h[6] \n" + "fmla v19.8h, v14.8h, v3.h[6] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.8h, v15.8h, v0.h[7] \n" + "fmla v17.8h, v15.8h, v1.h[7] \n" + "fmla v18.8h, v15.8h, v2.h[7] \n" + "fmla v19.8h, v15.8h, v3.h[7] \n" + + "bne 0b \n" + + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } + for (; i + 1 < size; i += 2) + { + const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.8h}, [%8] \n" + "mov v17.16b, v16.16b \n" + + "0: \n" + + "prfm pldl1keep, [%2, #256] \n" + "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 + + "fmla v16.8h, v8.8h, v0.h[0] \n" + "fmla v17.8h, v8.8h, v1.h[0] \n" + + "fmla v16.8h, v9.8h, v0.h[1] \n" + "fmla v17.8h, v9.8h, v1.h[1] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 + + "fmla v16.8h, v10.8h, v0.h[2] \n" + "fmla v17.8h, v10.8h, v1.h[2] \n" + + "fmla v16.8h, v11.8h, v0.h[3] \n" + "fmla v17.8h, v11.8h, v1.h[3] \n" + + "fmla v16.8h, v12.8h, v0.h[4] \n" + "fmla v17.8h, v12.8h, v1.h[4] \n" + + "fmla v16.8h, v13.8h, v0.h[5] \n" + "fmla v17.8h, v13.8h, v1.h[5] \n" + + "fmla v16.8h, v14.8h, v0.h[6] \n" + "fmla v17.8h, v14.8h, v1.h[6] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.8h, v15.8h, v0.h[7] \n" + "fmla v17.8h, v15.8h, v1.h[7] \n" + + "bne 0b \n" + + "st1 {v16.8h, v17.8h}, [%1], #32 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); + } + for (; i < size; i++) + { + const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk; // inch always > 0 + + asm volatile( + "ld1 {v16.8h}, [%8] \n" + + "0: \n" + + "prfm pldl1keep, [%2, #128] \n" + "ld1 {v0.8h}, [%2], #16 \n" // r0 + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123 + + "fmla v16.8h, v8.8h, v0.h[0] \n" + "fmla v16.8h, v9.8h, v0.h[1] \n" + + "prfm pldl1keep, [%3, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567 + + "fmla v16.8h, v10.8h, v0.h[2] \n" + "fmla v16.8h, v11.8h, v0.h[3] \n" + + "fmla v16.8h, v12.8h, v0.h[4] \n" + "fmla v16.8h, v13.8h, v0.h[5] \n" + + "subs %w0, %w0, #1 \n" + + "fmla v16.8h, v14.8h, v0.h[6] \n" + "fmla v16.8h, v15.8h, v0.h[7] \n" + + "bne 0b \n" + + "st1 {v16.8h}, [%1], #16 \n" + + : "=r"(nn), // %0 + "=r"(outptr0), // %1 + "=r"(tmpptr), // %2 + "=r"(kptr0) // %3 + : "0"(nn), + "1"(outptr0), + "2"(tmpptr), + "3"(kptr0), + "r"(biasptr) // %8 + : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16"); + } + } +} + +static void convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) +{ + const int maxk = kernel_w * kernel_h; + + // interleave + // src = maxk-inch-outch + // dst = 8b-8a-maxk-inch/8a-outch/8b + Mat kernel = _kernel.reshape(maxk, inch, outch); + kernel_tm.create(64 * maxk, inch / 8, outch / 8, 2u); + + for (int q = 0; q + 7 < outch; q += 8) + { + Mat g0 = kernel_tm.channel(q / 8); + + for (int p = 0; p + 7 < inch; p += 8) + { + __fp16* g00 = g0.row<__fp16>(p / 8); + + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < 8; i++) + { + for (int j = 0; j < 8; j++) + { + const float* k00 = kernel.channel(q + j).row(p + i); + + g00[0] = (__fp16)k00[k]; + + g00++; + } + } + } + } + } +} + +static void convolution_im2col_sgemm_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 16u, 8, opt.workspace_allocator); + { + const int gap = (w * stride_h - outw * stride_w) * 8; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + __fp16* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) + { + const __fp16* sptr = img.row(dilation_h * u) + dilation_w * v * 8; + + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j + 3 < outw; j += 4) + { + float16x8_t _val0 = vld1q_f16(sptr); + float16x8_t _val1 = vld1q_f16(sptr + stride_w * 8); + float16x8_t _val2 = vld1q_f16(sptr + stride_w * 16); + float16x8_t _val3 = vld1q_f16(sptr + stride_w * 24); + vst1q_f16(ptr, _val0); + vst1q_f16(ptr + 8, _val1); + vst1q_f16(ptr + 16, _val2); + vst1q_f16(ptr + 24, _val3); + + sptr += stride_w * 32; + ptr += 32; + } + for (; j + 1 < outw; j += 2) + { + float16x8_t _val0 = vld1q_f16(sptr); + float16x8_t _val1 = vld1q_f16(sptr + stride_w * 8); + vst1q_f16(ptr, _val0); + vst1q_f16(ptr + 8, _val1); + + sptr += stride_w * 16; + ptr += 16; + } + for (; j < outw; j++) + { + float16x8_t _val = vld1q_f16(sptr); + vst1q_f16(ptr, _val); + + sptr += stride_w * 8; + ptr += 4; + } + + sptr += gap; + } + } + } + } + } + + im2col_sgemm_pack8_fp16sa_neon(bottom_im2col, top_blob, kernel, _bias, opt); +} diff --git a/tests/test_convolution.cpp b/tests/test_convolution.cpp index 953eeefaf..88154719b 100644 --- a/tests/test_convolution.cpp +++ b/tests/test_convolution.cpp @@ -117,7 +117,8 @@ static int test_convolution_0() || test_convolution(13, 16, 16, 24, 3, 1, 1, 1, 1) || test_convolution(8, 8, 16, 24, 3, 1, 1, 1, 0) || test_convolution(4, 8, 16, 24, 3, 1, 1, 1, 1) - || test_convolution(4, 20, 16, 24, 3, 1, 1, 1, 0); + || test_convolution(4, 20, 16, 24, 3, 1, 1, 1, 0) + || test_convolution(6, 7, 64, 64, 3, 1, 2, 0, 1); } static int test_convolution_vec(int w, int outch, int kernel, int dilation, int stride, int pad, int bias)