From 9b8272e86d2c836fbf881dcb3baada9ed0a879fd Mon Sep 17 00:00:00 2001 From: nihui Date: Mon, 11 Jul 2022 17:10:17 +0800 Subject: [PATCH] arm edsp and arm neon optimization for convolution int8 winograd (#4017) --- src/layer/arm/convolution_3x3_int8.h | 3563 +---------------- src/layer/arm/convolution_3x3_pack8to1_int8.h | 995 +---- src/layer/arm/convolution_3x3_pack8to4_int8.h | 2109 +--------- src/layer/arm/convolution_arm.cpp | 139 +- src/layer/arm/convolution_arm.h | 3 +- src/layer/arm/convolution_winograd_dot_int8.h | 1005 +++++ .../convolution_winograd_dot_pack8to1_int8.h | 774 ++++ .../convolution_winograd_dot_pack8to4_int8.h | 1835 +++++++++ .../arm/convolution_winograd_transform_int8.h | 230 ++ ...onvolution_winograd_transform_pack4_int8.h | 178 + ...onvolution_winograd_transform_pack8_int8.h | 131 + .../convolution_winograd_transform_int8.h | 5 - 12 files changed, 4332 insertions(+), 6635 deletions(-) create mode 100644 src/layer/arm/convolution_winograd_dot_int8.h create mode 100644 src/layer/arm/convolution_winograd_dot_pack8to1_int8.h create mode 100644 src/layer/arm/convolution_winograd_dot_pack8to4_int8.h create mode 100644 src/layer/arm/convolution_winograd_transform_int8.h create mode 100644 src/layer/arm/convolution_winograd_transform_pack4_int8.h create mode 100644 src/layer/arm/convolution_winograd_transform_pack8_int8.h diff --git a/src/layer/arm/convolution_3x3_int8.h b/src/layer/arm/convolution_3x3_int8.h index 73e2fdc72..826ed8a82 100644 --- a/src/layer/arm/convolution_3x3_int8.h +++ b/src/layer/arm/convolution_3x3_int8.h @@ -1,10 +1,6 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. +// Tencent is pleased to support the open source community by making ncnn available. // -// author:BUG1989 (https://github.com/BUG1989/) Long-term support. -// author:FuGuangping (https://github.com/fu1899) Implemented the first version of INT8 quantization on ARMv7. -// -// Copyright (C) 2019 BUG1989. All rights reserved. -// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. // // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -16,16 +12,18 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -static void conv3x3s1_winograd23_transform_kernel_int8_neon(const Mat& kernel, std::vector& kernel_tm2, int inch, int outch, const Option& opt) +static void conv3x3s1_winograd43_transform_kernel_int8_neon(const Mat& kernel, Mat& kernel_tm_packed, int inch, int outch, const Option& opt) { - Mat kernel_tm(4 * 4, inch, outch, (size_t)2u); - - // G - const short ktm[4][3] = { - {2, 0, 0}, - {1, 1, 1}, - {1, -1, 1}, - {0, 0, 2} + // winograd43 transform kernel + Mat kernel_tm(6 * 6, inch, outch, (size_t)2u); + + const short ktm[6][3] = { + {6, 0, 0}, + {-4, -4, -4}, + {-4, 4, -4}, + {1, 2, 4}, + {1, -2, 4}, + {0, 0, 6} }; #pragma omp parallel for num_threads(opt.num_threads) @@ -42,3491 +40,200 @@ static void conv3x3s1_winograd23_transform_kernel_int8_neon(const Mat& kernel, s const signed char* k2 = kernel0 + 6; // h - short tmp[4][3]; - for (int i = 0; i < 4; i++) + short tmp[6][3]; + for (int i = 0; i < 6; i++) { - tmp[i][0] = (short)k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; - tmp[i][1] = (short)k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; - tmp[i][2] = (short)k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; } // U - for (int j = 0; j < 4; j++) + for (int j = 0; j < 6; j++) { short* tmpp = &tmp[j][0]; - for (int i = 0; i < 4; i++) + for (int i = 0; i < 6; i++) { - kernel_tm0[j * 4 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; + kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; } } } } - for (int r = 0; r < 4; r++) + // interleave + // src = 36-inch-outch + // dst = 8a-8b-inch/8a-36-outch/8b +#if __ARM_NEON + if (outch >= 8) + { + kernel_tm_packed.create(inch, 36, outch / 8 + (outch % 8) / 4 + outch % 4, (size_t)2u * 8, 8); + } + else if (outch >= 4) + { + kernel_tm_packed.create(inch, 36, outch / 4 + outch % 4, (size_t)2u * 4, 4); + } +#else // __ARM_NEON + if (outch >= 2) { - Mat kernel_tm_test(4 * 8, inch, outch / 8 + (outch % 8) / 4 + outch % 4, (size_t)2u); + kernel_tm_packed.create(inch, 36, outch / 2 + outch % 2, (size_t)2u * 2, 2); + } +#endif // __ARM_NEON + else + { + kernel_tm_packed.create(inch, 36, outch, (size_t)2u, 1); + } - int p = 0; - for (; p + 7 < outch; p += 8) - { - const short* kernel0 = (const short*)kernel_tm + (p + 0) * inch * 16; - const short* kernel1 = (const short*)kernel_tm + (p + 1) * inch * 16; - const short* kernel2 = (const short*)kernel_tm + (p + 2) * inch * 16; - const short* kernel3 = (const short*)kernel_tm + (p + 3) * inch * 16; - const short* kernel4 = (const short*)kernel_tm + (p + 4) * inch * 16; - const short* kernel5 = (const short*)kernel_tm + (p + 5) * inch * 16; - const short* kernel6 = (const short*)kernel_tm + (p + 6) * inch * 16; - const short* kernel7 = (const short*)kernel_tm + (p + 7) * inch * 16; + int p = 0; +#if __ARM_NEON + for (; p + 7 < outch; p += 8) + { + Mat g0 = kernel_tm_packed.channel(p / 8); - short* ktmp = kernel_tm_test.channel(p / 8); + for (int k = 0; k < 36; k++) + { + short* g00 = g0.row(k); for (int q = 0; q < inch; q++) { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp[4] = kernel1[r * 4 + 0]; - ktmp[5] = kernel1[r * 4 + 1]; - ktmp[6] = kernel1[r * 4 + 2]; - ktmp[7] = kernel1[r * 4 + 3]; - - ktmp[8] = kernel2[r * 4 + 0]; - ktmp[9] = kernel2[r * 4 + 1]; - ktmp[10] = kernel2[r * 4 + 2]; - ktmp[11] = kernel2[r * 4 + 3]; - - ktmp[12] = kernel3[r * 4 + 0]; - ktmp[13] = kernel3[r * 4 + 1]; - ktmp[14] = kernel3[r * 4 + 2]; - ktmp[15] = kernel3[r * 4 + 3]; - - ktmp[16] = kernel4[r * 4 + 0]; - ktmp[17] = kernel4[r * 4 + 1]; - ktmp[18] = kernel4[r * 4 + 2]; - ktmp[19] = kernel4[r * 4 + 3]; - - ktmp[20] = kernel5[r * 4 + 0]; - ktmp[21] = kernel5[r * 4 + 1]; - ktmp[22] = kernel5[r * 4 + 2]; - ktmp[23] = kernel5[r * 4 + 3]; - - ktmp[24] = kernel6[r * 4 + 0]; - ktmp[25] = kernel6[r * 4 + 1]; - ktmp[26] = kernel6[r * 4 + 2]; - ktmp[27] = kernel6[r * 4 + 3]; - - ktmp[28] = kernel7[r * 4 + 0]; - ktmp[29] = kernel7[r * 4 + 1]; - ktmp[30] = kernel7[r * 4 + 2]; - ktmp[31] = kernel7[r * 4 + 3]; - - ktmp += 32; - kernel0 += 16; - kernel1 += 16; - kernel2 += 16; - kernel3 += 16; - kernel4 += 16; - kernel5 += 16; - kernel6 += 16; - kernel7 += 16; + for (int i = 0; i < 8; i++) + { + g00[0] = kernel_tm.channel(p + i).row(q)[k]; + g00++; + } } } + } + for (; p + 3 < outch; p += 4) + { + const Mat k0 = kernel_tm.channel(p); + const Mat k1 = kernel_tm.channel(p + 1); + const Mat k2 = kernel_tm.channel(p + 2); + const Mat k3 = kernel_tm.channel(p + 3); - for (; p + 3 < outch; p += 4) - { - const short* kernel0 = (const short*)kernel_tm + (p + 0) * inch * 16; - const short* kernel1 = (const short*)kernel_tm + (p + 1) * inch * 16; - const short* kernel2 = (const short*)kernel_tm + (p + 2) * inch * 16; - const short* kernel3 = (const short*)kernel_tm + (p + 3) * inch * 16; + Mat g0 = kernel_tm_packed.channel(p / 8 + (p % 8) / 4); - short* ktmp = kernel_tm_test.channel(p / 8 + (p % 8) / 4); + for (int k = 0; k < 36; k++) + { + short* g00 = g0.row(k); for (int q = 0; q < inch; q++) { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp[4] = kernel1[r * 4 + 0]; - ktmp[5] = kernel1[r * 4 + 1]; - ktmp[6] = kernel1[r * 4 + 2]; - ktmp[7] = kernel1[r * 4 + 3]; - - ktmp[8] = kernel2[r * 4 + 0]; - ktmp[9] = kernel2[r * 4 + 1]; - ktmp[10] = kernel2[r * 4 + 2]; - ktmp[11] = kernel2[r * 4 + 3]; - - ktmp[12] = kernel3[r * 4 + 0]; - ktmp[13] = kernel3[r * 4 + 1]; - ktmp[14] = kernel3[r * 4 + 2]; - ktmp[15] = kernel3[r * 4 + 3]; - - ktmp += 16; - kernel0 += 16; - kernel1 += 16; - kernel2 += 16; - kernel3 += 16; + g00[0] = k0.row(q)[k]; + g00[1] = k1.row(q)[k]; + g00[2] = k2.row(q)[k]; + g00[3] = k3.row(q)[k]; + g00 += 4; } } + } +#else // __ARM_NEON + for (; p + 1 < outch; p += 2) + { + const Mat k0 = kernel_tm.channel(p); + const Mat k1 = kernel_tm.channel(p + 1); - for (; p < outch; p++) + Mat g0 = kernel_tm_packed.channel(p / 2); + + for (int k = 0; k < 36; k++) { - const short* kernel0 = (const short*)kernel_tm + p * inch * 16; + short* g00 = g0.row(k); + + int q = 0; +#if __ARM_FEATURE_SIMD32 + for (; q + 1 < inch; q += 2) + { + g00[0] = k0.row(q)[k]; + g00[2] = k1.row(q)[k]; + g00[1] = k0.row(q + 1)[k]; + g00[3] = k1.row(q + 1)[k]; + g00 += 4; + } +#endif // __ARM_FEATURE_SIMD32 + for (; q < inch; q++) + { + g00[0] = k0.row(q)[k]; + g00[1] = k1.row(q)[k]; + g00 += 2; + } + } + } +#endif // __ARM_NEON + for (; p < outch; p++) + { + const Mat k0 = kernel_tm.channel(p); + +#if __ARM_NEON + Mat g0 = kernel_tm_packed.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + Mat g0 = kernel_tm_packed.channel(p / 2 + p % 2); +#endif - short* ktmp = kernel_tm_test.channel(p / 8 + (p % 8) / 4 + p % 4); + for (int k = 0; k < 36; k++) + { + short* g00 = g0.row(k); for (int q = 0; q < inch; q++) { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp += 4; - kernel0 += 16; + g00[0] = k0.row(q)[k]; + g00 += 1; } } - kernel_tm2.push_back(kernel_tm_test); } } -static void conv3x3s1_winograd23_int8_neon(const Mat& bottom_blob, Mat& top_blob, const std::vector& kernel_tm_test, const Option& opt) +static void conv3x3s1_winograd43_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Option& opt) { int w = bottom_blob.w; int h = bottom_blob.h; int inch = bottom_blob.c; + // size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; int outw = top_blob.w; int outh = top_blob.h; int outch = top_blob.c; - // pad to 2n+2, winograd F(2,3) + // pad to 4n+2 Mat bottom_blob_bordered = bottom_blob; - outw = (outw + 1) / 2 * 2; - outh = (outh + 1) / 2 * 2; + outw = (outw + 3) / 4 * 4; + outh = (outh + 3) / 4 * 4; w = outw + 2; h = outh + 2; - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt_b); + copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt); // BEGIN transform input Mat bottom_blob_tm; { - int w_tm = outw / 2 * 4; - int h_tm = outh / 2 * 4; - - int nColBlocks = h_tm / 4; // may be the block num in FeatherCNN - int nRowBlocks = w_tm / 4; - - const int tiles = nColBlocks * nRowBlocks; - - bottom_blob_tm.create(4, inch, tiles * 4, 2u, opt.workspace_allocator); + int w_tiles = outw / 4; + int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; - // BT - // const float itm[4][4] = { - // {1.0f, 0.0f, -1.0f, 0.0f}, - // {0.0f, 1.0f, 1.00f, 0.0f}, - // {0.0f, -1.0f, 1.00f, 0.0f}, - // {0.0f, -1.0f, 0.00f, 1.0f} - // }; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int q = 0; q < inch; q++) - { - const signed char* img = bottom_blob_bordered.channel(q); - - for (int j = 0; j < nColBlocks; j++) - { - const signed char* r0 = img + w * j * 2; - const signed char* r1 = r0 + w; - const signed char* r2 = r1 + w; - const signed char* r3 = r2 + w; - - for (int i = 0; i < nRowBlocks; i++) - { - short* out_tm0 = bottom_blob_tm.channel(tiles * 0 + j * nRowBlocks + i).row(q); - short* out_tm1 = bottom_blob_tm.channel(tiles * 1 + j * nRowBlocks + i).row(q); - short* out_tm2 = bottom_blob_tm.channel(tiles * 2 + j * nRowBlocks + i).row(q); - short* out_tm3 = bottom_blob_tm.channel(tiles * 3 + j * nRowBlocks + i).row(q); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // load - "prfm pldl1keep, [%0, #64] \n" - "ld1 {v0.8b}, [%0] \n" - "prfm pldl1keep, [%1, #64] \n" - "ld1 {v1.8b}, [%1] \n" - "prfm pldl1keep, [%2, #64] \n" - "ld1 {v2.8b}, [%2] \n" - "prfm pldl1keep, [%3, #64] \n" - "ld1 {v3.8b}, [%3] \n" - // w = B_t * d, trans int8 to int16 - "ssubl v4.8h, v0.8b, v2.8b \n" // d4 - "saddl v5.8h, v1.8b, v2.8b \n" // d6 - "ssubl v6.8h, v2.8b, v1.8b \n" // d8 - "ssubl v7.8h, v3.8b, v1.8b \n" // d10 - // transpose w to w_t - "trn1 v8.4h, v4.4h, v5.4h \n" - "trn2 v9.4h, v4.4h, v5.4h \n" - "trn1 v10.4h, v6.4h, v7.4h \n" - "trn2 v11.4h, v6.4h, v7.4h \n" - - "trn1 v0.2s, v8.2s, v10.2s \n" - "trn2 v2.2s, v8.2s, v10.2s \n" - "trn1 v1.2s, v9.2s, v11.2s \n" - "trn2 v3.2s, v9.2s, v11.2s \n" - // U = B_t * d_t - "sub v4.4h, v0.4h, v2.4h \n" - "add v5.4h, v1.4h, v2.4h \n" - "sub v6.4h, v2.4h, v1.4h \n" - "sub v7.4h, v3.4h, v1.4h \n" - // save - "st1 {v4.4h}, [%4] \n" - "st1 {v5.4h}, [%5] \n" - "st1 {v6.4h}, [%6] \n" - "st1 {v7.4h}, [%7] \n" - : "=r"(r0), // %0 - "=r"(r1), // %1 - "=r"(r2), // %2 - "=r"(r3), // %3 - "=r"(out_tm0), // %4 - "=r"(out_tm1), // %5 - "=r"(out_tm2), // %6 - "=r"(out_tm3) // %7 - : "0"(r0), - "1"(r1), - "2"(r2), - "3"(r3), - "4"(out_tm0), - "5"(out_tm1), - "6"(out_tm2), - "7"(out_tm3) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"); -#else - asm volatile( - // load - "pld [%0, #64] \n" - "vld1.s8 {d0}, [%0] \n" - "pld [%1, #64] \n" - "vld1.s8 {d1}, [%1] \n" - "pld [%2, #64] \n" - "vld1.s8 {d2}, [%2] \n" - "pld [%3, #64] \n" - "vld1.s8 {d3}, [%3] \n" - // w = B_t * d, trans int8 to int16 - "vsubl.s8 q2, d0, d2 \n" // d4 - "vaddl.s8 q3, d1, d2 \n" // d6 - "vsubl.s8 q4, d2, d1 \n" // d8 - "vsubl.s8 q5, d3, d1 \n" // d10 - // transpose w to w_t - "vtrn.s16 d4, d6 \n" - "vtrn.s16 d8, d10 \n" - "vtrn.s32 d4, d8 \n" - "vtrn.s32 d6, d10 \n" - // U = B_t * d_t - "vsub.s16 d11, d4, d8 \n" - "vadd.s16 d12, d6, d8 \n" - "vsub.s16 d13, d8, d6 \n" - "vsub.s16 d14, d10, d6 \n" - // save - "vst1.s32 {d11}, [%4] \n" - "vst1.s32 {d12}, [%5] \n" - "vst1.s32 {d13}, [%6] \n" - "vst1.s32 {d14}, [%7] \n" - : "=r"(r0), // %0 - "=r"(r1), // %1 - "=r"(r2), // %2 - "=r"(r3), // %3 - "=r"(out_tm0), // %4 - "=r"(out_tm1), // %5 - "=r"(out_tm2), // %6 - "=r"(out_tm3) // %7 - : "0"(r0), - "1"(r1), - "2"(r2), - "3"(r3), - "4"(out_tm0), - "5"(out_tm1), - "6"(out_tm2), - "7"(out_tm3) - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"); -#endif // __aarch64__ -#else - short d0[4], d1[4], d2[4], d3[4]; - short w0[4], w1[4], w2[4], w3[4]; - short t0[4], t1[4], t2[4], t3[4]; - // load - for (int n = 0; n < 4; n++) - { - d0[n] = r0[n]; - d1[n] = r1[n]; - d2[n] = r2[n]; - d3[n] = r3[n]; - } - // w = B_t * d - for (int n = 0; n < 4; n++) - { - w0[n] = d0[n] - d2[n]; - w1[n] = d1[n] + d2[n]; - w2[n] = d2[n] - d1[n]; - w3[n] = d3[n] - d1[n]; - } - // transpose d to d_t - { - t0[0] = w0[0]; - t1[0] = w0[1]; - t2[0] = w0[2]; - t3[0] = w0[3]; - t0[1] = w1[0]; - t1[1] = w1[1]; - t2[1] = w1[2]; - t3[1] = w1[3]; - t0[2] = w2[0]; - t1[2] = w2[1]; - t2[2] = w2[2]; - t3[2] = w2[3]; - t0[3] = w3[0]; - t1[3] = w3[1]; - t2[3] = w3[2]; - t3[3] = w3[3]; - } - // U = B_t * d_t - for (int n = 0; n < 4; n++) - { - d0[n] = t0[n] - t2[n]; - d1[n] = t1[n] + t2[n]; - d2[n] = t2[n] - t1[n]; - d3[n] = t3[n] - t1[n]; - } - // save to out_tm - for (int n = 0; n < 4; n++) - { - out_tm0[n] = d0[n]; - out_tm1[n] = d1[n]; - out_tm2[n] = d2[n]; - out_tm3[n] = d3[n]; - } -#endif - r0 += 2; - r1 += 2; - r2 += 2; - r3 += 2; - } - } - } + bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator); + conv3x3s1_winograd43_transform_input_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt); } bottom_blob_bordered = Mat(); + // END transform input // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 2 * 4; - int h_tm = outh / 2 * 4; - - int nColBlocks = h_tm / 4; // may be the block num in FeatherCNN - int nRowBlocks = w_tm / 4; - - const int tiles = nColBlocks * nRowBlocks; - - top_blob_tm.create(16, tiles, outch, 4u, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 4; r++) - { - int nn_outch = 0; - int remain_outch_start = 0; - - nn_outch = outch >> 3; - remain_outch_start = nn_outch << 3; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - int* output4_tm = top_blob_tm.channel(p + 4); - int* output5_tm = top_blob_tm.channel(p + 5); - int* output6_tm = top_blob_tm.channel(p + 6); - int* output7_tm = top_blob_tm.channel(p + 7); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - output4_tm = output4_tm + r * 4; - output5_tm = output5_tm + r * 4; - output6_tm = output6_tm + r * 4; - output7_tm = output7_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "eor v4.16b, v4.16b, v4.16b \n" - "eor v5.16b, v5.16b, v5.16b \n" - "eor v6.16b, v6.16b, v6.16b \n" - "eor v7.16b, v7.16b, v7.16b \n" - "mov w4, %w20 \n" - - "0: \n" // for (int q=0; q> 2; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8 + (p % 8) / 4); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "mov w4, %w12 \n" - - "0: \n" // for (int q=0; q> 2 - "sshl v1.2s, v1.2s, %6.2s \n" // o1 = o1 >> 2 - - "st1 {v0.2s}, [%1], #8 \n" - "st1 {v1.2s}, [%2], #8 \n" - : "=r"(out_tile), // %0 - "=r"(outRow0), // %1 - "=r"(outRow1) // %2 - : "0"(out_tile), - "1"(outRow0), - "2"(outRow1), - "w"(_shift) // %6 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); -#else - asm volatile( - "pld [%0, #512] \n" - "vldm %0!, {d0-d7} \n" - - "vaddq.s32 q0, q0, q1 \n" // s0 = s0 + s1 + s2; - "vsubq.s32 q1, q1, q2 \n" - "vaddq.s32 q0, q0, q2 \n" // s1 = s1 - s2 + s3; - "vaddq.s32 q1, q1, q3 \n" - - "vtrn.s32 q0, q1 \n" - - "vadd.s32 d8, d0, d2 \n" // o0 = d0 + d1 + d2; - "vsub.s32 d9, d2, d1 \n" - "vadd.s32 d8, d8, d1 \n" // o1 = d1 - d2 + d3; - "vadd.s32 d9, d9, d3 \n" - - "vshl.s32 d8, d8, %P6 \n" // o0 = o0 >> 2 - "vshl.s32 d9, d9, %P6 \n" // o1 = o1 >> 2 - - "vst1.s32 {d8}, [%1]! \n" - "vst1.s32 {d9}, [%2]! \n" - : "=r"(out_tile), // %0 - "=r"(outRow0), // %1 - "=r"(outRow1) // %2 - : "0"(out_tile), - "1"(outRow0), - "2"(outRow1), - "w"(_shift) // %6 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4"); -#endif // __aarch64__ -#else - int s0[4], s1[4], s2[4], s3[4]; - int w0[4], w1[4]; - int d0[2], d1[2], d2[2], d3[2]; - int o0[2], o1[2]; - // load - for (int n = 0; n < 4; n++) - { - s0[n] = out_tile[n]; - s1[n] = out_tile[n + 4]; - s2[n] = out_tile[n + 8]; - s3[n] = out_tile[n + 12]; - } - // w = A_T * W - for (int n = 0; n < 4; n++) - { - w0[n] = s0[n] + s1[n] + s2[n]; - w1[n] = s1[n] - s2[n] + s3[n]; - } - // transpose w to w_t - { - d0[0] = w0[0]; - d0[1] = w1[0]; - d1[0] = w0[1]; - d1[1] = w1[1]; - d2[0] = w0[2]; - d2[1] = w1[2]; - d3[0] = w0[3]; - d3[1] = w1[3]; - } - // Y = A_T * w_t - for (int n = 0; n < 2; n++) - { - o0[n] = d0[n] + d1[n] + d2[n]; - o1[n] = d1[n] - d2[n] + d3[n]; - } - // save to top blob tm,why right 2,because the G' = G*2 - outRow0[0] = o0[0] >> 2; - outRow0[1] = o0[1] >> 2; - outRow1[0] = o1[0] >> 2; - outRow1[1] = o1[1] >> 2; - - out_tile += 16; - - outRow0 += 2; - outRow1 += 2; -#endif // __ARM_NEON - } - - outRow0 += outw; - outRow1 += outw; - } - } + top_blob_bordered = top_blob; } - // END transform output - - // cut result pad - copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt); -} - -static void conv3x3s1_winograd43_transform_kernel_int8_neon(const Mat& kernel, std::vector& kernel_tm2, int inch, int outch, const Option& opt) -{ - Mat kernel_tm(6 * 6, inch, outch, (size_t)2u); - - // G - // const float ktm[6][3] = { - // { 1.0f/4, 0.0f, 0.0f}, - // { -1.0f/6, -1.0f/6, -1.0f/6}, - // { -1.0f/6, 1.0f/6, -1.0f/6}, - // { 1.0f/24, 1.0f/12, 1.0f/6}, - // { 1.0f/24, -1.0f/12, 1.0f/6}, - // { 0.0f, 0.0f, 1.0f} - // }; - const short ktm[6][3] = { - {6, 0, 0}, - {-4, -4, -4}, - {-4, 4, -4}, - {1, 2, 4}, - {1, -2, 4}, - {0, 0, 6} - }; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) + else { - for (int q = 0; q < inch; q++) - { - const signed char* kernel0 = (const signed char*)kernel + p * inch * 9 + q * 9; - short* kernel_tm0 = kernel_tm.channel(p).row(q); - - // transform kernel - const signed char* k0 = kernel0; - const signed char* k1 = kernel0 + 3; - const signed char* k2 = kernel0 + 6; - - // h - short tmp[6][3]; - for (int i = 0; i < 6; i++) - { - tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; - tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; - tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; - } - - // U - for (int j = 0; j < 6; j++) - { - short* tmpp = &tmp[j][0]; - - for (int i = 0; i < 6; i++) - { - kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; - } - } - } - } - - for (int r = 0; r < 9; r++) - { - Mat kernel_tm_test(4 * 8, inch, outch / 8 + (outch % 8) / 4 + outch % 4, (size_t)2u); - - int p = 0; - for (; p + 7 < outch; p += 8) - { - const short* kernel0 = (const short*)kernel_tm.channel(p); - const short* kernel1 = (const short*)kernel_tm.channel(p + 1); - const short* kernel2 = (const short*)kernel_tm.channel(p + 2); - const short* kernel3 = (const short*)kernel_tm.channel(p + 3); - const short* kernel4 = (const short*)kernel_tm.channel(p + 4); - const short* kernel5 = (const short*)kernel_tm.channel(p + 5); - const short* kernel6 = (const short*)kernel_tm.channel(p + 6); - const short* kernel7 = (const short*)kernel_tm.channel(p + 7); - - short* ktmp = kernel_tm_test.channel(p / 8); - - for (int q = 0; q < inch; q++) - { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp[4] = kernel1[r * 4 + 0]; - ktmp[5] = kernel1[r * 4 + 1]; - ktmp[6] = kernel1[r * 4 + 2]; - ktmp[7] = kernel1[r * 4 + 3]; - - ktmp[8] = kernel2[r * 4 + 0]; - ktmp[9] = kernel2[r * 4 + 1]; - ktmp[10] = kernel2[r * 4 + 2]; - ktmp[11] = kernel2[r * 4 + 3]; - - ktmp[12] = kernel3[r * 4 + 0]; - ktmp[13] = kernel3[r * 4 + 1]; - ktmp[14] = kernel3[r * 4 + 2]; - ktmp[15] = kernel3[r * 4 + 3]; - - ktmp[16] = kernel4[r * 4 + 0]; - ktmp[17] = kernel4[r * 4 + 1]; - ktmp[18] = kernel4[r * 4 + 2]; - ktmp[19] = kernel4[r * 4 + 3]; - - ktmp[20] = kernel5[r * 4 + 0]; - ktmp[21] = kernel5[r * 4 + 1]; - ktmp[22] = kernel5[r * 4 + 2]; - ktmp[23] = kernel5[r * 4 + 3]; - - ktmp[24] = kernel6[r * 4 + 0]; - ktmp[25] = kernel6[r * 4 + 1]; - ktmp[26] = kernel6[r * 4 + 2]; - ktmp[27] = kernel6[r * 4 + 3]; - - ktmp[28] = kernel7[r * 4 + 0]; - ktmp[29] = kernel7[r * 4 + 1]; - ktmp[30] = kernel7[r * 4 + 2]; - ktmp[31] = kernel7[r * 4 + 3]; - - ktmp += 32; - kernel0 += 36; - kernel1 += 36; - kernel2 += 36; - kernel3 += 36; - kernel4 += 36; - kernel5 += 36; - kernel6 += 36; - kernel7 += 36; - } - } - - for (; p + 3 < outch; p += 4) - { - const short* kernel0 = (const short*)kernel_tm.channel(p); - const short* kernel1 = (const short*)kernel_tm.channel(p + 1); - const short* kernel2 = (const short*)kernel_tm.channel(p + 2); - const short* kernel3 = (const short*)kernel_tm.channel(p + 3); - - short* ktmp = kernel_tm_test.channel(p / 8 + (p % 8) / 4); - - for (int q = 0; q < inch; q++) - { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp[4] = kernel1[r * 4 + 0]; - ktmp[5] = kernel1[r * 4 + 1]; - ktmp[6] = kernel1[r * 4 + 2]; - ktmp[7] = kernel1[r * 4 + 3]; - - ktmp[8] = kernel2[r * 4 + 0]; - ktmp[9] = kernel2[r * 4 + 1]; - ktmp[10] = kernel2[r * 4 + 2]; - ktmp[11] = kernel2[r * 4 + 3]; - - ktmp[12] = kernel3[r * 4 + 0]; - ktmp[13] = kernel3[r * 4 + 1]; - ktmp[14] = kernel3[r * 4 + 2]; - ktmp[15] = kernel3[r * 4 + 3]; - - ktmp += 16; - kernel0 += 36; - kernel1 += 36; - kernel2 += 36; - kernel3 += 36; - } - } - - for (; p < outch; p++) - { - const short* kernel0 = (const short*)kernel_tm.channel(p); - - short* ktmp = kernel_tm_test.channel(p / 8 + (p % 8) / 4 + p % 4); - - for (int q = 0; q < inch; q++) - { - ktmp[0] = kernel0[r * 4 + 0]; - ktmp[1] = kernel0[r * 4 + 1]; - ktmp[2] = kernel0[r * 4 + 2]; - ktmp[3] = kernel0[r * 4 + 3]; - - ktmp += 4; - kernel0 += 36; - } - } - kernel_tm2.push_back(kernel_tm_test); - } -} - -static void conv3x3s1_winograd43_int8_neon(const Mat& bottom_blob, Mat& top_blob, const std::vector& kernel_tm_test, const Option& opt) -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - int inch = bottom_blob.c; - - int outw = top_blob.w; - int outh = top_blob.h; - int outch = top_blob.c; - - // pad to 4n+2, winograd F(4,3) - Mat bottom_blob_bordered = bottom_blob; - - outw = (outw + 3) / 4 * 4; - outh = (outh + 3) / 4 * 4; - - w = outw + 2; - h = outh + 2; - - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt_b); - - // BEGIN transform input - Mat bottom_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - const int tiles = nColBlocks * nRowBlocks; - - bottom_blob_tm.create(4, inch, tiles * 9, 2u, opt.workspace_allocator); - - // BT - // const float itm[4][4] = { - // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, - // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, - // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, - // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} - // }; - - // 0 = 4 * r00 - 5 * r02 + r04 - // 1 = -4 * (r01 + r02) + r03 + r04 - // 2 = 4 * (r01 - r02) - r03 + r04 - // 3 = -2 * r01 - r02 + 2 * r03 + r04 - // 4 = 2 * r01 - r02 - 2 * r03 + r04 - // 5 = 4 * r01 - 5 * r03 + r05 - - #pragma omp parallel for num_threads(opt.num_threads) - for (int q = 0; q < inch; q++) - { - const signed char* img = bottom_blob_bordered.channel(q); - - for (int j = 0; j < nColBlocks; j++) - { - const signed char* r0 = img + w * j * 4; - const signed char* r1 = r0 + w; - const signed char* r2 = r1 + w; - const signed char* r3 = r2 + w; - const signed char* r4 = r3 + w; - const signed char* r5 = r4 + w; - - for (int i = 0; i < nRowBlocks; i++) - { - short* out_tm0 = bottom_blob_tm.channel(tiles * 0 + j * nRowBlocks + i).row(q); - short* out_tm1 = bottom_blob_tm.channel(tiles * 1 + j * nRowBlocks + i).row(q); - short* out_tm2 = bottom_blob_tm.channel(tiles * 2 + j * nRowBlocks + i).row(q); - short* out_tm3 = bottom_blob_tm.channel(tiles * 3 + j * nRowBlocks + i).row(q); - short* out_tm4 = bottom_blob_tm.channel(tiles * 4 + j * nRowBlocks + i).row(q); - short* out_tm5 = bottom_blob_tm.channel(tiles * 5 + j * nRowBlocks + i).row(q); - short* out_tm6 = bottom_blob_tm.channel(tiles * 6 + j * nRowBlocks + i).row(q); - short* out_tm7 = bottom_blob_tm.channel(tiles * 7 + j * nRowBlocks + i).row(q); - short* out_tm8 = bottom_blob_tm.channel(tiles * 8 + j * nRowBlocks + i).row(q); -#if __ARM_NEON - int8x8_t _d0, _d1, _d2, _d3, _d4, _d5; - int16x8_t _w0, _w1, _w2, _w3, _w4, _w5; - int16x8_t _t0, _t1, _t2, _t3, _t4, _t5; - int16x8_t _n0, _n1, _n2, _n3, _n4, _n5; - // load - _d0 = vld1_s8(r0); - _d1 = vld1_s8(r1); - _d2 = vld1_s8(r2); - _d3 = vld1_s8(r3); - _d4 = vld1_s8(r4); - _d5 = vld1_s8(r5); - - int8x8_t _1_n = vdup_n_s8(-1); - int8x8_t _2_p = vdup_n_s8(2); - int8x8_t _2_n = vdup_n_s8(-2); - int8x8_t _4_p = vdup_n_s8(4); - int8x8_t _4_n = vdup_n_s8(-4); - int8x8_t _5_n = vdup_n_s8(-5); - - int16x8_t _1_n_s16 = vdupq_n_s16(-1); - int16x8_t _2_p_s16 = vdupq_n_s16(2); - int16x8_t _2_n_s16 = vdupq_n_s16(-2); - int16x8_t _4_p_s16 = vdupq_n_s16(4); - int16x8_t _4_n_s16 = vdupq_n_s16(-4); - int16x8_t _5_n_s16 = vdupq_n_s16(-5); - // w = B_t * d - _w0 = vmull_s8(_d0, _4_p); - _w0 = vmlal_s8(_w0, _d2, _5_n); - _w0 = vaddw_s8(_w0, _d4); - - _w1 = vmull_s8(_d1, _4_n); - _w1 = vmlal_s8(_w1, _d2, _4_n); - _w1 = vaddw_s8(_w1, _d3); - _w1 = vaddw_s8(_w1, _d4); - - _w2 = vmull_s8(_d1, _4_p); - _w2 = vmlal_s8(_w2, _d2, _4_n); - _w2 = vmlal_s8(_w2, _d3, _1_n); - _w2 = vaddw_s8(_w2, _d4); - - _w3 = vmull_s8(_d1, _2_n); - _w3 = vmlal_s8(_w3, _d2, _1_n); - _w3 = vmlal_s8(_w3, _d3, _2_p); - _w3 = vaddw_s8(_w3, _d4); - - _w4 = vmull_s8(_d1, _2_p); - _w4 = vmlal_s8(_w4, _d2, _1_n); - _w4 = vmlal_s8(_w4, _d3, _2_n); - _w4 = vaddw_s8(_w4, _d4); - - _w5 = vmull_s8(_d1, _4_p); - _w5 = vmlal_s8(_w5, _d3, _5_n); - _w5 = vaddw_s8(_w5, _d5); - // transpose d to d_t - { - _t0[0] = _w0[0]; - _t1[0] = _w0[1]; - _t2[0] = _w0[2]; - _t3[0] = _w0[3]; - _t4[0] = _w0[4]; - _t5[0] = _w0[5]; - _t0[1] = _w1[0]; - _t1[1] = _w1[1]; - _t2[1] = _w1[2]; - _t3[1] = _w1[3]; - _t4[1] = _w1[4]; - _t5[1] = _w1[5]; - _t0[2] = _w2[0]; - _t1[2] = _w2[1]; - _t2[2] = _w2[2]; - _t3[2] = _w2[3]; - _t4[2] = _w2[4]; - _t5[2] = _w2[5]; - _t0[3] = _w3[0]; - _t1[3] = _w3[1]; - _t2[3] = _w3[2]; - _t3[3] = _w3[3]; - _t4[3] = _w3[4]; - _t5[3] = _w3[5]; - _t0[4] = _w4[0]; - _t1[4] = _w4[1]; - _t2[4] = _w4[2]; - _t3[4] = _w4[3]; - _t4[4] = _w4[4]; - _t5[4] = _w4[5]; - _t0[5] = _w5[0]; - _t1[5] = _w5[1]; - _t2[5] = _w5[2]; - _t3[5] = _w5[3]; - _t4[5] = _w5[4]; - _t5[5] = _w5[5]; - } - // d = B_t * d_t - _n0 = vmulq_s16(_t0, _4_p_s16); - _n0 = vmlaq_s16(_n0, _t2, _5_n_s16); - _n0 = vaddq_s16(_n0, _t4); - - _n1 = vmulq_s16(_t1, _4_n_s16); - _n1 = vmlaq_s16(_n1, _t2, _4_n_s16); - _n1 = vaddq_s16(_n1, _t3); - _n1 = vaddq_s16(_n1, _t4); - - _n2 = vmulq_s16(_t1, _4_p_s16); - _n2 = vmlaq_s16(_n2, _t2, _4_n_s16); - _n2 = vmlaq_s16(_n2, _t3, _1_n_s16); - _n2 = vaddq_s16(_n2, _t4); - - _n3 = vmulq_s16(_t1, _2_n_s16); - _n3 = vmlaq_s16(_n3, _t2, _1_n_s16); - _n3 = vmlaq_s16(_n3, _t3, _2_p_s16); - _n3 = vaddq_s16(_n3, _t4); - - _n4 = vmulq_s16(_t1, _2_p_s16); - _n4 = vmlaq_s16(_n4, _t2, _1_n_s16); - _n4 = vmlaq_s16(_n4, _t3, _2_n_s16); - _n4 = vaddq_s16(_n4, _t4); - - _n5 = vmulq_s16(_t1, _4_p_s16); - _n5 = vmlaq_s16(_n5, _t3, _5_n_s16); - _n5 = vaddq_s16(_n5, _t5); - // save to out_tm - out_tm0[0] = _n0[0]; - out_tm0[1] = _n0[1]; - out_tm0[2] = _n0[2]; - out_tm0[3] = _n0[3]; - out_tm1[0] = _n0[4]; - out_tm1[1] = _n0[5]; - out_tm1[2] = _n1[0]; - out_tm1[3] = _n1[1]; - out_tm2[0] = _n1[2]; - out_tm2[1] = _n1[3]; - out_tm2[2] = _n1[4]; - out_tm2[3] = _n1[5]; - - out_tm3[0] = _n2[0]; - out_tm3[1] = _n2[1]; - out_tm3[2] = _n2[2]; - out_tm3[3] = _n2[3]; - out_tm4[0] = _n2[4]; - out_tm4[1] = _n2[5]; - out_tm4[2] = _n3[0]; - out_tm4[3] = _n3[1]; - out_tm5[0] = _n3[2]; - out_tm5[1] = _n3[3]; - out_tm5[2] = _n3[4]; - out_tm5[3] = _n3[5]; - - out_tm6[0] = _n4[0]; - out_tm6[1] = _n4[1]; - out_tm6[2] = _n4[2]; - out_tm6[3] = _n4[3]; - out_tm7[0] = _n4[4]; - out_tm7[1] = _n4[5]; - out_tm7[2] = _n5[0]; - out_tm7[3] = _n5[1]; - out_tm8[0] = _n5[2]; - out_tm8[1] = _n5[3]; - out_tm8[2] = _n5[4]; - out_tm8[3] = _n5[5]; -#else - short d0[6], d1[6], d2[6], d3[6], d4[6], d5[6]; - short w0[6], w1[6], w2[6], w3[6], w4[6], w5[6]; - short t0[6], t1[6], t2[6], t3[6], t4[6], t5[6]; - - // load - for (int n = 0; n < 6; n++) - { - d0[n] = r0[n]; - d1[n] = r1[n]; - d2[n] = r2[n]; - d3[n] = r3[n]; - d4[n] = r4[n]; - d5[n] = r5[n]; - } - // w = B_t * d - for (int n = 0; n < 6; n++) - { - w0[n] = 4 * d0[n] - 5 * d2[n] + d4[n]; - w1[n] = -4 * d1[n] - 4 * d2[n] + d3[n] + d4[n]; - w2[n] = 4 * d1[n] - 4 * d2[n] - d3[n] + d4[n]; - w3[n] = -2 * d1[n] - d2[n] + 2 * d3[n] + d4[n]; - w4[n] = 2 * d1[n] - d2[n] - 2 * d3[n] + d4[n]; - w5[n] = 4 * d1[n] - 5 * d3[n] + d5[n]; - } - // transpose d to d_t - { - t0[0] = w0[0]; - t1[0] = w0[1]; - t2[0] = w0[2]; - t3[0] = w0[3]; - t4[0] = w0[4]; - t5[0] = w0[5]; - t0[1] = w1[0]; - t1[1] = w1[1]; - t2[1] = w1[2]; - t3[1] = w1[3]; - t4[1] = w1[4]; - t5[1] = w1[5]; - t0[2] = w2[0]; - t1[2] = w2[1]; - t2[2] = w2[2]; - t3[2] = w2[3]; - t4[2] = w2[4]; - t5[2] = w2[5]; - t0[3] = w3[0]; - t1[3] = w3[1]; - t2[3] = w3[2]; - t3[3] = w3[3]; - t4[3] = w3[4]; - t5[3] = w3[5]; - t0[4] = w4[0]; - t1[4] = w4[1]; - t2[4] = w4[2]; - t3[4] = w4[3]; - t4[4] = w4[4]; - t5[4] = w4[5]; - t0[5] = w5[0]; - t1[5] = w5[1]; - t2[5] = w5[2]; - t3[5] = w5[3]; - t4[5] = w5[4]; - t5[5] = w5[5]; - } - // d = B_t * d_t - for (int n = 0; n < 6; n++) - { - d0[n] = 4 * t0[n] - 5 * t2[n] + t4[n]; - d1[n] = -4 * t1[n] - 4 * t2[n] + t3[n] + t4[n]; - d2[n] = 4 * t1[n] - 4 * t2[n] - t3[n] + t4[n]; - d3[n] = -2 * t1[n] - t2[n] + 2 * t3[n] + t4[n]; - d4[n] = 2 * t1[n] - t2[n] - 2 * t3[n] + t4[n]; - d5[n] = 4 * t1[n] - 5 * t3[n] + t5[n]; - } - // save to out_tm - { - out_tm0[0] = d0[0]; - out_tm0[1] = d0[1]; - out_tm0[2] = d0[2]; - out_tm0[3] = d0[3]; - out_tm1[0] = d0[4]; - out_tm1[1] = d0[5]; - out_tm1[2] = d1[0]; - out_tm1[3] = d1[1]; - out_tm2[0] = d1[2]; - out_tm2[1] = d1[3]; - out_tm2[2] = d1[4]; - out_tm2[3] = d1[5]; - - out_tm3[0] = d2[0]; - out_tm3[1] = d2[1]; - out_tm3[2] = d2[2]; - out_tm3[3] = d2[3]; - out_tm4[0] = d2[4]; - out_tm4[1] = d2[5]; - out_tm4[2] = d3[0]; - out_tm4[3] = d3[1]; - out_tm5[0] = d3[2]; - out_tm5[1] = d3[3]; - out_tm5[2] = d3[4]; - out_tm5[3] = d3[5]; - - out_tm6[0] = d4[0]; - out_tm6[1] = d4[1]; - out_tm6[2] = d4[2]; - out_tm6[3] = d4[3]; - out_tm7[0] = d4[4]; - out_tm7[1] = d4[5]; - out_tm7[2] = d5[0]; - out_tm7[3] = d5[1]; - out_tm8[0] = d5[2]; - out_tm8[1] = d5[3]; - out_tm8[2] = d5[4]; - out_tm8[3] = d5[5]; - } -#endif // __ARM_NEON - r0 += 4; - r1 += 4; - r2 += 4; - r3 += 4; - r4 += 4; - r5 += 4; - } - } - } - } - bottom_blob_bordered = Mat(); - - // BEGIN dot - Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - const int tiles = nColBlocks * nRowBlocks; - - top_blob_tm.create(36, tiles, outch, 4u, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 9; r++) - { - int nn_outch = 0; - int remain_outch_start = 0; - - nn_outch = outch >> 3; - remain_outch_start = nn_outch << 3; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - int* output4_tm = top_blob_tm.channel(p + 4); - int* output5_tm = top_blob_tm.channel(p + 5); - int* output6_tm = top_blob_tm.channel(p + 6); - int* output7_tm = top_blob_tm.channel(p + 7); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - output4_tm = output4_tm + r * 4; - output5_tm = output5_tm + r * 4; - output6_tm = output6_tm + r * 4; - output7_tm = output7_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "eor v4.16b, v4.16b, v4.16b \n" - "eor v5.16b, v5.16b, v5.16b \n" - "eor v6.16b, v6.16b, v6.16b \n" - "eor v7.16b, v7.16b, v7.16b \n" - "mov w4, %w20 \n" - - "0: \n" // for (int q=0; q> 2; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8 + (p % 8) / 4); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "mov w4, %w12 \n" - - "0: \n" // for (int q=0; q(i); - - // int sum0[36] = {0}; - - // for (int q=0; q(i); - // const short* k0 = kernel0_tm.row(q); - - // for (int n=0; n<36; n++) - // { - // sum0[n] += (int)r0[n] * k0[n]; - // } - // } - - // for (int n=0; n<36; n++) - // { - // output0_tm[n] = sum0[n]; - // } - // } - // } - } - } - bottom_blob_tm = Mat(); - // END dot - - // BEGIN transform output - Mat top_blob_bordered; - top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator); - { - // AT - // const float itm[4][6] = { - // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, - // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} - // }; - - // 0 = r00 + r01 + r02 + r03 + r04 - // 1 = r01 - r02 + 2 * (r03 - r04) - // 2 = r01 + r02 + 4 * (r03 + r04) - // 3 = r01 - r02 + 8 * (r03 - r04) + r05 - - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - int* out_tile = top_blob_tm.channel(p); - int* outRow0 = top_blob_bordered.channel(p); - int* outRow1 = outRow0 + outw; - int* outRow2 = outRow0 + outw * 2; - int* outRow3 = outRow0 + outw * 3; - - for (int j = 0; j < nColBlocks; j++) - { - for (int i = 0; i < nRowBlocks; i++) - { -#if __ARM_NEON - int32x4_t _s0, _s1, _s2, _s3, _s4, _s5; - int32x2_t _s0n, _s1n, _s2n, _s3n, _s4n, _s5n; - int32x4_t _w0, _w3; - int32x2_t _w0n, _w3n; - int32x4_t _d0, _d1, _d2, _d3, _d4, _d5; - int32x4_t _o0, _o1, _o2, _o3; - // load - _s0 = vld1q_s32(out_tile); - _s0n = vld1_s32(out_tile + 4); - _s1 = vld1q_s32(out_tile + 6); - _s1n = vld1_s32(out_tile + 10); - _s2 = vld1q_s32(out_tile + 12); - _s2n = vld1_s32(out_tile + 16); - _s3 = vld1q_s32(out_tile + 18); - _s3n = vld1_s32(out_tile + 22); - _s4 = vld1q_s32(out_tile + 24); - _s4n = vld1_s32(out_tile + 28); - _s5 = vld1q_s32(out_tile + 30); - _s5n = vld1_s32(out_tile + 34); - // w = A_T * W - int32x2_t _tp0 = {1, 4}; - int32x2_t _tp1 = {2, 8}; - - // 4*s5[n] - int32x4_t _s5x4 = vshlq_n_s32(_s5, 2); - int32x2_t _s5x4n = vshl_n_s32(_s5n, 2); - - int32x4_t _t1p2 = vaddq_s32(_s1, _s2); - int32x2_t _t1p2n = vadd_s32(_s1n, _s2n); - int32x4_t _t3p4 = vaddq_s32(_s3, _s4); - int32x2_t _t3p4n = vadd_s32(_s3n, _s4n); - int32x4_t _t1s2 = vsubq_s32(_s1, _s2); - int32x2_t _t1s2n = vsub_s32(_s1n, _s2n); - int32x4_t _t3s4 = vsubq_s32(_s3, _s4); - int32x2_t _t3s4n = vsub_s32(_s3n, _s4n); - - _w0 = vaddq_s32(_s0, _t1p2); - _w0n = vadd_s32(_s0n, _t1p2n); - _w0 = vaddq_s32(_w0, _t3p4); - _w0n = vadd_s32(_w0n, _t3p4n); - _w0n = vmul_s32(_w0n, _tp0); - - // _w2,_w2n - _t1p2 = vmlaq_lane_s32(_t1p2, _t3p4, _tp0, 1); - _t1p2n = vmla_lane_s32(_t1p2n, _t3p4n, _tp0, 1); - _t1p2n = vmul_s32(_t1p2n, _tp0); - - _w3 = vaddq_s32(_s5x4, _t1s2); - _w3n = vadd_s32(_s5x4n, _t1s2n); - _w3 = vmlaq_lane_s32(_w3, _t3s4, _tp1, 1); - _w3n = vmla_lane_s32(_w3n, _t3s4n, _tp1, 1); - _w3n = vmul_s32(_w3n, _tp0); - - // _w1, _w1n - _t1s2 = vmlaq_lane_s32(_t1s2, _t3s4, _tp1, 0); - _t1s2n = vmla_lane_s32(_t1s2n, _t3s4n, _tp1, 0); - _t1s2n = vmul_s32(_t1s2n, _tp0); - - int32x4_t _w02n = vcombine_s32(_w0n, _t1p2n); - int32x4_t _w13n = vcombine_s32(_t1s2n, _w3n); - - // transpose w to w_t -#if __aarch64__ - int32x4_t _wt0 = vtrn1q_s32(_w0, _t1s2); - int32x4_t _wt1 = vtrn2q_s32(_w0, _t1s2); - int32x4_t _wt2 = vtrn1q_s32(_t1p2, _w3); - int32x4_t _wt3 = vtrn2q_s32(_t1p2, _w3); - int64x2_t _dt0 = vtrn1q_s64(vreinterpretq_s64_s32(_wt0), vreinterpretq_s64_s32(_wt2)); - int64x2_t _dt2 = vtrn2q_s64(vreinterpretq_s64_s32(_wt0), vreinterpretq_s64_s32(_wt2)); - int64x2_t _dt1 = vtrn1q_s64(vreinterpretq_s64_s32(_wt1), vreinterpretq_s64_s32(_wt3)); - int64x2_t _dt3 = vtrn2q_s64(vreinterpretq_s64_s32(_wt1), vreinterpretq_s64_s32(_wt3)); - _d0 = vreinterpretq_s32_s64(_dt0); - _d1 = vreinterpretq_s32_s64(_dt1); - _d2 = vreinterpretq_s32_s64(_dt2); - _d3 = vreinterpretq_s32_s64(_dt3); - _d4 = vtrn1q_s32(_w02n, _w13n); - _d5 = vtrn2q_s32(_w02n, _w13n); -#else - asm volatile( - "vtrn.32 %q[_w0], %q[_w1] \n" - "vtrn.32 %q[_w2], %q[_w3] \n" - "vswp %f[_w0], %e[_w2] \n" - "vswp %f[_w1], %e[_w3] \n" - "vtrn.32 %q[_w02n], %q[_w13n] \n" - : [_w0] "+w"(_w0), - [_w1] "+w"(_t1s2), - [_w2] "+w"(_t1p2), - [_w3] "+w"(_w3), - [_w02n] "+w"(_w02n), - [_w13n] "+w"(_w13n) - : - : "cc", "memory"); - _d0 = _w0; - _d1 = _t1s2; - _d2 = _t1p2; - _d3 = _w3; - _d4 = _w02n; - _d5 = _w13n; -#endif - // Y = A_T * w_t - _t1p2 = vaddq_s32(_d1, _d2); - _t3p4 = vaddq_s32(_d3, _d4); - _t1s2 = vsubq_s32(_d1, _d2); - _t3s4 = vsubq_s32(_d3, _d4); - - _o0 = vaddq_s32(_d0, _t1p2); - _o0 = vaddq_s32(_o0, _t3p4); - - // _o2 - _t1p2 = vmlaq_lane_s32(_t1p2, _t3p4, _tp0, 1); - - _o3 = vaddq_s32(_d5, _t1s2); - _o3 = vmlaq_lane_s32(_o3, _t3s4, _tp1, 1); - - // _o1 - _t1s2 = vmlaq_lane_s32(_t1s2, _t3s4, _tp1, 0); - - // save to top blob tm - float32x4_t _ot0 = vcvtq_f32_s32(_o0); - float32x4_t _ot1 = vcvtq_f32_s32(_t1s2); - float32x4_t _ot2 = vcvtq_f32_s32(_t1p2); - float32x4_t _ot3 = vcvtq_f32_s32(_o3); - - _ot0 = vmulq_n_f32(_ot0, 0.0017361112); - _ot1 = vmulq_n_f32(_ot1, 0.0017361112); - _ot2 = vmulq_n_f32(_ot2, 0.0017361112); - _ot3 = vmulq_n_f32(_ot3, 0.0017361112); - - _o0 = vcvtq_s32_f32(_ot0); - _o1 = vcvtq_s32_f32(_ot1); - _o2 = vcvtq_s32_f32(_ot2); - _o3 = vcvtq_s32_f32(_ot3); - - vst1q_s32(outRow0, _o0); - vst1q_s32(outRow1, _o1); - vst1q_s32(outRow2, _o2); - vst1q_s32(outRow3, _o3); -#else - int s0[6], s1[6], s2[6], s3[6], s4[6], s5[6]; - int w0[6], w1[6], w2[6], w3[6]; - int d0[4], d1[4], d2[4], d3[4], d4[4], d5[4]; - int o0[4], o1[4], o2[4], o3[4]; - - // load - for (int n = 0; n < 6; n++) - { - s0[n] = out_tile[n]; - s1[n] = out_tile[n + 6]; - s2[n] = out_tile[n + 12]; - s3[n] = out_tile[n + 18]; - s4[n] = out_tile[n + 24]; - s5[n] = out_tile[n + 30]; - } - // w = A_T * W - for (int n = 0; n < 5; n++) - { - w0[n] = s0[n] + s1[n] + s2[n] + s3[n] + s4[n]; - w1[n] = s1[n] - s2[n] + 2 * s3[n] - 2 * s4[n]; - w2[n] = s1[n] + s2[n] + 4 * s3[n] + 4 * s4[n]; - w3[n] = s1[n] - s2[n] + 8 * s3[n] - 8 * s4[n] + 4 * s5[n]; - } - for (int n = 5; n < 6; n++) - { - w0[n] = 4 * (s0[n] + s1[n] + s2[n] + s3[n] + s4[n]); - w1[n] = 4 * (s1[n] - s2[n] + 2 * s3[n] - 2 * s4[n]); - w2[n] = 4 * (s1[n] + s2[n] + 4 * s3[n] + 4 * s4[n]); - w3[n] = 4 * (s1[n] - s2[n] + 8 * s3[n] - 8 * s4[n] + 4 * s5[n]); - } - // transpose w to w_t - { - d0[0] = w0[0]; - d0[1] = w1[0]; - d0[2] = w2[0]; - d0[3] = w3[0]; - d1[0] = w0[1]; - d1[1] = w1[1]; - d1[2] = w2[1]; - d1[3] = w3[1]; - d2[0] = w0[2]; - d2[1] = w1[2]; - d2[2] = w2[2]; - d2[3] = w3[2]; - d3[0] = w0[3]; - d3[1] = w1[3]; - d3[2] = w2[3]; - d3[3] = w3[3]; - d4[0] = w0[4]; - d4[1] = w1[4]; - d4[2] = w2[4]; - d4[3] = w3[4]; - d5[0] = w0[5]; - d5[1] = w1[5]; - d5[2] = w2[5]; - d5[3] = w3[5]; - } - // Y = A_T * w_t - for (int n = 0; n < 4; n++) - { - o0[n] = d0[n] + d1[n] + d2[n] + d3[n] + d4[n]; - o1[n] = d1[n] - d2[n] + 2 * d3[n] - 2 * d4[n]; - o2[n] = d1[n] + d2[n] + 4 * d3[n] + 4 * d4[n]; - o3[n] = d1[n] - d2[n] + 8 * d3[n] - 8 * d4[n] + d5[n]; - } - // save to top blob tm - for (int n = 0; n < 4; n++) - { - outRow0[n] = o0[n] / 576; - outRow1[n] = o1[n] / 576; - outRow2[n] = o2[n] / 576; - outRow3[n] = o3[n] / 576; - } -#endif // __ARM_NEON - out_tile += 36; - - outRow0 += 4; - outRow1 += 4; - outRow2 += 4; - outRow3 += 4; - } - - outRow0 += outw * 3; - outRow1 += outw * 3; - outRow2 += outw * 3; - outRow3 += outw * 3; - } - } + top_blob_bordered.create(outw, outh, outch, 4u, 1, opt.workspace_allocator); } - // END transform output - - // cut result pad - copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt); -} - -static void conv3x3s1_winograd43_dequant_int8_neon(const Mat& bottom_blob, Mat& top_blob, const std::vector& kernel_tm_test, const Mat& _bias, std::vector scales_dequant, const Option& opt) -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - int inch = bottom_blob.c; - - int outw = top_blob.w; - int outh = top_blob.h; - int outch = top_blob.c; - - const float* bias = _bias; - - // pad to 4n+2, winograd F(4,3) - Mat bottom_blob_bordered = bottom_blob; - - outw = (outw + 3) / 4 * 4; - outh = (outh + 3) / 4 * 4; - - w = outw + 2; - h = outh + 2; - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt_b); - - // BEGIN transform input - Mat bottom_blob_tm; { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - const int tiles = nColBlocks * nRowBlocks; - - bottom_blob_tm.create(4, inch, tiles * 9, 2u, opt.workspace_allocator); - - // BT - // const float itm[4][4] = { - // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, - // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, - // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, - // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} - // }; - - // 0 = 4 * r00 - 5 * r02 + r04 - // 1 = -4 * (r01 + r02) + r03 + r04 - // 2 = 4 * (r01 - r02) - r03 + r04 - // 3 = -2 * r01 - r02 + 2 * r03 + r04 - // 4 = 2 * r01 - r02 - 2 * r03 + r04 - // 5 = 4 * r01 - 5 * r03 + r05 - - #pragma omp parallel for num_threads(opt.num_threads) - for (int q = 0; q < inch; q++) - { - const signed char* img = bottom_blob_bordered.channel(q); - - for (int j = 0; j < nColBlocks; j++) - { - const signed char* r0 = img + w * j * 4; - const signed char* r1 = r0 + w; - const signed char* r2 = r1 + w; - const signed char* r3 = r2 + w; - const signed char* r4 = r3 + w; - const signed char* r5 = r4 + w; - - for (int i = 0; i < nRowBlocks; i++) - { - short* out_tm0 = bottom_blob_tm.channel(tiles * 0 + j * nRowBlocks + i).row(q); - short* out_tm1 = bottom_blob_tm.channel(tiles * 1 + j * nRowBlocks + i).row(q); - short* out_tm2 = bottom_blob_tm.channel(tiles * 2 + j * nRowBlocks + i).row(q); - short* out_tm3 = bottom_blob_tm.channel(tiles * 3 + j * nRowBlocks + i).row(q); - short* out_tm4 = bottom_blob_tm.channel(tiles * 4 + j * nRowBlocks + i).row(q); - short* out_tm5 = bottom_blob_tm.channel(tiles * 5 + j * nRowBlocks + i).row(q); - short* out_tm6 = bottom_blob_tm.channel(tiles * 6 + j * nRowBlocks + i).row(q); - short* out_tm7 = bottom_blob_tm.channel(tiles * 7 + j * nRowBlocks + i).row(q); - short* out_tm8 = bottom_blob_tm.channel(tiles * 8 + j * nRowBlocks + i).row(q); -#if __ARM_NEON - int8x8_t _d0, _d1, _d2, _d3, _d4, _d5; - int16x8_t _w0, _w1, _w2, _w3, _w4, _w5; - int16x8_t _t0, _t1, _t2, _t3, _t4, _t5; - int16x8_t _n0, _n1, _n2, _n3, _n4, _n5; - // load - _d0 = vld1_s8(r0); - _d1 = vld1_s8(r1); - _d2 = vld1_s8(r2); - _d3 = vld1_s8(r3); - _d4 = vld1_s8(r4); - _d5 = vld1_s8(r5); - - int8x8_t _1_n = vdup_n_s8(-1); - int8x8_t _2_p = vdup_n_s8(2); - int8x8_t _2_n = vdup_n_s8(-2); - int8x8_t _4_p = vdup_n_s8(4); - int8x8_t _4_n = vdup_n_s8(-4); - int8x8_t _5_n = vdup_n_s8(-5); - - int16x8_t _1_n_s16 = vdupq_n_s16(-1); - int16x8_t _2_p_s16 = vdupq_n_s16(2); - int16x8_t _2_n_s16 = vdupq_n_s16(-2); - int16x8_t _4_p_s16 = vdupq_n_s16(4); - int16x8_t _4_n_s16 = vdupq_n_s16(-4); - int16x8_t _5_n_s16 = vdupq_n_s16(-5); - // w = B_t * d - _w0 = vmull_s8(_d0, _4_p); - _w0 = vmlal_s8(_w0, _d2, _5_n); - _w0 = vaddw_s8(_w0, _d4); - - _w1 = vmull_s8(_d1, _4_n); - _w1 = vmlal_s8(_w1, _d2, _4_n); - _w1 = vaddw_s8(_w1, _d3); - _w1 = vaddw_s8(_w1, _d4); - - _w2 = vmull_s8(_d1, _4_p); - _w2 = vmlal_s8(_w2, _d2, _4_n); - _w2 = vmlal_s8(_w2, _d3, _1_n); - _w2 = vaddw_s8(_w2, _d4); - - _w3 = vmull_s8(_d1, _2_n); - _w3 = vmlal_s8(_w3, _d2, _1_n); - _w3 = vmlal_s8(_w3, _d3, _2_p); - _w3 = vaddw_s8(_w3, _d4); - - _w4 = vmull_s8(_d1, _2_p); - _w4 = vmlal_s8(_w4, _d2, _1_n); - _w4 = vmlal_s8(_w4, _d3, _2_n); - _w4 = vaddw_s8(_w4, _d4); - - _w5 = vmull_s8(_d1, _4_p); - _w5 = vmlal_s8(_w5, _d3, _5_n); - _w5 = vaddw_s8(_w5, _d5); - // transpose d to d_t - { - _t0[0] = _w0[0]; - _t1[0] = _w0[1]; - _t2[0] = _w0[2]; - _t3[0] = _w0[3]; - _t4[0] = _w0[4]; - _t5[0] = _w0[5]; - _t0[1] = _w1[0]; - _t1[1] = _w1[1]; - _t2[1] = _w1[2]; - _t3[1] = _w1[3]; - _t4[1] = _w1[4]; - _t5[1] = _w1[5]; - _t0[2] = _w2[0]; - _t1[2] = _w2[1]; - _t2[2] = _w2[2]; - _t3[2] = _w2[3]; - _t4[2] = _w2[4]; - _t5[2] = _w2[5]; - _t0[3] = _w3[0]; - _t1[3] = _w3[1]; - _t2[3] = _w3[2]; - _t3[3] = _w3[3]; - _t4[3] = _w3[4]; - _t5[3] = _w3[5]; - _t0[4] = _w4[0]; - _t1[4] = _w4[1]; - _t2[4] = _w4[2]; - _t3[4] = _w4[3]; - _t4[4] = _w4[4]; - _t5[4] = _w4[5]; - _t0[5] = _w5[0]; - _t1[5] = _w5[1]; - _t2[5] = _w5[2]; - _t3[5] = _w5[3]; - _t4[5] = _w5[4]; - _t5[5] = _w5[5]; - } - // d = B_t * d_t - _n0 = vmulq_s16(_t0, _4_p_s16); - _n0 = vmlaq_s16(_n0, _t2, _5_n_s16); - _n0 = vaddq_s16(_n0, _t4); - - _n1 = vmulq_s16(_t1, _4_n_s16); - _n1 = vmlaq_s16(_n1, _t2, _4_n_s16); - _n1 = vaddq_s16(_n1, _t3); - _n1 = vaddq_s16(_n1, _t4); - - _n2 = vmulq_s16(_t1, _4_p_s16); - _n2 = vmlaq_s16(_n2, _t2, _4_n_s16); - _n2 = vmlaq_s16(_n2, _t3, _1_n_s16); - _n2 = vaddq_s16(_n2, _t4); - - _n3 = vmulq_s16(_t1, _2_n_s16); - _n3 = vmlaq_s16(_n3, _t2, _1_n_s16); - _n3 = vmlaq_s16(_n3, _t3, _2_p_s16); - _n3 = vaddq_s16(_n3, _t4); - - _n4 = vmulq_s16(_t1, _2_p_s16); - _n4 = vmlaq_s16(_n4, _t2, _1_n_s16); - _n4 = vmlaq_s16(_n4, _t3, _2_n_s16); - _n4 = vaddq_s16(_n4, _t4); - - _n5 = vmulq_s16(_t1, _4_p_s16); - _n5 = vmlaq_s16(_n5, _t3, _5_n_s16); - _n5 = vaddq_s16(_n5, _t5); - // save to out_tm - out_tm0[0] = _n0[0]; - out_tm0[1] = _n0[1]; - out_tm0[2] = _n0[2]; - out_tm0[3] = _n0[3]; - out_tm1[0] = _n0[4]; - out_tm1[1] = _n0[5]; - out_tm1[2] = _n1[0]; - out_tm1[3] = _n1[1]; - out_tm2[0] = _n1[2]; - out_tm2[1] = _n1[3]; - out_tm2[2] = _n1[4]; - out_tm2[3] = _n1[5]; - - out_tm3[0] = _n2[0]; - out_tm3[1] = _n2[1]; - out_tm3[2] = _n2[2]; - out_tm3[3] = _n2[3]; - out_tm4[0] = _n2[4]; - out_tm4[1] = _n2[5]; - out_tm4[2] = _n3[0]; - out_tm4[3] = _n3[1]; - out_tm5[0] = _n3[2]; - out_tm5[1] = _n3[3]; - out_tm5[2] = _n3[4]; - out_tm5[3] = _n3[5]; - - out_tm6[0] = _n4[0]; - out_tm6[1] = _n4[1]; - out_tm6[2] = _n4[2]; - out_tm6[3] = _n4[3]; - out_tm7[0] = _n4[4]; - out_tm7[1] = _n4[5]; - out_tm7[2] = _n5[0]; - out_tm7[3] = _n5[1]; - out_tm8[0] = _n5[2]; - out_tm8[1] = _n5[3]; - out_tm8[2] = _n5[4]; - out_tm8[3] = _n5[5]; -#else - short d0[6], d1[6], d2[6], d3[6], d4[6], d5[6]; - short w0[6], w1[6], w2[6], w3[6], w4[6], w5[6]; - short t0[6], t1[6], t2[6], t3[6], t4[6], t5[6]; - - // load - for (int n = 0; n < 6; n++) - { - d0[n] = r0[n]; - d1[n] = r1[n]; - d2[n] = r2[n]; - d3[n] = r3[n]; - d4[n] = r4[n]; - d5[n] = r5[n]; - } - // w = B_t * d - for (int n = 0; n < 6; n++) - { - w0[n] = 4 * d0[n] - 5 * d2[n] + d4[n]; - w1[n] = -4 * d1[n] - 4 * d2[n] + d3[n] + d4[n]; - w2[n] = 4 * d1[n] - 4 * d2[n] - d3[n] + d4[n]; - w3[n] = -2 * d1[n] - d2[n] + 2 * d3[n] + d4[n]; - w4[n] = 2 * d1[n] - d2[n] - 2 * d3[n] + d4[n]; - w5[n] = 4 * d1[n] - 5 * d3[n] + d5[n]; - } - // transpose d to d_t - { - t0[0] = w0[0]; - t1[0] = w0[1]; - t2[0] = w0[2]; - t3[0] = w0[3]; - t4[0] = w0[4]; - t5[0] = w0[5]; - t0[1] = w1[0]; - t1[1] = w1[1]; - t2[1] = w1[2]; - t3[1] = w1[3]; - t4[1] = w1[4]; - t5[1] = w1[5]; - t0[2] = w2[0]; - t1[2] = w2[1]; - t2[2] = w2[2]; - t3[2] = w2[3]; - t4[2] = w2[4]; - t5[2] = w2[5]; - t0[3] = w3[0]; - t1[3] = w3[1]; - t2[3] = w3[2]; - t3[3] = w3[3]; - t4[3] = w3[4]; - t5[3] = w3[5]; - t0[4] = w4[0]; - t1[4] = w4[1]; - t2[4] = w4[2]; - t3[4] = w4[3]; - t4[4] = w4[4]; - t5[4] = w4[5]; - t0[5] = w5[0]; - t1[5] = w5[1]; - t2[5] = w5[2]; - t3[5] = w5[3]; - t4[5] = w5[4]; - t5[5] = w5[5]; - } - // d = B_t * d_t - for (int n = 0; n < 6; n++) - { - d0[n] = 4 * t0[n] - 5 * t2[n] + t4[n]; - d1[n] = -4 * t1[n] - 4 * t2[n] + t3[n] + t4[n]; - d2[n] = 4 * t1[n] - 4 * t2[n] - t3[n] + t4[n]; - d3[n] = -2 * t1[n] - t2[n] + 2 * t3[n] + t4[n]; - d4[n] = 2 * t1[n] - t2[n] - 2 * t3[n] + t4[n]; - d5[n] = 4 * t1[n] - 5 * t3[n] + t5[n]; - } - // save to out_tm - { - out_tm0[0] = d0[0]; - out_tm0[1] = d0[1]; - out_tm0[2] = d0[2]; - out_tm0[3] = d0[3]; - out_tm1[0] = d0[4]; - out_tm1[1] = d0[5]; - out_tm1[2] = d1[0]; - out_tm1[3] = d1[1]; - out_tm2[0] = d1[2]; - out_tm2[1] = d1[3]; - out_tm2[2] = d1[4]; - out_tm2[3] = d1[5]; - - out_tm3[0] = d2[0]; - out_tm3[1] = d2[1]; - out_tm3[2] = d2[2]; - out_tm3[3] = d2[3]; - out_tm4[0] = d2[4]; - out_tm4[1] = d2[5]; - out_tm4[2] = d3[0]; - out_tm4[3] = d3[1]; - out_tm5[0] = d3[2]; - out_tm5[1] = d3[3]; - out_tm5[2] = d3[4]; - out_tm5[3] = d3[5]; - - out_tm6[0] = d4[0]; - out_tm6[1] = d4[1]; - out_tm6[2] = d4[2]; - out_tm6[3] = d4[3]; - out_tm7[0] = d4[4]; - out_tm7[1] = d4[5]; - out_tm7[2] = d5[0]; - out_tm7[3] = d5[1]; - out_tm8[0] = d5[2]; - out_tm8[1] = d5[3]; - out_tm8[2] = d5[4]; - out_tm8[3] = d5[5]; - } -#endif // __ARM_NEON - r0 += 4; - r1 += 4; - r2 += 4; - r3 += 4; - r4 += 4; - r5 += 4; - } - } - } - } - bottom_blob_bordered = Mat(); - - // BEGIN dot - Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - const int tiles = nColBlocks * nRowBlocks; - - top_blob_tm.create(36, tiles, outch, 4u, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 9; r++) - { - int nn_outch = 0; - int remain_outch_start = 0; - - nn_outch = outch >> 3; - remain_outch_start = nn_outch << 3; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - int* output4_tm = top_blob_tm.channel(p + 4); - int* output5_tm = top_blob_tm.channel(p + 5); - int* output6_tm = top_blob_tm.channel(p + 6); - int* output7_tm = top_blob_tm.channel(p + 7); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - output4_tm = output4_tm + r * 4; - output5_tm = output5_tm + r * 4; - output6_tm = output6_tm + r * 4; - output7_tm = output7_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "eor v4.16b, v4.16b, v4.16b \n" - "eor v5.16b, v5.16b, v5.16b \n" - "eor v6.16b, v6.16b, v6.16b \n" - "eor v7.16b, v7.16b, v7.16b \n" - "mov w4, %w20 \n" - - "0: \n" // for (int q=0; q> 2; - - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - - output0_tm = output0_tm + r * 4; - output1_tm = output1_tm + r * 4; - output2_tm = output2_tm + r * 4; - output3_tm = output3_tm + r * 4; - - for (int i = 0; i < tiles; i++) - { - const short* kptr = kernel_tm_test[r].channel(p / 8 + (p % 8) / 4); - const short* r0 = bottom_blob_tm.channel(tiles * r + i); -#if __ARM_NEON -#if __aarch64__ - asm volatile( - // inch loop - "eor v0.16b, v0.16b, v0.16b \n" - "eor v1.16b, v1.16b, v1.16b \n" - "eor v2.16b, v2.16b, v2.16b \n" - "eor v3.16b, v3.16b, v3.16b \n" - "mov w4, %w12 \n" - - "0: \n" // for (int q=0; q(i); - - // int sum0[36] = {0}; - - // for (int q=0; q(i); - // const short* k0 = kernel0_tm.row(q); - - // for (int n=0; n<36; n++) - // { - // sum0[n] += (int)r0[n] * k0[n]; - // } - // } - - // for (int n=0; n<36; n++) - // { - // output0_tm[n] = sum0[n]; - // } - // } - // } - } - } - bottom_blob_tm = Mat(); - // END dot - - // BEGIN transform output - Mat top_blob_bordered; - top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator); - { - // AT - // const float itm[4][6] = { - // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, - // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} - // }; - - // 0 = r00 + r01 + r02 + r03 + r04 - // 1 = r01 - r02 + 2 * (r03 - r04) - // 2 = r01 + r02 + 4 * (r03 + r04) - // 3 = r01 - r02 + 8 * (r03 - r04) + r05 - - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - int nColBlocks = h_tm / 6; // may be the block num in Feathercnn - int nRowBlocks = w_tm / 6; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - int* out_tile = top_blob_tm.channel(p); - float* outRow0 = top_blob_bordered.channel(p); - float* outRow1 = outRow0 + outw; - float* outRow2 = outRow0 + outw * 2; - float* outRow3 = outRow0 + outw * 3; - - const float bias0 = bias ? bias[p] : 0.f; - - const float scale_dequant0 = scales_dequant[p]; - - const float scale0 = scale_dequant0 / 576.0; - - for (int j = 0; j < nColBlocks; j++) - { - for (int i = 0; i < nRowBlocks; i++) - { -#if __ARM_NEON - int32x4_t _s0, _s1, _s2, _s3, _s4, _s5; - int32x2_t _s0n, _s1n, _s2n, _s3n, _s4n, _s5n; - int32x4_t _w0, _w3; - int32x2_t _w0n, _w3n; - int32x4_t _d0, _d1, _d2, _d3, _d4, _d5; - int32x4_t _o0, _o3; - // load - _s0 = vld1q_s32(out_tile); - _s0n = vld1_s32(out_tile + 4); - _s1 = vld1q_s32(out_tile + 6); - _s1n = vld1_s32(out_tile + 10); - _s2 = vld1q_s32(out_tile + 12); - _s2n = vld1_s32(out_tile + 16); - _s3 = vld1q_s32(out_tile + 18); - _s3n = vld1_s32(out_tile + 22); - _s4 = vld1q_s32(out_tile + 24); - _s4n = vld1_s32(out_tile + 28); - _s5 = vld1q_s32(out_tile + 30); - _s5n = vld1_s32(out_tile + 34); - // w = A_T * W - int32x2_t _tp0 = {1, 4}; - int32x2_t _tp1 = {2, 8}; - - // 4*s5[n] - int32x4_t _s5x4 = vshlq_n_s32(_s5, 2); - int32x2_t _s5x4n = vshl_n_s32(_s5n, 2); - - int32x4_t _t1p2 = vaddq_s32(_s1, _s2); - int32x2_t _t1p2n = vadd_s32(_s1n, _s2n); - int32x4_t _t3p4 = vaddq_s32(_s3, _s4); - int32x2_t _t3p4n = vadd_s32(_s3n, _s4n); - int32x4_t _t1s2 = vsubq_s32(_s1, _s2); - int32x2_t _t1s2n = vsub_s32(_s1n, _s2n); - int32x4_t _t3s4 = vsubq_s32(_s3, _s4); - int32x2_t _t3s4n = vsub_s32(_s3n, _s4n); - - _w0 = vaddq_s32(_s0, _t1p2); - _w0n = vadd_s32(_s0n, _t1p2n); - _w0 = vaddq_s32(_w0, _t3p4); - _w0n = vadd_s32(_w0n, _t3p4n); - _w0n = vmul_s32(_w0n, _tp0); - - // _w2,_w2n - _t1p2 = vmlaq_lane_s32(_t1p2, _t3p4, _tp0, 1); - _t1p2n = vmla_lane_s32(_t1p2n, _t3p4n, _tp0, 1); - _t1p2n = vmul_s32(_t1p2n, _tp0); - - _w3 = vaddq_s32(_s5x4, _t1s2); - _w3n = vadd_s32(_s5x4n, _t1s2n); - _w3 = vmlaq_lane_s32(_w3, _t3s4, _tp1, 1); - _w3n = vmla_lane_s32(_w3n, _t3s4n, _tp1, 1); - _w3n = vmul_s32(_w3n, _tp0); - - // _w1, _w1n - _t1s2 = vmlaq_lane_s32(_t1s2, _t3s4, _tp1, 0); - _t1s2n = vmla_lane_s32(_t1s2n, _t3s4n, _tp1, 0); - _t1s2n = vmul_s32(_t1s2n, _tp0); - - int32x4_t _w02n = vcombine_s32(_w0n, _t1p2n); - int32x4_t _w13n = vcombine_s32(_t1s2n, _w3n); - - // transpose w to w_t -#if __aarch64__ - int32x4_t _wt0 = vtrn1q_s32(_w0, _t1s2); - int32x4_t _wt1 = vtrn2q_s32(_w0, _t1s2); - int32x4_t _wt2 = vtrn1q_s32(_t1p2, _w3); - int32x4_t _wt3 = vtrn2q_s32(_t1p2, _w3); - int64x2_t _dt0 = vtrn1q_s64(vreinterpretq_s64_s32(_wt0), vreinterpretq_s64_s32(_wt2)); - int64x2_t _dt2 = vtrn2q_s64(vreinterpretq_s64_s32(_wt0), vreinterpretq_s64_s32(_wt2)); - int64x2_t _dt1 = vtrn1q_s64(vreinterpretq_s64_s32(_wt1), vreinterpretq_s64_s32(_wt3)); - int64x2_t _dt3 = vtrn2q_s64(vreinterpretq_s64_s32(_wt1), vreinterpretq_s64_s32(_wt3)); - _d0 = vreinterpretq_s32_s64(_dt0); - _d1 = vreinterpretq_s32_s64(_dt1); - _d2 = vreinterpretq_s32_s64(_dt2); - _d3 = vreinterpretq_s32_s64(_dt3); - _d4 = vtrn1q_s32(_w02n, _w13n); - _d5 = vtrn2q_s32(_w02n, _w13n); -#else - asm volatile( - "vtrn.32 %q[_w0], %q[_w1] \n" - "vtrn.32 %q[_w2], %q[_w3] \n" - "vswp %f[_w0], %e[_w2] \n" - "vswp %f[_w1], %e[_w3] \n" - "vtrn.32 %q[_w02n], %q[_w13n] \n" - : [_w0] "+w"(_w0), - [_w1] "+w"(_t1s2), - [_w2] "+w"(_t1p2), - [_w3] "+w"(_w3), - [_w02n] "+w"(_w02n), - [_w13n] "+w"(_w13n) - : - : "cc", "memory"); - _d0 = _w0; - _d1 = _t1s2; - _d2 = _t1p2; - _d3 = _w3; - _d4 = _w02n; - _d5 = _w13n; -#endif - // Y = A_T * w_t - _t1p2 = vaddq_s32(_d1, _d2); - _t3p4 = vaddq_s32(_d3, _d4); - _t1s2 = vsubq_s32(_d1, _d2); - _t3s4 = vsubq_s32(_d3, _d4); - - _o0 = vaddq_s32(_d0, _t1p2); - _o0 = vaddq_s32(_o0, _t3p4); - - // _o2 - _t1p2 = vmlaq_lane_s32(_t1p2, _t3p4, _tp0, 1); - - _o3 = vaddq_s32(_d5, _t1s2); - _o3 = vmlaq_lane_s32(_o3, _t3s4, _tp1, 1); - - // _o1 - _t1s2 = vmlaq_lane_s32(_t1s2, _t3s4, _tp1, 0); - - // save to top blob tm - float32x4_t _scale0 = vdupq_n_f32(scale0); - float32x4_t _out0_f32 = vdupq_n_f32(bias0); - float32x4_t _out1_f32 = vdupq_n_f32(bias0); - float32x4_t _out2_f32 = vdupq_n_f32(bias0); - float32x4_t _out3_f32 = vdupq_n_f32(bias0); - - _out0_f32 = vmlaq_f32(_out0_f32, vcvtq_f32_s32(_o0), _scale0); - _out1_f32 = vmlaq_f32(_out1_f32, vcvtq_f32_s32(_t1s2), _scale0); - _out2_f32 = vmlaq_f32(_out2_f32, vcvtq_f32_s32(_t1p2), _scale0); - _out3_f32 = vmlaq_f32(_out3_f32, vcvtq_f32_s32(_o3), _scale0); - - vst1q_f32(outRow0, _out0_f32); - vst1q_f32(outRow1, _out1_f32); - vst1q_f32(outRow2, _out2_f32); - vst1q_f32(outRow3, _out3_f32); -#else - int s0[6], s1[6], s2[6], s3[6], s4[6], s5[6]; - int w0[6], w1[6], w2[6], w3[6]; - int d0[4], d1[4], d2[4], d3[4], d4[4], d5[4]; - int o0[4], o1[4], o2[4], o3[4]; - - // load - for (int n = 0; n < 6; n++) - { - s0[n] = out_tile[n]; - s1[n] = out_tile[n + 6]; - s2[n] = out_tile[n + 12]; - s3[n] = out_tile[n + 18]; - s4[n] = out_tile[n + 24]; - s5[n] = out_tile[n + 30]; - } - // w = A_T * W - for (int n = 0; n < 5; n++) - { - w0[n] = s0[n] + s1[n] + s2[n] + s3[n] + s4[n]; - w1[n] = s1[n] - s2[n] + 2 * s3[n] - 2 * s4[n]; - w2[n] = s1[n] + s2[n] + 4 * s3[n] + 4 * s4[n]; - w3[n] = s1[n] - s2[n] + 8 * s3[n] - 8 * s4[n] + 4 * s5[n]; - } - for (int n = 5; n < 6; n++) - { - w0[n] = 4 * (s0[n] + s1[n] + s2[n] + s3[n] + s4[n]); - w1[n] = 4 * (s1[n] - s2[n] + 2 * s3[n] - 2 * s4[n]); - w2[n] = 4 * (s1[n] + s2[n] + 4 * s3[n] + 4 * s4[n]); - w3[n] = 4 * (s1[n] - s2[n] + 8 * s3[n] - 8 * s4[n] + 4 * s5[n]); - } - // transpose w to w_t - { - d0[0] = w0[0]; - d0[1] = w1[0]; - d0[2] = w2[0]; - d0[3] = w3[0]; - d1[0] = w0[1]; - d1[1] = w1[1]; - d1[2] = w2[1]; - d1[3] = w3[1]; - d2[0] = w0[2]; - d2[1] = w1[2]; - d2[2] = w2[2]; - d2[3] = w3[2]; - d3[0] = w0[3]; - d3[1] = w1[3]; - d3[2] = w2[3]; - d3[3] = w3[3]; - d4[0] = w0[4]; - d4[1] = w1[4]; - d4[2] = w2[4]; - d4[3] = w3[4]; - d5[0] = w0[5]; - d5[1] = w1[5]; - d5[2] = w2[5]; - d5[3] = w3[5]; - } - // Y = A_T * w_t - for (int n = 0; n < 4; n++) - { - o0[n] = d0[n] + d1[n] + d2[n] + d3[n] + d4[n]; - o1[n] = d1[n] - d2[n] + 2 * d3[n] - 2 * d4[n]; - o2[n] = d1[n] + d2[n] + 4 * d3[n] + 4 * d4[n]; - o3[n] = d1[n] - d2[n] + 8 * d3[n] - 8 * d4[n] + d5[n]; - } - // save to top blob tm - for (int n = 0; n < 4; n++) - { - outRow0[n] = (float)o0[n] * scale0 + bias0; - outRow1[n] = (float)o1[n] * scale0 + bias0; - outRow2[n] = (float)o2[n] * scale0 + bias0; - outRow3[n] = (float)o3[n] * scale0 + bias0; - } -#endif // __ARM_NEON - out_tile += 36; - - outRow0 += 4; - outRow1 += 4; - outRow2 += 4; - outRow3 += 4; - } - - outRow0 += outw * 3; - outRow1 += outw * 3; - outRow2 += outw * 3; - outRow3 += outw * 3; - } - } + conv3x3s1_winograd43_transform_output_int8_neon(top_blob_tm, top_blob_bordered, opt); } // END transform output diff --git a/src/layer/arm/convolution_3x3_pack8to1_int8.h b/src/layer/arm/convolution_3x3_pack8to1_int8.h index b4f64d77e..5af9f5938 100644 --- a/src/layer/arm/convolution_3x3_pack8to1_int8.h +++ b/src/layer/arm/convolution_3x3_pack8to1_int8.h @@ -150,896 +150,19 @@ static void conv3x3s1_winograd43_pack8to1_int8_neon(const Mat& bottom_blob, Mat& // BEGIN transform input Mat bottom_blob_tm; { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = w_tm / 6 * h_tm / 6; + int w_tiles = outw / 4; + int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator); - - // const float itm[4][4] = { - // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, - // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, - // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, - // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} - // }; - - // 0 = 4 * r00 - 5 * r02 + r04 - // 1 = -4 * (r01 + r02) + r04 + r03 - // 2 = 4 * (r01 - r02) + r04 - r03 - // 3 = -2 * (r01 - r03) + r04 - r02 - // 4 = 2 * (r01 - r03) + r04 - r02 - // 5 = 4 * r01 - 5 * r03 + r05 - - #pragma omp parallel for num_threads(opt.num_threads) - for (int q = 0; q < inch; q++) - { - const Mat img0 = bottom_blob_bordered.channel(q); - Mat img0_tm = bottom_blob_tm.channel(q); - - short tmp[6][6][8]; - - // tile - for (int i = 0; i < h_tm / 6; i++) - { - for (int j = 0; j < w_tm / 6; j++) - { - const signed char* r0 = img0.row(i * 4) + (j * 4) * 8; - - for (int m = 0; m < 6; m++) - { - int8x8_t _r00 = vld1_s8(r0); - int8x8_t _r01 = vld1_s8(r0 + 8); - int8x8_t _r02 = vld1_s8(r0 + 16); - int8x8_t _r03 = vld1_s8(r0 + 24); - int8x8_t _r04 = vld1_s8(r0 + 32); - int8x8_t _r05 = vld1_s8(r0 + 40); - - int8x8_t _v4s8 = vdup_n_s8(4); - int8x8_t _v5s8 = vdup_n_s8(5); - int16x8_t _v2 = vdupq_n_s16(2); - int16x8_t _v4 = vdupq_n_s16(4); - - // int16x8_t _tmp0m = vfmsq_n_f16(vfmaq_n_f16(_r04, _r00, 4.f), _r02, 5.f); - int16x8_t _tmp0m = vsubq_s16(vaddw_s8(vmull_s8(_r00, _v4s8), _r04), vmull_s8(_r02, _v5s8)); - - // int16x8_t _tmp1m = vfmsq_n_f16(vaddq_f16(_r04, _r03), vaddq_f16(_r01, _r02), 4.f); - int16x8_t _tmp1m = vmlsq_s16(vaddl_s8(_r04, _r03), vaddl_s8(_r01, _r02), _v4); - - // int16x8_t _tmp2m = vfmaq_n_f16(vsubq_f16(_r04, _r03), vsubq_f16(_r01, _r02), 4.f); - int16x8_t _tmp2m = vmlaq_s16(vsubl_s8(_r04, _r03), vsubl_s8(_r01, _r02), _v4); - - // int16x8_t _tmp3m = vfmsq_n_f16(vsubq_f16(_r04, _r02), vsubq_f16(_r01, _r03), 2.f); - int16x8_t _tmp3m = vmlsq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); - - // int16x8_t _tmp4m = vfmaq_n_f16(vsubq_f16(_r04, _r02), vsubq_f16(_r01, _r03), 2.f); - int16x8_t _tmp4m = vmlaq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); - - // int16x8_t _tmp5m = vfmsq_n_f16(vfmaq_n_f16(_r05, _r01, 4.f), _r03, 5.f); - int16x8_t _tmp5m = vsubq_s16(vaddw_s8(vmull_s8(_r01, _v4s8), _r05), vmull_s8(_r03, _v5s8)); - - vst1q_s16(tmp[0][m], _tmp0m); - vst1q_s16(tmp[1][m], _tmp1m); - vst1q_s16(tmp[2][m], _tmp2m); - vst1q_s16(tmp[3][m], _tmp3m); - vst1q_s16(tmp[4][m], _tmp4m); - vst1q_s16(tmp[5][m], _tmp5m); - - r0 += w * 8; - } - - short* r0_tm_0 = (short*)img0_tm + (i * w_tm / 6 + j) * 8; - short* r0_tm_1 = r0_tm_0 + tiles * 8; - short* r0_tm_2 = r0_tm_0 + tiles * 16; - short* r0_tm_3 = r0_tm_0 + tiles * 24; - short* r0_tm_4 = r0_tm_0 + tiles * 32; - short* r0_tm_5 = r0_tm_0 + tiles * 40; - - for (int m = 0; m < 6; m++) - { - int16x8_t _tmp00 = vld1q_s16(tmp[m][0]); - int16x8_t _tmp01 = vld1q_s16(tmp[m][1]); - int16x8_t _tmp02 = vld1q_s16(tmp[m][2]); - int16x8_t _tmp03 = vld1q_s16(tmp[m][3]); - int16x8_t _tmp04 = vld1q_s16(tmp[m][4]); - int16x8_t _tmp05 = vld1q_s16(tmp[m][5]); - - int16x8_t _v2 = vdupq_n_s16(2); - int16x8_t _v4 = vdupq_n_s16(4); - int16x8_t _v5 = vdupq_n_s16(5); - - int16x8_t _r0tm0 = vmlsq_s16(vmlaq_s16(_tmp04, _tmp00, _v4), _tmp02, _v5); - int16x8_t _r0tm1 = vmlsq_s16(vaddq_s16(_tmp04, _tmp03), vaddq_s16(_tmp01, _tmp02), _v4); - int16x8_t _r0tm2 = vmlaq_s16(vsubq_s16(_tmp04, _tmp03), vsubq_s16(_tmp01, _tmp02), _v4); - int16x8_t _r0tm3 = vmlsq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); - int16x8_t _r0tm4 = vmlaq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); - int16x8_t _r0tm5 = vmlsq_s16(vmlaq_s16(_tmp05, _tmp01, _v4), _tmp03, _v5); - - vst1q_s16(r0_tm_0, _r0tm0); - vst1q_s16(r0_tm_1, _r0tm1); - vst1q_s16(r0_tm_2, _r0tm2); - vst1q_s16(r0_tm_3, _r0tm3); - vst1q_s16(r0_tm_4, _r0tm4); - vst1q_s16(r0_tm_5, _r0tm5); - - r0_tm_0 += tiles * 48; - r0_tm_1 += tiles * 48; - r0_tm_2 += tiles * 48; - r0_tm_3 += tiles * 48; - r0_tm_4 += tiles * 48; - r0_tm_5 += tiles * 48; - } - } - } - } + conv3x3s1_winograd43_transform_input_pack8_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt); } bottom_blob_bordered = Mat(); // END transform input // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = h_tm / 6 * w_tm / 6; - - // permute - // bottom_blob_tm.create(tiles, 36, inch, elemsize, elempack, opt.workspace_allocator); - Mat bottom_blob_tm2; -#if __aarch64__ - if (tiles >= 8) - bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + tiles % 4, 36, 2u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 36, 2u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 36, 2u * elempack, elempack, opt.workspace_allocator); -#else - if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 36, 2u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 36, 2u * elempack, elempack, opt.workspace_allocator); -#endif // __aarch64__ - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 36; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; -#if __aarch64__ - for (; i + 7 < tiles; i += 8) - { - short* tm2p = tm2.row(i / 8); - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { - // transpose 8x8 - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" - "sub %0, %0, #64 \n" - - "uzp1 v16.8h, v0.8h, v4.8h \n" - "uzp2 v20.8h, v0.8h, v4.8h \n" - "uzp1 v17.8h, v1.8h, v5.8h \n" - "uzp2 v21.8h, v1.8h, v5.8h \n" - "uzp1 v18.8h, v2.8h, v6.8h \n" - "uzp2 v22.8h, v2.8h, v6.8h \n" - "uzp1 v19.8h, v3.8h, v7.8h \n" - "uzp2 v23.8h, v3.8h, v7.8h \n" - - "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - - r0 += bottom_blob_tm.cstep * 8; - } - } -#endif - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - short* tm2p = tm2.row(i / 8 + (i % 8) / 4); -#else - short* tm2p = tm2.row(i / 4); -#endif - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { - // transpose 8x4 -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" - "st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "v0", "v1", "v2", "v3"); -#else - asm volatile( - "pld [%0, #512] \n" - "vldm %0, {d0-d7} \n" - "vswp d1, d2 \n" - "vswp d5, d6 \n" - "vswp q1, q2 \n" - "vst4.s16 {d0-d3}, [%1 :64]! \n" - "vst4.s16 {d4-d7}, [%1 :64]! \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "q0", "q1", "q2", "q3"); -#endif // __aarch64__ - r0 += bottom_blob_tm.cstep * 8; - } - } - for (; i < tiles; i++) - { -#if __aarch64__ - short* tm2p = tm2.row(i / 8 + (i % 8) / 4 + i % 4); -#else - short* tm2p = tm2.row(i / 4 + i % 4); -#endif - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #128] \n" - "ld1 {v0.8h}, [%0] \n" - "st1 {v0.8h}, [%1], #16 \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "v0"); -#else - asm volatile( - "pld [%0, #128] \n" - "vld1.s16 {d0-d1}, [%0 :64] \n" - "vst1.s16 {d0-d1}, [%1 :64]! \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "q0"); -#endif // __aarch64__ - r0 += bottom_blob_tm.cstep * 8; - } - } - } - - bottom_blob_tm = Mat(); - // permute end - - top_blob_tm.create(tiles, 36, outch, 4u, 1, opt.workspace_allocator); - - int nn_outch = 0; - int remain_outch_start = 0; - - nn_outch = outch >> 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - int* output2_tm = top_blob_tm.channel(p + 2); - int* output3_tm = top_blob_tm.channel(p + 3); - int* output4_tm = top_blob_tm.channel(p + 4); - int* output5_tm = top_blob_tm.channel(p + 5); - int* output6_tm = top_blob_tm.channel(p + 6); - int* output7_tm = top_blob_tm.channel(p + 7); - - const Mat kernel01_tm = kernel_tm.channel(p / 8); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; -#if __aarch64__ - for (; i + 7 < tiles; i += 8) - { - const short* r0 = bb2.row(i / 8); - const short* kptr = kernel01_tm.row(r); - - int nn = inch; // inch always > 0 - - asm volatile( - "eor v16.16b, v16.16b, v16.16b \n" - "eor v17.16b, v17.16b, v17.16b \n" - "eor v18.16b, v18.16b, v18.16b \n" - "eor v19.16b, v19.16b, v19.16b \n" - "eor v20.16b, v20.16b, v20.16b \n" - "eor v21.16b, v21.16b, v21.16b \n" - "eor v22.16b, v22.16b, v22.16b \n" - "eor v23.16b, v23.16b, v23.16b \n" - "eor v24.16b, v24.16b, v24.16b \n" - "eor v25.16b, v25.16b, v25.16b \n" - "eor v26.16b, v26.16b, v26.16b \n" - "eor v27.16b, v27.16b, v27.16b \n" - "eor v28.16b, v28.16b, v28.16b \n" - "eor v29.16b, v29.16b, v29.16b \n" - "eor v30.16b, v30.16b, v30.16b \n" - "eor v31.16b, v31.16b, v31.16b \n" - - "0: \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%9], #64 \n" - - "prfm pldl1keep, [%10, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%10], #64 \n" - - "smlal v16.4s, v8.4h, v0.h[0] \n" - "smlal2 v17.4s, v8.8h, v0.h[0] \n" - "smlal v18.4s, v8.4h, v0.h[1] \n" - "smlal2 v19.4s, v8.8h, v0.h[1] \n" - "smlal v20.4s, v8.4h, v0.h[2] \n" - "smlal2 v21.4s, v8.8h, v0.h[2] \n" - "smlal v22.4s, v8.4h, v0.h[3] \n" - "smlal2 v23.4s, v8.8h, v0.h[3] \n" - "smlal v24.4s, v8.4h, v0.h[4] \n" - "smlal2 v25.4s, v8.8h, v0.h[4] \n" - "smlal v26.4s, v8.4h, v0.h[5] \n" - "smlal2 v27.4s, v8.8h, v0.h[5] \n" - "smlal v28.4s, v8.4h, v0.h[6] \n" - "smlal2 v29.4s, v8.8h, v0.h[6] \n" - "smlal v30.4s, v8.4h, v0.h[7] \n" - "smlal2 v31.4s, v8.8h, v0.h[7] \n" - - "smlal v16.4s, v9.4h, v1.h[0] \n" - "smlal2 v17.4s, v9.8h, v1.h[0] \n" - "smlal v18.4s, v9.4h, v1.h[1] \n" - "smlal2 v19.4s, v9.8h, v1.h[1] \n" - "smlal v20.4s, v9.4h, v1.h[2] \n" - "smlal2 v21.4s, v9.8h, v1.h[2] \n" - "smlal v22.4s, v9.4h, v1.h[3] \n" - "smlal2 v23.4s, v9.8h, v1.h[3] \n" - "smlal v24.4s, v9.4h, v1.h[4] \n" - "smlal2 v25.4s, v9.8h, v1.h[4] \n" - "smlal v26.4s, v9.4h, v1.h[5] \n" - "smlal2 v27.4s, v9.8h, v1.h[5] \n" - "smlal v28.4s, v9.4h, v1.h[6] \n" - "smlal2 v29.4s, v9.8h, v1.h[6] \n" - "smlal v30.4s, v9.4h, v1.h[7] \n" - "smlal2 v31.4s, v9.8h, v1.h[7] \n" - - "prfm pldl1keep, [%9, #512] \n" - "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%9], #64 \n" - - "smlal v16.4s, v10.4h, v2.h[0] \n" - "smlal2 v17.4s, v10.8h, v2.h[0] \n" - "smlal v18.4s, v10.4h, v2.h[1] \n" - "smlal2 v19.4s, v10.8h, v2.h[1] \n" - "smlal v20.4s, v10.4h, v2.h[2] \n" - "smlal2 v21.4s, v10.8h, v2.h[2] \n" - "smlal v22.4s, v10.4h, v2.h[3] \n" - "smlal2 v23.4s, v10.8h, v2.h[3] \n" - "smlal v24.4s, v10.4h, v2.h[4] \n" - "smlal2 v25.4s, v10.8h, v2.h[4] \n" - "smlal v26.4s, v10.4h, v2.h[5] \n" - "smlal2 v27.4s, v10.8h, v2.h[5] \n" - "smlal v28.4s, v10.4h, v2.h[6] \n" - "smlal2 v29.4s, v10.8h, v2.h[6] \n" - "smlal v30.4s, v10.4h, v2.h[7] \n" - "smlal2 v31.4s, v10.8h, v2.h[7] \n" - - "prfm pldl1keep, [%10, #512] \n" - "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%10], #64 \n" - - "smlal v16.4s, v11.4h, v3.h[0] \n" - "smlal2 v17.4s, v11.8h, v3.h[0] \n" - "smlal v18.4s, v11.4h, v3.h[1] \n" - "smlal2 v19.4s, v11.8h, v3.h[1] \n" - "smlal v20.4s, v11.4h, v3.h[2] \n" - "smlal2 v21.4s, v11.8h, v3.h[2] \n" - "smlal v22.4s, v11.4h, v3.h[3] \n" - "smlal2 v23.4s, v11.8h, v3.h[3] \n" - "smlal v24.4s, v11.4h, v3.h[4] \n" - "smlal2 v25.4s, v11.8h, v3.h[4] \n" - "smlal v26.4s, v11.4h, v3.h[5] \n" - "smlal2 v27.4s, v11.8h, v3.h[5] \n" - "smlal v28.4s, v11.4h, v3.h[6] \n" - "smlal2 v29.4s, v11.8h, v3.h[6] \n" - "smlal v30.4s, v11.4h, v3.h[7] \n" - "smlal2 v31.4s, v11.8h, v3.h[7] \n" - - "smlal v16.4s, v12.4h, v4.h[0] \n" - "smlal2 v17.4s, v12.8h, v4.h[0] \n" - "smlal v18.4s, v12.4h, v4.h[1] \n" - "smlal2 v19.4s, v12.8h, v4.h[1] \n" - "smlal v20.4s, v12.4h, v4.h[2] \n" - "smlal2 v21.4s, v12.8h, v4.h[2] \n" - "smlal v22.4s, v12.4h, v4.h[3] \n" - "smlal2 v23.4s, v12.8h, v4.h[3] \n" - "smlal v24.4s, v12.4h, v4.h[4] \n" - "smlal2 v25.4s, v12.8h, v4.h[4] \n" - "smlal v26.4s, v12.4h, v4.h[5] \n" - "smlal2 v27.4s, v12.8h, v4.h[5] \n" - "smlal v28.4s, v12.4h, v4.h[6] \n" - "smlal2 v29.4s, v12.8h, v4.h[6] \n" - "smlal v30.4s, v12.4h, v4.h[7] \n" - "smlal2 v31.4s, v12.8h, v4.h[7] \n" - - "smlal v16.4s, v13.4h, v5.h[0] \n" - "smlal2 v17.4s, v13.8h, v5.h[0] \n" - "smlal v18.4s, v13.4h, v5.h[1] \n" - "smlal2 v19.4s, v13.8h, v5.h[1] \n" - "smlal v20.4s, v13.4h, v5.h[2] \n" - "smlal2 v21.4s, v13.8h, v5.h[2] \n" - "smlal v22.4s, v13.4h, v5.h[3] \n" - "smlal2 v23.4s, v13.8h, v5.h[3] \n" - "smlal v24.4s, v13.4h, v5.h[4] \n" - "smlal2 v25.4s, v13.8h, v5.h[4] \n" - "smlal v26.4s, v13.4h, v5.h[5] \n" - "smlal2 v27.4s, v13.8h, v5.h[5] \n" - "smlal v28.4s, v13.4h, v5.h[6] \n" - "smlal2 v29.4s, v13.8h, v5.h[6] \n" - "smlal v30.4s, v13.4h, v5.h[7] \n" - "smlal2 v31.4s, v13.8h, v5.h[7] \n" - - "smlal v16.4s, v14.4h, v6.h[0] \n" - "smlal2 v17.4s, v14.8h, v6.h[0] \n" - "smlal v18.4s, v14.4h, v6.h[1] \n" - "smlal2 v19.4s, v14.8h, v6.h[1] \n" - "smlal v20.4s, v14.4h, v6.h[2] \n" - "smlal2 v21.4s, v14.8h, v6.h[2] \n" - "smlal v22.4s, v14.4h, v6.h[3] \n" - "smlal2 v23.4s, v14.8h, v6.h[3] \n" - "smlal v24.4s, v14.4h, v6.h[4] \n" - "smlal2 v25.4s, v14.8h, v6.h[4] \n" - "smlal v26.4s, v14.4h, v6.h[5] \n" - "smlal2 v27.4s, v14.8h, v6.h[5] \n" - "smlal v28.4s, v14.4h, v6.h[6] \n" - "smlal2 v29.4s, v14.8h, v6.h[6] \n" - "smlal v30.4s, v14.4h, v6.h[7] \n" - "smlal2 v31.4s, v14.8h, v6.h[7] \n" - - "subs %w0, %w0, #1 \n" - - "smlal v16.4s, v15.4h, v7.h[0] \n" - "smlal2 v17.4s, v15.8h, v7.h[0] \n" - "smlal v18.4s, v15.4h, v7.h[1] \n" - "smlal2 v19.4s, v15.8h, v7.h[1] \n" - "smlal v20.4s, v15.4h, v7.h[2] \n" - "smlal2 v21.4s, v15.8h, v7.h[2] \n" - "smlal v22.4s, v15.4h, v7.h[3] \n" - "smlal2 v23.4s, v15.8h, v7.h[3] \n" - "smlal v24.4s, v15.4h, v7.h[4] \n" - "smlal2 v25.4s, v15.8h, v7.h[4] \n" - "smlal v26.4s, v15.4h, v7.h[5] \n" - "smlal2 v27.4s, v15.8h, v7.h[5] \n" - "smlal v28.4s, v15.4h, v7.h[6] \n" - "smlal2 v29.4s, v15.8h, v7.h[6] \n" - "smlal v30.4s, v15.4h, v7.h[7] \n" - "smlal2 v31.4s, v15.8h, v7.h[7] \n" - - "bne 0b \n" - - "st1 {v16.4s, v17.4s}, [%1], #32 \n" - "st1 {v18.4s, v19.4s}, [%2], #32 \n" - "st1 {v20.4s, v21.4s}, [%3], #32 \n" - "st1 {v22.4s, v23.4s}, [%4], #32 \n" - "st1 {v24.4s, v25.4s}, [%5], #32 \n" - "st1 {v26.4s, v27.4s}, [%6], #32 \n" - "st1 {v28.4s, v29.4s}, [%7], #32 \n" - "st1 {v30.4s, v31.4s}, [%8], #32 \n" - - : "=r"(nn), // %0 - "=r"(output0_tm), // %1 - "=r"(output1_tm), // %2 - "=r"(output2_tm), // %3 - "=r"(output3_tm), // %4 - "=r"(output4_tm), // %5 - "=r"(output5_tm), // %6 - "=r"(output6_tm), // %7 - "=r"(output7_tm), // %8 - "=r"(r0), // %9 - "=r"(kptr) // %10 - : "0"(nn), - "1"(output0_tm), - "2"(output1_tm), - "3"(output2_tm), - "4"(output3_tm), - "5"(output4_tm), - "6"(output5_tm), - "7"(output6_tm), - "8"(output7_tm), - "9"(r0), - "10"(kptr) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } -#endif - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 8 + (i % 8) / 4); -#else - const short* r0 = bb2.row(i / 4); -#endif - const short* k0 = kernel01_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - int32x4_t _sum4 = vdupq_n_s32(0); - int32x4_t _sum5 = vdupq_n_s32(0); - int32x4_t _sum6 = vdupq_n_s32(0); - int32x4_t _sum7 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - int16x8_t _val2 = vld1q_s16(r0 + 16); - int16x8_t _val3 = vld1q_s16(r0 + 24); - - int16x8_t _w0 = vld1q_s16(k0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val0), vget_low_s16(_w0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val0), vget_low_s16(_w0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val0), vget_low_s16(_w0), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val0), vget_low_s16(_w0), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val0), vget_high_s16(_w0), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val0), vget_high_s16(_w0), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val0), vget_high_s16(_w0), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val0), vget_high_s16(_w0), 3); - - int16x8_t _w1 = vld1q_s16(k0 + 8); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val0), vget_low_s16(_w1), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val0), vget_low_s16(_w1), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val0), vget_low_s16(_w1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val0), vget_low_s16(_w1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val0), vget_high_s16(_w1), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val0), vget_high_s16(_w1), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val0), vget_high_s16(_w1), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val0), vget_high_s16(_w1), 3); - - int16x8_t _w2 = vld1q_s16(k0 + 16); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val1), vget_low_s16(_w2), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val1), vget_low_s16(_w2), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val1), vget_low_s16(_w2), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val1), vget_low_s16(_w2), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val1), vget_high_s16(_w2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val1), vget_high_s16(_w2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val1), vget_high_s16(_w2), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val1), vget_high_s16(_w2), 3); - - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val1), vget_low_s16(_w3), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val1), vget_low_s16(_w3), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val1), vget_low_s16(_w3), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val1), vget_low_s16(_w3), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val1), vget_high_s16(_w3), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val1), vget_high_s16(_w3), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val1), vget_high_s16(_w3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val1), vget_high_s16(_w3), 3); - - int16x8_t _w4 = vld1q_s16(k0 + 32); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val2), vget_low_s16(_w4), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val2), vget_low_s16(_w4), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val2), vget_low_s16(_w4), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val2), vget_low_s16(_w4), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val2), vget_high_s16(_w4), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val2), vget_high_s16(_w4), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val2), vget_high_s16(_w4), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val2), vget_high_s16(_w4), 3); - - int16x8_t _w5 = vld1q_s16(k0 + 40); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val2), vget_low_s16(_w5), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val2), vget_low_s16(_w5), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val2), vget_low_s16(_w5), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val2), vget_low_s16(_w5), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val2), vget_high_s16(_w5), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val2), vget_high_s16(_w5), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val2), vget_high_s16(_w5), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val2), vget_high_s16(_w5), 3); - - int16x8_t _w6 = vld1q_s16(k0 + 48); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val3), vget_low_s16(_w6), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val3), vget_low_s16(_w6), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val3), vget_low_s16(_w6), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val3), vget_low_s16(_w6), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val3), vget_high_s16(_w6), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val3), vget_high_s16(_w6), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val3), vget_high_s16(_w6), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val3), vget_high_s16(_w6), 3); - - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val3), vget_low_s16(_w7), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val3), vget_low_s16(_w7), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val3), vget_low_s16(_w7), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val3), vget_low_s16(_w7), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val3), vget_high_s16(_w7), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val3), vget_high_s16(_w7), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val3), vget_high_s16(_w7), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val3), vget_high_s16(_w7), 3); - - r0 += 32; - k0 += 64; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output1_tm, _sum1); - vst1q_s32(output2_tm, _sum2); - vst1q_s32(output3_tm, _sum3); - vst1q_s32(output4_tm, _sum4); - vst1q_s32(output5_tm, _sum5); - vst1q_s32(output6_tm, _sum6); - vst1q_s32(output7_tm, _sum7); - - output0_tm += 4; - output1_tm += 4; - output2_tm += 4; - output3_tm += 4; - output4_tm += 4; - output5_tm += 4; - output6_tm += 4; - output7_tm += 4; - } - for (; i < tiles; i++) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); -#else - const short* r0 = bb2.row(i / 4 + i % 4); -#endif - const short* k0 = kernel01_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - - int16x8_t _w0 = vld1q_s16(k0); - int16x8_t _w1 = vld1q_s16(k0 + 8); - int16x8_t _w2 = vld1q_s16(k0 + 16); - int16x8_t _w3 = vld1q_s16(k0 + 24); - int16x8_t _w4 = vld1q_s16(k0 + 32); - int16x8_t _w5 = vld1q_s16(k0 + 40); - int16x8_t _w6 = vld1q_s16(k0 + 48); - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); - - r0 += 8; - k0 += 64; - } - - output0_tm[0] = vgetq_lane_s32(_sum0, 0); - output1_tm[0] = vgetq_lane_s32(_sum0, 1); - output2_tm[0] = vgetq_lane_s32(_sum0, 2); - output3_tm[0] = vgetq_lane_s32(_sum0, 3); - output4_tm[0] = vgetq_lane_s32(_sum1, 0); - output5_tm[0] = vgetq_lane_s32(_sum1, 1); - output6_tm[0] = vgetq_lane_s32(_sum1, 2); - output7_tm[0] = vgetq_lane_s32(_sum1, 3); - output0_tm += 1; - output1_tm += 1; - output2_tm += 1; - output3_tm += 1; - output4_tm += 1; - output5_tm += 1; - output6_tm += 1; - output7_tm += 1; - } - } - } - - remain_outch_start += nn_outch << 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - int* output0_tm = top_blob_tm.channel(p); - - const Mat kernel0_tm = kernel_tm.channel(p / 8 + p % 8); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; -#if __aarch64__ - for (; i + 7 < tiles; i += 8) - { - const short* r0 = bb2.row(i / 8); - - const short* kptr = kernel0_tm.row(r); - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - - for (int q = 0; q < inch; q++) - { - int16x8_t _r0 = vld1q_s16(r0); - int16x8_t _r1 = vld1q_s16(r0 + 8); - int16x8_t _r2 = vld1q_s16(r0 + 16); - int16x8_t _r3 = vld1q_s16(r0 + 24); - int16x8_t _r4 = vld1q_s16(r0 + 32); - int16x8_t _r5 = vld1q_s16(r0 + 40); - int16x8_t _r6 = vld1q_s16(r0 + 48); - int16x8_t _r7 = vld1q_s16(r0 + 56); - - int16x8_t _k0 = vld1q_s16(kptr); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r1), vget_low_s16(_k0), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r1), vget_low_s16(_k0), 1); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_low_s16(_k0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_low_s16(_k0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r3), vget_low_s16(_k0), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r3), vget_low_s16(_k0), 3); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r4), vget_high_s16(_k0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r4), vget_high_s16(_k0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r5), vget_high_s16(_k0), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r5), vget_high_s16(_k0), 1); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r6), vget_high_s16(_k0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r6), vget_high_s16(_k0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r7), vget_high_s16(_k0), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r7), vget_high_s16(_k0), 3); - - kptr += 8; - r0 += 64; - } - - _sum0 = vaddq_s32(_sum0, _sum2); - _sum1 = vaddq_s32(_sum1, _sum3); - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output0_tm + 4, _sum1); - - output0_tm += 8; - } -#endif - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 8 + (i % 8) / 4); -#else - const short* r0 = bb2.row(i / 4); -#endif - const short* kptr = kernel0_tm.row(r); - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - - for (int q = 0; q < inch; q++) - { - int16x8_t _r0 = vld1q_s16(r0); - int16x8_t _r1 = vld1q_s16(r0 + 8); - int16x8_t _r2 = vld1q_s16(r0 + 16); - int16x8_t _r3 = vld1q_s16(r0 + 24); - - int16x8_t _k0 = vld1q_s16(kptr); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 1); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r1), vget_low_s16(_k0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r1), vget_low_s16(_k0), 3); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_high_s16(_k0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_high_s16(_k0), 1); - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r3), vget_high_s16(_k0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r3), vget_high_s16(_k0), 3); - - kptr += 8; - r0 += 32; - } - - int32x4_t _sum01 = vaddq_s32(_sum0, _sum1); - - vst1q_s32(output0_tm, _sum01); - - output0_tm += 4; - } - for (; i < tiles; i++) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); -#else - const short* r0 = bb2.row(i / 4 + i % 4); -#endif - const short* kptr = kernel0_tm.row(r); - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - - for (int q = 0; q < inch; q++) - { - int16x8_t _r0 = vld1q_s16(r0); - - int16x8_t _k0 = vld1q_s16(kptr); - - _sum0 = vmlal_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0)); - _sum1 = vmlal_s16(_sum1, vget_high_s16(_r0), vget_high_s16(_k0)); - - kptr += 8; - r0 += 8; - } - - int32x4_t _sum = vaddq_s32(_sum0, _sum1); -#if __aarch64__ - int sum = vaddvq_s32(_sum); // dot -#else - int32x2_t _ss = vadd_s32(vget_low_s32(_sum), vget_high_s32(_sum)); - _ss = vpadd_s32(_ss, _ss); - int sum = vget_lane_s32(_ss, 0); -#endif - - output0_tm[0] = sum; - - output0_tm++; - } - } - } - } - bottom_blob_tm = Mat(); + convolution_winograd_dot_pack8to1_int8_neon(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output @@ -1053,113 +176,7 @@ static void conv3x3s1_winograd43_pack8to1_int8_neon(const Mat& bottom_blob, Mat& top_blob_bordered.create(outw, outh, outch, 4u, 1, opt.workspace_allocator); } { - // const float otm[4][6] = { - // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, - // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} - // }; - - // 0 = r00 + (r01 + r02) + (r03 + r04) - // 1 = (r01 - r02) + (r03 - r04) * 2 - // 2 = (r01 + r02) + (r03 + r04) * 4 - // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 - - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - const int tiles = w_tm / 6 * h_tm / 6; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - const Mat out0_tm = top_blob_tm.channel(p); - Mat out0 = top_blob_bordered.channel(p); - - int tmp[4][6]; - - // tile - for (int i = 0; i < outh / 4; i++) - { - for (int j = 0; j < outw / 4; j++) - { - // top_blob_tm.create(tiles, 36, outch, 4u, 1, opt.workspace_allocator); - - const int* output0_tm_0 = (const int*)out0_tm + (i * w_tm / 6 + j) * 1; - const int* output0_tm_1 = output0_tm_0 + tiles * 1; - const int* output0_tm_2 = output0_tm_0 + tiles * 2; - const int* output0_tm_3 = output0_tm_0 + tiles * 3; - const int* output0_tm_4 = output0_tm_0 + tiles * 4; - const int* output0_tm_5 = output0_tm_0 + tiles * 5; - - int* output0 = out0.row(i * 4) + j * 4; - - // 0 = r00 + (r01 + r02) + (r03 + r04) - // 1 = (r01 - r02) + (r03 - r04) * 2 - // 2 = (r01 + r02) + (r03 + r04) * 4 - // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 - - // TODO neon optimize - for (int m = 0; m < 5; m++) - { - int tmp02a = output0_tm_1[0] + output0_tm_2[0]; - int tmp13a = output0_tm_1[0] - output0_tm_2[0]; - - int tmp02b = output0_tm_3[0] + output0_tm_4[0]; - int tmp13b = output0_tm_3[0] - output0_tm_4[0]; - - tmp[0][m] = output0_tm_0[0] + tmp02a + tmp02b; - tmp[1][m] = tmp13a + tmp13b * 2; - tmp[2][m] = tmp02a + tmp02b * 4; - tmp[3][m] = output0_tm_5[0] * 4 + tmp13a + tmp13b * 8; - - output0_tm_0 += tiles * 6; - output0_tm_1 += tiles * 6; - output0_tm_2 += tiles * 6; - output0_tm_3 += tiles * 6; - output0_tm_4 += tiles * 6; - output0_tm_5 += tiles * 6; - } - for (int m = 5; m < 6; m++) - { - int tmp02a = output0_tm_1[0] + output0_tm_2[0]; - int tmp13a = output0_tm_1[0] - output0_tm_2[0]; - - int tmp02b = output0_tm_3[0] + output0_tm_4[0]; - int tmp13b = output0_tm_3[0] - output0_tm_4[0]; - - tmp[0][m] = (output0_tm_0[0] + tmp02a + tmp02b) * 4; - tmp[1][m] = (tmp13a + tmp13b * 2) * 4; - tmp[2][m] = (tmp02a + tmp02b * 4) * 4; - tmp[3][m] = (output0_tm_5[0] * 4 + tmp13a + tmp13b * 8) * 4; - - output0_tm_0 += tiles * 6; - output0_tm_1 += tiles * 6; - output0_tm_2 += tiles * 6; - output0_tm_3 += tiles * 6; - output0_tm_4 += tiles * 6; - output0_tm_5 += tiles * 6; - } - - for (int m = 0; m < 4; m++) - { - const int* tmp0 = tmp[m]; - - int tmp02a = tmp0[1] + tmp0[2]; - int tmp13a = tmp0[1] - tmp0[2]; - - int tmp02b = tmp0[3] + tmp0[4]; - int tmp13b = tmp0[3] - tmp0[4]; - - output0[0] = (tmp0[0] + tmp02a + tmp02b) / 576; - output0[1] = (tmp13a + tmp13b * 2) / 576; - output0[2] = (tmp02a + tmp02b * 4) / 576; - output0[3] = (tmp0[5] + tmp13a + tmp13b * 8) / 576; - - output0 += outw; - } - } - } - } + conv3x3s1_winograd43_transform_output_int8_neon(top_blob_tm, top_blob_bordered, opt); } // END transform output diff --git a/src/layer/arm/convolution_3x3_pack8to4_int8.h b/src/layer/arm/convolution_3x3_pack8to4_int8.h index 29c435bf7..ee67ba61e 100644 --- a/src/layer/arm/convolution_3x3_pack8to4_int8.h +++ b/src/layer/arm/convolution_3x3_pack8to4_int8.h @@ -170,1957 +170,19 @@ static void conv3x3s1_winograd43_pack8to4_int8_neon(const Mat& bottom_blob, Mat& // BEGIN transform input Mat bottom_blob_tm; { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = w_tm / 6 * h_tm / 6; + int w_tiles = outw / 4; + int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator); - - // const float itm[4][4] = { - // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, - // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, - // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, - // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, - // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} - // }; - - // 0 = 4 * r00 - 5 * r02 + r04 - // 1 = -4 * (r01 + r02) + r04 + r03 - // 2 = 4 * (r01 - r02) + r04 - r03 - // 3 = -2 * (r01 - r03) + r04 - r02 - // 4 = 2 * (r01 - r03) + r04 - r02 - // 5 = 4 * r01 - 5 * r03 + r05 - - #pragma omp parallel for num_threads(opt.num_threads) - for (int q = 0; q < inch; q++) - { - const Mat img0 = bottom_blob_bordered.channel(q); - Mat img0_tm = bottom_blob_tm.channel(q); - - short tmp[6][6][8]; - - // tile - for (int i = 0; i < h_tm / 6; i++) - { - for (int j = 0; j < w_tm / 6; j++) - { - const signed char* r0 = img0.row(i * 4) + (j * 4) * 8; - - for (int m = 0; m < 6; m++) - { - int8x8_t _r00 = vld1_s8(r0); - int8x8_t _r01 = vld1_s8(r0 + 8); - int8x8_t _r02 = vld1_s8(r0 + 16); - int8x8_t _r03 = vld1_s8(r0 + 24); - int8x8_t _r04 = vld1_s8(r0 + 32); - int8x8_t _r05 = vld1_s8(r0 + 40); - - int8x8_t _v4s8 = vdup_n_s8(4); - int8x8_t _v5s8 = vdup_n_s8(5); - int16x8_t _v2 = vdupq_n_s16(2); - int16x8_t _v4 = vdupq_n_s16(4); - - // int16x8_t _tmp0m = vfmsq_n_f16(vfmaq_n_f16(_r04, _r00, 4.f), _r02, 5.f); - int16x8_t _tmp0m = vsubq_s16(vaddw_s8(vmull_s8(_r00, _v4s8), _r04), vmull_s8(_r02, _v5s8)); - - // int16x8_t _tmp1m = vfmsq_n_f16(vaddq_f16(_r04, _r03), vaddq_f16(_r01, _r02), 4.f); - int16x8_t _tmp1m = vmlsq_s16(vaddl_s8(_r04, _r03), vaddl_s8(_r01, _r02), _v4); - - // int16x8_t _tmp2m = vfmaq_n_f16(vsubq_f16(_r04, _r03), vsubq_f16(_r01, _r02), 4.f); - int16x8_t _tmp2m = vmlaq_s16(vsubl_s8(_r04, _r03), vsubl_s8(_r01, _r02), _v4); - - // int16x8_t _tmp3m = vfmsq_n_f16(vsubq_f16(_r04, _r02), vsubq_f16(_r01, _r03), 2.f); - int16x8_t _tmp3m = vmlsq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); - - // int16x8_t _tmp4m = vfmaq_n_f16(vsubq_f16(_r04, _r02), vsubq_f16(_r01, _r03), 2.f); - int16x8_t _tmp4m = vmlaq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); - - // int16x8_t _tmp5m = vfmsq_n_f16(vfmaq_n_f16(_r05, _r01, 4.f), _r03, 5.f); - int16x8_t _tmp5m = vsubq_s16(vaddw_s8(vmull_s8(_r01, _v4s8), _r05), vmull_s8(_r03, _v5s8)); - - vst1q_s16(tmp[0][m], _tmp0m); - vst1q_s16(tmp[1][m], _tmp1m); - vst1q_s16(tmp[2][m], _tmp2m); - vst1q_s16(tmp[3][m], _tmp3m); - vst1q_s16(tmp[4][m], _tmp4m); - vst1q_s16(tmp[5][m], _tmp5m); - - r0 += w * 8; - } - - short* r0_tm_0 = (short*)img0_tm + (i * w_tm / 6 + j) * 8; - short* r0_tm_1 = r0_tm_0 + tiles * 8; - short* r0_tm_2 = r0_tm_0 + tiles * 16; - short* r0_tm_3 = r0_tm_0 + tiles * 24; - short* r0_tm_4 = r0_tm_0 + tiles * 32; - short* r0_tm_5 = r0_tm_0 + tiles * 40; - - for (int m = 0; m < 6; m++) - { - int16x8_t _tmp00 = vld1q_s16(tmp[m][0]); - int16x8_t _tmp01 = vld1q_s16(tmp[m][1]); - int16x8_t _tmp02 = vld1q_s16(tmp[m][2]); - int16x8_t _tmp03 = vld1q_s16(tmp[m][3]); - int16x8_t _tmp04 = vld1q_s16(tmp[m][4]); - int16x8_t _tmp05 = vld1q_s16(tmp[m][5]); - - int16x8_t _v2 = vdupq_n_s16(2); - int16x8_t _v4 = vdupq_n_s16(4); - int16x8_t _v5 = vdupq_n_s16(5); - - int16x8_t _r0tm0 = vmlsq_s16(vmlaq_s16(_tmp04, _tmp00, _v4), _tmp02, _v5); - int16x8_t _r0tm1 = vmlsq_s16(vaddq_s16(_tmp04, _tmp03), vaddq_s16(_tmp01, _tmp02), _v4); - int16x8_t _r0tm2 = vmlaq_s16(vsubq_s16(_tmp04, _tmp03), vsubq_s16(_tmp01, _tmp02), _v4); - int16x8_t _r0tm3 = vmlsq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); - int16x8_t _r0tm4 = vmlaq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); - int16x8_t _r0tm5 = vmlsq_s16(vmlaq_s16(_tmp05, _tmp01, _v4), _tmp03, _v5); - - vst1q_s16(r0_tm_0, _r0tm0); - vst1q_s16(r0_tm_1, _r0tm1); - vst1q_s16(r0_tm_2, _r0tm2); - vst1q_s16(r0_tm_3, _r0tm3); - vst1q_s16(r0_tm_4, _r0tm4); - vst1q_s16(r0_tm_5, _r0tm5); - - r0_tm_0 += tiles * 48; - r0_tm_1 += tiles * 48; - r0_tm_2 += tiles * 48; - r0_tm_3 += tiles * 48; - r0_tm_4 += tiles * 48; - r0_tm_5 += tiles * 48; - } - } - } - } + conv3x3s1_winograd43_transform_input_pack8_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt); } bottom_blob_bordered = Mat(); // END transform input // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = h_tm / 6 * w_tm / 6; - - // permute - // bottom_blob_tm.create(tiles, 36, inch, elemsize, elempack, opt.workspace_allocator); - Mat bottom_blob_tm2; -#if __aarch64__ - if (tiles >= 12) - bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 8) - bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 2) - bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 36, 2u * elempack, elempack, opt.workspace_allocator); -#else - if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 2) - bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 36, 2u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 36, 2u * elempack, elempack, opt.workspace_allocator); -#endif - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 36; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; -#if __aarch64__ - for (; i + 11 < tiles; i += 12) - { - short* tm2p = tm2.row(i / 12); - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { - // transpose 12x8 - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n" - "ld4 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n" - - "sub %0, %0, #128 \n" - - "uzp1 v20.8h, v0.8h, v4.8h \n" // 0 - "uzp1 v21.8h, v16.8h, v1.8h \n" // 1 - "uzp1 v22.8h, v5.8h, v17.8h \n" // 2 - "uzp1 v23.8h, v2.8h, v6.8h \n" // 3 - "uzp1 v24.8h, v18.8h, v3.8h \n" // 4 - "uzp1 v25.8h, v7.8h, v19.8h \n" // 5 - "uzp2 v26.8h, v0.8h, v4.8h \n" // 6 - "uzp2 v27.8h, v16.8h, v1.8h \n" // 7 - "uzp2 v28.8h, v5.8h, v17.8h \n" // 8 - "uzp2 v29.8h, v2.8h, v6.8h \n" // 9 - "uzp2 v30.8h, v18.8h, v3.8h \n" // 10 - "uzp2 v31.8h, v7.8h, v19.8h \n" // 11 - - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" - "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" - : "=r"(r0), // %0 - "=r"(tm2p) // %1 - : "0"(r0), - "1"(tm2p) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - - r0 += bottom_blob_tm.cstep * 8; - } - } - for (; i + 7 < tiles; i += 8) - { - short* tmpptr = tm2.row(i / 12 + (i % 12) / 8); - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { - // transpose 8x8 - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" - "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" - "sub %0, %0, #64 \n" - - "uzp1 v16.8h, v0.8h, v4.8h \n" - "uzp2 v20.8h, v0.8h, v4.8h \n" - "uzp1 v17.8h, v1.8h, v5.8h \n" - "uzp2 v21.8h, v1.8h, v5.8h \n" - "uzp1 v18.8h, v2.8h, v6.8h \n" - "uzp2 v22.8h, v2.8h, v6.8h \n" - "uzp1 v19.8h, v3.8h, v7.8h \n" - "uzp2 v23.8h, v3.8h, v7.8h \n" - - "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" - "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); - - r0 += bottom_blob_tm.cstep * 8; - } - } -#endif // __aarch64__ - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); -#else - short* tmpptr = tm2.row(i / 4); -#endif - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #512] \n" - "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" - "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "v0", "v1", "v2", "v3"); -#else - asm volatile( - "pld [%0, #512] \n" - "vldm %0, {d0-d7} \n" - "vstm %1!, {d0-d7} \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "q0", "q1", "q2", "q3"); -#endif - r0 += bottom_blob_tm.cstep * 8; - } - } - for (; i + 1 < tiles; i += 2) - { -#if __aarch64__ - short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); -#else - short* tmpptr = tm2.row(i / 4 + (i % 4) / 2); -#endif - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #256] \n" - "ld1 {v0.8h, v1.8h}, [%0] \n" - "st1 {v0.8h, v1.8h}, [%1], #32 \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "v0", "v1"); -#else - asm volatile( - "pld [%0, #256] \n" - "vld1.s16 {d0-d3}, [%0 :128] \n" - "vst1.s16 {d0-d3}, [%1 :128]! \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "q0", "q1"); -#endif - r0 += bottom_blob_tm.cstep * 8; - } - } - for (; i < tiles; i++) - { -#if __aarch64__ - short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); -#else - short* tmpptr = tm2.row(i / 4 + (i % 4) / 2 + i % 2); -#endif - - const short* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 8; - - for (int q = 0; q < inch; q++) - { -#if __aarch64__ - asm volatile( - "prfm pldl1keep, [%0, #128] \n" - "ld1 {v0.8h}, [%0] \n" - "st1 {v0.8h}, [%1], #16 \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "v0"); -#else - asm volatile( - "pld [%0, #128] \n" - "vld1.s16 {d0-d1}, [%0 :128] \n" - "vst1.s16 {d0-d1}, [%1 :128]! \n" - : "=r"(r0), // %0 - "=r"(tmpptr) // %1 - : "0"(r0), - "1"(tmpptr) - : "memory", "q0"); -#endif - r0 += bottom_blob_tm.cstep * 8; - } - } - } - - bottom_blob_tm = Mat(); - // permute end - - top_blob_tm.create(tiles, 36, outch, 4u * 4, 4, opt.workspace_allocator); - - int nn_outch = 0; - int remain_outch_start = 0; - - nn_outch = outch >> 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 2; - - int* output0_tm = top_blob_tm.channel(p); - int* output1_tm = top_blob_tm.channel(p + 1); - - const Mat kernel0_tm = kernel_tm.channel(p / 2); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; -#if __aarch64__ - for (; i + 11 < tiles; i += 12) - { - const short* r0 = bb2.row(i / 12); - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r01 - - "eor v8.16b, v8.16b, v8.16b \n" - "eor v9.16b, v9.16b, v9.16b \n" - - "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w01 - - "eor v10.16b, v10.16b, v10.16b \n" - "eor v11.16b, v11.16b, v11.16b \n" - - "prfm pldl1keep, [%3, #256] \n" - - "eor v12.16b, v12.16b, v12.16b \n" - "eor v13.16b, v13.16b, v13.16b \n" - - "prfm pldl1keep, [%4, #256] \n" - - "eor v14.16b, v14.16b, v14.16b \n" - "eor v15.16b, v15.16b, v15.16b \n" - "eor v16.16b, v16.16b, v16.16b \n" - "eor v17.16b, v17.16b, v17.16b \n" - "eor v18.16b, v18.16b, v18.16b \n" - "eor v19.16b, v19.16b, v19.16b \n" - "eor v20.16b, v20.16b, v20.16b \n" - "eor v21.16b, v21.16b, v21.16b \n" - "eor v22.16b, v22.16b, v22.16b \n" - "eor v23.16b, v23.16b, v23.16b \n" - "eor v24.16b, v24.16b, v24.16b \n" - "eor v25.16b, v25.16b, v25.16b \n" - "eor v26.16b, v26.16b, v26.16b \n" - "eor v27.16b, v27.16b, v27.16b \n" - "eor v28.16b, v28.16b, v28.16b \n" - "eor v29.16b, v29.16b, v29.16b \n" - "eor v30.16b, v30.16b, v30.16b \n" - "eor v31.16b, v31.16b, v31.16b \n" - - "0: \n" - - "smlal v8.4s, v4.4h, v0.h[0] \n" - "smlal2 v20.4s, v4.8h, v0.h[0] \n" - "smlal v9.4s, v4.4h, v0.h[1] \n" - "smlal2 v21.4s, v4.8h, v0.h[1] \n" - "smlal v10.4s, v4.4h, v0.h[2] \n" - "smlal2 v22.4s, v4.8h, v0.h[2] \n" - "smlal v11.4s, v4.4h, v0.h[3] \n" - "smlal2 v23.4s, v4.8h, v0.h[3] \n" - "smlal v12.4s, v4.4h, v0.h[4] \n" - "smlal2 v24.4s, v4.8h, v0.h[4] \n" - "smlal v13.4s, v4.4h, v0.h[5] \n" - "smlal2 v25.4s, v4.8h, v0.h[5] \n" - "smlal v14.4s, v4.4h, v0.h[6] \n" - "smlal2 v26.4s, v4.8h, v0.h[6] \n" - "smlal v15.4s, v4.4h, v0.h[7] \n" - "smlal2 v27.4s, v4.8h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r23 - - "smlal v16.4s, v4.4h, v1.h[0] \n" - "smlal2 v28.4s, v4.8h, v1.h[0] \n" - "smlal v17.4s, v4.4h, v1.h[1] \n" - "smlal2 v29.4s, v4.8h, v1.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v18.4s, v4.4h, v1.h[2] \n" - "smlal2 v30.4s, v4.8h, v1.h[2] \n" - "smlal v19.4s, v4.4h, v1.h[3] \n" - "smlal2 v31.4s, v4.8h, v1.h[3] \n" - - "ld1 {v6.8h, v7.8h}, [%4], #32 \n" // w23 - - "smlal v8.4s, v5.4h, v1.h[4] \n" - "smlal2 v20.4s, v5.8h, v1.h[4] \n" - "smlal v9.4s, v5.4h, v1.h[5] \n" - "smlal2 v21.4s, v5.8h, v1.h[5] \n" - - "prfm pldl1keep, [%4, #256] \n" - - "smlal v10.4s, v5.4h, v1.h[6] \n" - "smlal2 v22.4s, v5.8h, v1.h[6] \n" - "smlal v11.4s, v5.4h, v1.h[7] \n" - "smlal2 v23.4s, v5.8h, v1.h[7] \n" - "smlal v12.4s, v5.4h, v2.h[0] \n" - "smlal2 v24.4s, v5.8h, v2.h[0] \n" - "smlal v13.4s, v5.4h, v2.h[1] \n" - "smlal2 v25.4s, v5.8h, v2.h[1] \n" - "smlal v14.4s, v5.4h, v2.h[2] \n" - "smlal2 v26.4s, v5.8h, v2.h[2] \n" - "smlal v15.4s, v5.4h, v2.h[3] \n" - "smlal2 v27.4s, v5.8h, v2.h[3] \n" - "smlal v16.4s, v5.4h, v2.h[4] \n" - "smlal2 v28.4s, v5.8h, v2.h[4] \n" - "smlal v17.4s, v5.4h, v2.h[5] \n" - "smlal2 v29.4s, v5.8h, v2.h[5] \n" - "smlal v18.4s, v5.4h, v2.h[6] \n" - "smlal2 v30.4s, v5.8h, v2.h[6] \n" - "smlal v19.4s, v5.4h, v2.h[7] \n" - "smlal2 v31.4s, v5.8h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r45 - - "smlal v8.4s, v6.4h, v3.h[0] \n" - "smlal2 v20.4s, v6.8h, v3.h[0] \n" - "smlal v9.4s, v6.4h, v3.h[1] \n" - "smlal2 v21.4s, v6.8h, v3.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v10.4s, v6.4h, v3.h[2] \n" - "smlal2 v22.4s, v6.8h, v3.h[2] \n" - "smlal v11.4s, v6.4h, v3.h[3] \n" - "smlal2 v23.4s, v6.8h, v3.h[3] \n" - "smlal v12.4s, v6.4h, v3.h[4] \n" - "smlal2 v24.4s, v6.8h, v3.h[4] \n" - "smlal v13.4s, v6.4h, v3.h[5] \n" - "smlal2 v25.4s, v6.8h, v3.h[5] \n" - "smlal v14.4s, v6.4h, v3.h[6] \n" - "smlal2 v26.4s, v6.8h, v3.h[6] \n" - "smlal v15.4s, v6.4h, v3.h[7] \n" - "smlal2 v27.4s, v6.8h, v3.h[7] \n" - - "smlal v16.4s, v6.4h, v0.h[0] \n" - "smlal2 v28.4s, v6.8h, v0.h[0] \n" - "smlal v17.4s, v6.4h, v0.h[1] \n" - "smlal2 v29.4s, v6.8h, v0.h[1] \n" - "smlal v18.4s, v6.4h, v0.h[2] \n" - "smlal2 v30.4s, v6.8h, v0.h[2] \n" - "smlal v19.4s, v6.4h, v0.h[3] \n" - "smlal2 v31.4s, v6.8h, v0.h[3] \n" - - "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w45 - - "smlal v8.4s, v7.4h, v0.h[4] \n" - "smlal2 v20.4s, v7.8h, v0.h[4] \n" - "smlal v9.4s, v7.4h, v0.h[5] \n" - "smlal2 v21.4s, v7.8h, v0.h[5] \n" - - "prfm pldl1keep, [%4, #256] \n" - - "smlal v10.4s, v7.4h, v0.h[6] \n" - "smlal2 v22.4s, v7.8h, v0.h[6] \n" - "smlal v11.4s, v7.4h, v0.h[7] \n" - "smlal2 v23.4s, v7.8h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r67 - - "smlal v12.4s, v7.4h, v1.h[0] \n" - "smlal2 v24.4s, v7.8h, v1.h[0] \n" - "smlal v13.4s, v7.4h, v1.h[1] \n" - "smlal2 v25.4s, v7.8h, v1.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v14.4s, v7.4h, v1.h[2] \n" - "smlal2 v26.4s, v7.8h, v1.h[2] \n" - "smlal v15.4s, v7.4h, v1.h[3] \n" - "smlal2 v27.4s, v7.8h, v1.h[3] \n" - "smlal v16.4s, v7.4h, v1.h[4] \n" - "smlal2 v28.4s, v7.8h, v1.h[4] \n" - "smlal v17.4s, v7.4h, v1.h[5] \n" - "smlal2 v29.4s, v7.8h, v1.h[5] \n" - "smlal v18.4s, v7.4h, v1.h[6] \n" - "smlal2 v30.4s, v7.8h, v1.h[6] \n" - "smlal v19.4s, v7.4h, v1.h[7] \n" - "smlal2 v31.4s, v7.8h, v1.h[7] \n" - - "smlal v8.4s, v4.4h, v2.h[0] \n" - "smlal2 v20.4s, v4.8h, v2.h[0] \n" - "smlal v9.4s, v4.4h, v2.h[1] \n" - "smlal2 v21.4s, v4.8h, v2.h[1] \n" - "smlal v10.4s, v4.4h, v2.h[2] \n" - "smlal2 v22.4s, v4.8h, v2.h[2] \n" - "smlal v11.4s, v4.4h, v2.h[3] \n" - "smlal2 v23.4s, v4.8h, v2.h[3] \n" - "smlal v12.4s, v4.4h, v2.h[4] \n" - "smlal2 v24.4s, v4.8h, v2.h[4] \n" - "smlal v13.4s, v4.4h, v2.h[5] \n" - "smlal2 v25.4s, v4.8h, v2.h[5] \n" - "smlal v14.4s, v4.4h, v2.h[6] \n" - "smlal2 v26.4s, v4.8h, v2.h[6] \n" - "smlal v15.4s, v4.4h, v2.h[7] \n" - "smlal2 v27.4s, v4.8h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r89 - - "smlal v16.4s, v4.4h, v3.h[0] \n" - "smlal2 v28.4s, v4.8h, v3.h[0] \n" - "smlal v17.4s, v4.4h, v3.h[1] \n" - "smlal2 v29.4s, v4.8h, v3.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v18.4s, v4.4h, v3.h[2] \n" - "smlal2 v30.4s, v4.8h, v3.h[2] \n" - "smlal v19.4s, v4.4h, v3.h[3] \n" - "smlal2 v31.4s, v4.8h, v3.h[3] \n" - - "ld1 {v6.8h, v7.8h}, [%4], #32 \n" // w67 - - "smlal v8.4s, v5.4h, v3.h[4] \n" - "smlal2 v20.4s, v5.8h, v3.h[4] \n" - "smlal v9.4s, v5.4h, v3.h[5] \n" - "smlal2 v21.4s, v5.8h, v3.h[5] \n" - - "prfm pldl1keep, [%4, #256] \n" - - "smlal v10.4s, v5.4h, v3.h[6] \n" - "smlal2 v22.4s, v5.8h, v3.h[6] \n" - "smlal v11.4s, v5.4h, v3.h[7] \n" - "smlal2 v23.4s, v5.8h, v3.h[7] \n" - - "smlal v12.4s, v5.4h, v0.h[0] \n" - "smlal2 v24.4s, v5.8h, v0.h[0] \n" - "smlal v13.4s, v5.4h, v0.h[1] \n" - "smlal2 v25.4s, v5.8h, v0.h[1] \n" - "smlal v14.4s, v5.4h, v0.h[2] \n" - "smlal2 v26.4s, v5.8h, v0.h[2] \n" - "smlal v15.4s, v5.4h, v0.h[3] \n" - "smlal2 v27.4s, v5.8h, v0.h[3] \n" - "smlal v16.4s, v5.4h, v0.h[4] \n" - "smlal2 v28.4s, v5.8h, v0.h[4] \n" - "smlal v17.4s, v5.4h, v0.h[5] \n" - "smlal2 v29.4s, v5.8h, v0.h[5] \n" - "smlal v18.4s, v5.4h, v0.h[6] \n" - "smlal2 v30.4s, v5.8h, v0.h[6] \n" - "smlal v19.4s, v5.4h, v0.h[7] \n" - "smlal2 v31.4s, v5.8h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r1011 - - "smlal v8.4s, v6.4h, v1.h[0] \n" - "smlal2 v20.4s, v6.8h, v1.h[0] \n" - "smlal v9.4s, v6.4h, v1.h[1] \n" - "smlal2 v21.4s, v6.8h, v1.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v10.4s, v6.4h, v1.h[2] \n" - "smlal2 v22.4s, v6.8h, v1.h[2] \n" - "smlal v11.4s, v6.4h, v1.h[3] \n" - "smlal2 v23.4s, v6.8h, v1.h[3] \n" - "smlal v12.4s, v6.4h, v1.h[4] \n" - "smlal2 v24.4s, v6.8h, v1.h[4] \n" - "smlal v13.4s, v6.4h, v1.h[5] \n" - "smlal2 v25.4s, v6.8h, v1.h[5] \n" - "smlal v14.4s, v6.4h, v1.h[6] \n" - "smlal2 v26.4s, v6.8h, v1.h[6] \n" - "smlal v15.4s, v6.4h, v1.h[7] \n" - "smlal2 v27.4s, v6.8h, v1.h[7] \n" - "smlal v16.4s, v6.4h, v2.h[0] \n" - "smlal2 v28.4s, v6.8h, v2.h[0] \n" - "smlal v17.4s, v6.4h, v2.h[1] \n" - "smlal2 v29.4s, v6.8h, v2.h[1] \n" - "smlal v18.4s, v6.4h, v2.h[2] \n" - "smlal2 v30.4s, v6.8h, v2.h[2] \n" - "smlal v19.4s, v6.4h, v2.h[3] \n" - "smlal2 v31.4s, v6.8h, v2.h[3] \n" - - "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w01 - - "smlal v8.4s, v7.4h, v2.h[4] \n" - "smlal2 v20.4s, v7.8h, v2.h[4] \n" - "smlal v9.4s, v7.4h, v2.h[5] \n" - "smlal2 v21.4s, v7.8h, v2.h[5] \n" - - "prfm pldl1keep, [%4, #256] \n" - - "smlal v10.4s, v7.4h, v2.h[6] \n" - "smlal2 v22.4s, v7.8h, v2.h[6] \n" - "smlal v11.4s, v7.4h, v2.h[7] \n" - "smlal2 v23.4s, v7.8h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r01 - - "smlal v12.4s, v7.4h, v3.h[0] \n" - "smlal2 v24.4s, v7.8h, v3.h[0] \n" - "smlal v13.4s, v7.4h, v3.h[1] \n" - "smlal2 v25.4s, v7.8h, v3.h[1] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal v14.4s, v7.4h, v3.h[2] \n" - "smlal2 v26.4s, v7.8h, v3.h[2] \n" - "smlal v15.4s, v7.4h, v3.h[3] \n" - "smlal2 v27.4s, v7.8h, v3.h[3] \n" - "smlal v16.4s, v7.4h, v3.h[4] \n" - "smlal2 v28.4s, v7.8h, v3.h[4] \n" - "smlal v17.4s, v7.4h, v3.h[5] \n" - "smlal2 v29.4s, v7.8h, v3.h[5] \n" - - "subs %w0, %w0, #1 \n" - - "smlal v18.4s, v7.4h, v3.h[6] \n" - "smlal2 v30.4s, v7.8h, v3.h[6] \n" - "smlal v19.4s, v7.4h, v3.h[7] \n" - "smlal2 v31.4s, v7.8h, v3.h[7] \n" - - "bne 0b \n" - - "sub %3, %3, #32 \n" - "sub %4, %4, #32 \n" - - "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" - "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" - "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" - - : "=r"(nn), // %0 - "=r"(output0_tm), // %1 - "=r"(output1_tm), // %2 - "=r"(r0), // %3 - "=r"(k0) // %4 - : "0"(nn), - "1"(output0_tm), - "2"(output1_tm), - "3"(r0), - "4"(k0) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); - } - for (; i + 7 < tiles; i += 8) - { - const short* r0 = bb2.row(i / 12 + (i % 12) / 8); - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - int32x4_t _sum4 = vdupq_n_s32(0); - int32x4_t _sum5 = vdupq_n_s32(0); - int32x4_t _sum6 = vdupq_n_s32(0); - int32x4_t _sum7 = vdupq_n_s32(0); - int32x4_t _sum8 = vdupq_n_s32(0); - int32x4_t _sum9 = vdupq_n_s32(0); - int32x4_t _suma = vdupq_n_s32(0); - int32x4_t _sumb = vdupq_n_s32(0); - int32x4_t _sumc = vdupq_n_s32(0); - int32x4_t _sumd = vdupq_n_s32(0); - int32x4_t _sume = vdupq_n_s32(0); - int32x4_t _sumf = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - int16x8_t _val2 = vld1q_s16(r0 + 16); - int16x8_t _val3 = vld1q_s16(r0 + 24); - int16x8_t _val4 = vld1q_s16(r0 + 32); - int16x8_t _val5 = vld1q_s16(r0 + 40); - int16x8_t _val6 = vld1q_s16(r0 + 48); - int16x8_t _val7 = vld1q_s16(r0 + 56); - - int16x8_t _w0 = vld1q_s16(k0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val0), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val0), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val0), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val0), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val0), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val0), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w0), vget_high_s16(_val0), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w0), vget_high_s16(_val0), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w0), vget_high_s16(_val0), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w0), vget_high_s16(_val0), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w0), vget_high_s16(_val0), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w0), vget_high_s16(_val0), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w0), vget_high_s16(_val0), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w0), vget_high_s16(_val0), 3); - - int16x8_t _w1 = vld1q_s16(k0 + 8); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val1), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val1), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val1), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val1), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val1), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val1), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w1), vget_high_s16(_val1), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w1), vget_high_s16(_val1), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w1), vget_high_s16(_val1), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w1), vget_high_s16(_val1), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w1), vget_high_s16(_val1), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w1), vget_high_s16(_val1), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w1), vget_high_s16(_val1), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w1), vget_high_s16(_val1), 3); - - int16x8_t _w2 = vld1q_s16(k0 + 16); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val2), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val2), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val2), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val2), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_low_s16(_val2), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_low_s16(_val2), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_low_s16(_val2), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_low_s16(_val2), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w2), vget_high_s16(_val2), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w2), vget_high_s16(_val2), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w2), vget_high_s16(_val2), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w2), vget_high_s16(_val2), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w2), vget_high_s16(_val2), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w2), vget_high_s16(_val2), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w2), vget_high_s16(_val2), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w2), vget_high_s16(_val2), 3); - - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val3), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val3), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val3), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val3), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_low_s16(_val3), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_low_s16(_val3), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_low_s16(_val3), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_low_s16(_val3), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w3), vget_high_s16(_val3), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w3), vget_high_s16(_val3), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w3), vget_high_s16(_val3), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w3), vget_high_s16(_val3), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w3), vget_high_s16(_val3), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w3), vget_high_s16(_val3), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w3), vget_high_s16(_val3), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w3), vget_high_s16(_val3), 3); - - int16x8_t _w4 = vld1q_s16(k0 + 32); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_low_s16(_val4), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_low_s16(_val4), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_low_s16(_val4), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_low_s16(_val4), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w4), vget_low_s16(_val4), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w4), vget_low_s16(_val4), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w4), vget_low_s16(_val4), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w4), vget_low_s16(_val4), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w4), vget_high_s16(_val4), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w4), vget_high_s16(_val4), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w4), vget_high_s16(_val4), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w4), vget_high_s16(_val4), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w4), vget_high_s16(_val4), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w4), vget_high_s16(_val4), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w4), vget_high_s16(_val4), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w4), vget_high_s16(_val4), 3); - - int16x8_t _w5 = vld1q_s16(k0 + 40); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_low_s16(_val5), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_low_s16(_val5), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_low_s16(_val5), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_low_s16(_val5), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w5), vget_low_s16(_val5), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w5), vget_low_s16(_val5), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w5), vget_low_s16(_val5), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w5), vget_low_s16(_val5), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w5), vget_high_s16(_val5), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w5), vget_high_s16(_val5), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w5), vget_high_s16(_val5), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w5), vget_high_s16(_val5), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w5), vget_high_s16(_val5), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w5), vget_high_s16(_val5), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w5), vget_high_s16(_val5), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w5), vget_high_s16(_val5), 3); - - int16x8_t _w6 = vld1q_s16(k0 + 48); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_low_s16(_val6), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_low_s16(_val6), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_low_s16(_val6), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_low_s16(_val6), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w6), vget_low_s16(_val6), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w6), vget_low_s16(_val6), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w6), vget_low_s16(_val6), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w6), vget_low_s16(_val6), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w6), vget_high_s16(_val6), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w6), vget_high_s16(_val6), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w6), vget_high_s16(_val6), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w6), vget_high_s16(_val6), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w6), vget_high_s16(_val6), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w6), vget_high_s16(_val6), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w6), vget_high_s16(_val6), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w6), vget_high_s16(_val6), 3); - - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_low_s16(_val7), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_low_s16(_val7), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_low_s16(_val7), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_low_s16(_val7), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w7), vget_low_s16(_val7), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w7), vget_low_s16(_val7), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w7), vget_low_s16(_val7), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w7), vget_low_s16(_val7), 3); - _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w7), vget_high_s16(_val7), 0); - _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w7), vget_high_s16(_val7), 0); - _suma = vmlal_lane_s16(_suma, vget_low_s16(_w7), vget_high_s16(_val7), 1); - _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w7), vget_high_s16(_val7), 1); - _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w7), vget_high_s16(_val7), 2); - _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w7), vget_high_s16(_val7), 2); - _sume = vmlal_lane_s16(_sume, vget_low_s16(_w7), vget_high_s16(_val7), 3); - _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w7), vget_high_s16(_val7), 3); - - r0 += 64; - k0 += 64; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output1_tm, _sum1); - vst1q_s32(output0_tm + 4, _sum2); - vst1q_s32(output1_tm + 4, _sum3); - vst1q_s32(output0_tm + 8, _sum4); - vst1q_s32(output1_tm + 8, _sum5); - vst1q_s32(output0_tm + 12, _sum6); - vst1q_s32(output1_tm + 12, _sum7); - vst1q_s32(output0_tm + 16, _sum8); - vst1q_s32(output1_tm + 16, _sum9); - vst1q_s32(output0_tm + 20, _suma); - vst1q_s32(output1_tm + 20, _sumb); - vst1q_s32(output0_tm + 24, _sumc); - vst1q_s32(output1_tm + 24, _sumd); - vst1q_s32(output0_tm + 28, _sume); - vst1q_s32(output1_tm + 28, _sumf); - output0_tm += 32; - output1_tm += 32; - } -#endif // __aarch64__ - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); -#else - const short* r0 = bb2.row(i / 4); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - -#if __aarch64__ - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - int32x4_t _sum4 = vdupq_n_s32(0); - int32x4_t _sum5 = vdupq_n_s32(0); - int32x4_t _sum6 = vdupq_n_s32(0); - int32x4_t _sum7 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - int16x8_t _val2 = vld1q_s16(r0 + 16); - int16x8_t _val3 = vld1q_s16(r0 + 24); - - int16x8_t _w0 = vld1q_s16(k0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 0); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val2), 0); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val3), 0); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val3), 0); - - int16x8_t _w1 = vld1q_s16(k0 + 8); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val2), 1); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val3), 1); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val3), 1); - - int16x8_t _w2 = vld1q_s16(k0 + 16); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val1), 2); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_low_s16(_val2), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_low_s16(_val2), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_low_s16(_val3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_low_s16(_val3), 2); - - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val1), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_low_s16(_val2), 3); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_low_s16(_val2), 3); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_low_s16(_val3), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_low_s16(_val3), 3); - - int16x8_t _w4 = vld1q_s16(k0 + 32); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_high_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_high_s16(_val1), 0); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w4), vget_high_s16(_val2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w4), vget_high_s16(_val2), 0); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w4), vget_high_s16(_val3), 0); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w4), vget_high_s16(_val3), 0); - - int16x8_t _w5 = vld1q_s16(k0 + 40); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_high_s16(_val1), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_high_s16(_val1), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w5), vget_high_s16(_val2), 1); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w5), vget_high_s16(_val2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w5), vget_high_s16(_val3), 1); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w5), vget_high_s16(_val3), 1); - - int16x8_t _w6 = vld1q_s16(k0 + 48); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_high_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_high_s16(_val1), 2); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w6), vget_high_s16(_val2), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w6), vget_high_s16(_val2), 2); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w6), vget_high_s16(_val3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w6), vget_high_s16(_val3), 2); - - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_high_s16(_val1), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_high_s16(_val1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w7), vget_high_s16(_val2), 3); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w7), vget_high_s16(_val2), 3); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w7), vget_high_s16(_val3), 3); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w7), vget_high_s16(_val3), 3); - - r0 += 32; - k0 += 64; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output1_tm, _sum1); - vst1q_s32(output0_tm + 4, _sum2); - vst1q_s32(output1_tm + 4, _sum3); - vst1q_s32(output0_tm + 8, _sum4); - vst1q_s32(output1_tm + 8, _sum5); - vst1q_s32(output0_tm + 12, _sum6); - vst1q_s32(output1_tm + 12, _sum7); - output0_tm += 16; - output1_tm += 16; -#else - asm volatile( - "veor q8, q8 \n" - "veor q9, q9 \n" - "veor q10, q10 \n" - "veor q11, q11 \n" - "veor q12, q12 \n" - "veor q13, q13 \n" - "veor q14, q14 \n" - "veor q15, q15 \n" - - "0: \n" - - "pld [%3, #256] \n" - "pld [%3, #512] \n" - "vldm %3!, {d0-d7} \n" - - "pld [%4, #256] \n" - "vld1.s16 {d8-d11}, [%4 :128]! \n" - - "vmlal.s16 q8, d8, d0[0] \n" - "vmlal.s16 q12, d9, d0[0] \n" - "vmlal.s16 q9, d8, d2[0] \n" - "vmlal.s16 q13, d9, d2[0] \n" - "vmlal.s16 q10, d8, d4[0] \n" - "vmlal.s16 q14, d9, d4[0] \n" - "vmlal.s16 q11, d8, d6[0] \n" - "vmlal.s16 q15, d9, d6[0] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d8-d9}, [%4 :128]! \n" - - "vmlal.s16 q8, d10, d0[1] \n" - "vmlal.s16 q12, d11, d0[1] \n" - "vmlal.s16 q9, d10, d2[1] \n" - "vmlal.s16 q13, d11, d2[1] \n" - "vmlal.s16 q10, d10, d4[1] \n" - "vmlal.s16 q14, d11, d4[1] \n" - "vmlal.s16 q11, d10, d6[1] \n" - "vmlal.s16 q15, d11, d6[1] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d10-d11}, [%4 :128]! \n" - - "vmlal.s16 q8, d8, d0[2] \n" - "vmlal.s16 q12, d9, d0[2] \n" - "vmlal.s16 q9, d8, d2[2] \n" - "vmlal.s16 q13, d9, d2[2] \n" - "vmlal.s16 q10, d8, d4[2] \n" - "vmlal.s16 q14, d9, d4[2] \n" - "vmlal.s16 q11, d8, d6[2] \n" - "vmlal.s16 q15, d9, d6[2] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d8-d9}, [%4 :128]! \n" - - "vmlal.s16 q8, d10, d0[3] \n" - "vmlal.s16 q12, d11, d0[3] \n" - "vmlal.s16 q9, d10, d2[3] \n" - "vmlal.s16 q13, d11, d2[3] \n" - "vmlal.s16 q10, d10, d4[3] \n" - "vmlal.s16 q14, d11, d4[3] \n" - "vmlal.s16 q11, d10, d6[3] \n" - "vmlal.s16 q15, d11, d6[3] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d10-d11}, [%4 :128]! \n" - - "vmlal.s16 q8, d8, d1[0] \n" - "vmlal.s16 q12, d9, d1[0] \n" - "vmlal.s16 q9, d8, d3[0] \n" - "vmlal.s16 q13, d9, d3[0] \n" - "vmlal.s16 q10, d8, d5[0] \n" - "vmlal.s16 q14, d9, d5[0] \n" - "vmlal.s16 q11, d8, d7[0] \n" - "vmlal.s16 q15, d9, d7[0] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d8-d9}, [%4 :128]! \n" - - "vmlal.s16 q8, d10, d1[1] \n" - "vmlal.s16 q12, d11, d1[1] \n" - "vmlal.s16 q9, d10, d3[1] \n" - "vmlal.s16 q13, d11, d3[1] \n" - "vmlal.s16 q10, d10, d5[1] \n" - "vmlal.s16 q14, d11, d5[1] \n" - "vmlal.s16 q11, d10, d7[1] \n" - "vmlal.s16 q15, d11, d7[1] \n" - - "pld [%4, #128] \n" - "vld1.s16 {d10-d11}, [%4 :128]! \n" - - "vmlal.s16 q8, d8, d1[2] \n" - "vmlal.s16 q12, d9, d1[2] \n" - "vmlal.s16 q9, d8, d3[2] \n" - "vmlal.s16 q13, d9, d3[2] \n" - "vmlal.s16 q10, d8, d5[2] \n" - "vmlal.s16 q14, d9, d5[2] \n" - "vmlal.s16 q11, d8, d7[2] \n" - "vmlal.s16 q15, d9, d7[2] \n" - - "subs %0, %0, #1 \n" - - "vmlal.s16 q8, d10, d1[3] \n" - "vmlal.s16 q12, d11, d1[3] \n" - "vmlal.s16 q9, d10, d3[3] \n" - "vmlal.s16 q13, d11, d3[3] \n" - "vmlal.s16 q10, d10, d5[3] \n" - "vmlal.s16 q14, d11, d5[3] \n" - "vmlal.s16 q11, d10, d7[3] \n" - "vmlal.s16 q15, d11, d7[3] \n" - - "bne 0b \n" - - "vstm %1!, {d16-d23} \n" - "vstm %2!, {d24-d31} \n" - - : "=r"(nn), - "=r"(output0_tm), - "=r"(output1_tm), - "=r"(r0), - "=r"(k0) - : "0"(nn), - "1"(output0_tm), - "2"(output1_tm), - "3"(r0), - "4"(k0) - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif - } - for (; i + 1 < tiles; i += 2) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); -#else - const short* r0 = bb2.row(i / 4 + (i % 4) / 2); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - - int16x8_t _w0 = vld1q_s16(k0); - int16x8_t _w1 = vld1q_s16(k0 + 8); - int16x8_t _w2 = vld1q_s16(k0 + 16); - int16x8_t _w3 = vld1q_s16(k0 + 24); - int16x8_t _w4 = vld1q_s16(k0 + 32); - int16x8_t _w5 = vld1q_s16(k0 + 40); - int16x8_t _w6 = vld1q_s16(k0 + 48); - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val1), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val1), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val1), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_high_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_high_s16(_val1), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_high_s16(_val1), 1); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_high_s16(_val1), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_high_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_high_s16(_val1), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_high_s16(_val1), 3); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_high_s16(_val1), 3); - - r0 += 16; - k0 += 64; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output1_tm, _sum1); - vst1q_s32(output0_tm + 4, _sum2); - vst1q_s32(output1_tm + 4, _sum3); - output0_tm += 8; - output1_tm += 8; - } - for (; i < tiles; i++) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); -#else - const short* r0 = bb2.row(i / 4 + (i % 4) / 2 + i % 2); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - - int16x8_t _w0 = vld1q_s16(k0); - int16x8_t _w1 = vld1q_s16(k0 + 8); - int16x8_t _w2 = vld1q_s16(k0 + 16); - int16x8_t _w3 = vld1q_s16(k0 + 24); - int16x8_t _w4 = vld1q_s16(k0 + 32); - int16x8_t _w5 = vld1q_s16(k0 + 40); - int16x8_t _w6 = vld1q_s16(k0 + 48); - int16x8_t _w7 = vld1q_s16(k0 + 56); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); - - r0 += 8; - k0 += 64; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output1_tm, _sum1); - output0_tm += 4; - output1_tm += 4; - } - } - } - - remain_outch_start += nn_outch << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - int* output0_tm = top_blob_tm.channel(p); - - const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; -#if __aarch64__ - for (; i + 11 < tiles; i += 12) - { - const short* r0 = bb2.row(i / 12); - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - asm volatile( - "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 - - "eor v8.16b, v8.16b, v8.16b \n" - "eor v9.16b, v9.16b, v9.16b \n" - - "ld1 {v4.8h, v5.8h}, [%3], #32 \n" // w01 - - "eor v10.16b, v10.16b, v10.16b \n" - "eor v11.16b, v11.16b, v11.16b \n" - - "prfm pldl1keep, [%2, #256] \n" - - "eor v12.16b, v12.16b, v12.16b \n" - "eor v13.16b, v13.16b, v13.16b \n" - - "prfm pldl1keep, [%3, #256] \n" - - "eor v14.16b, v14.16b, v14.16b \n" - "eor v15.16b, v15.16b, v15.16b \n" - "eor v16.16b, v16.16b, v16.16b \n" - "eor v17.16b, v17.16b, v17.16b \n" - "eor v18.16b, v18.16b, v18.16b \n" - "eor v19.16b, v19.16b, v19.16b \n" - - "0: \n" - - "smlal v8.4s, v4.4h, v0.h[0] \n" - "smlal v9.4s, v4.4h, v0.h[1] \n" - "smlal v10.4s, v4.4h, v0.h[2] \n" - "smlal v11.4s, v4.4h, v0.h[3] \n" - "smlal v12.4s, v4.4h, v0.h[4] \n" - "smlal v13.4s, v4.4h, v0.h[5] \n" - "smlal v14.4s, v4.4h, v0.h[6] \n" - "smlal v15.4s, v4.4h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r23 - - "smlal v16.4s, v4.4h, v1.h[0] \n" - "smlal v17.4s, v4.4h, v1.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal v18.4s, v4.4h, v1.h[2] \n" - "smlal v19.4s, v4.4h, v1.h[3] \n" - - "smlal2 v8.4s, v4.8h, v1.h[4] \n" - "smlal2 v9.4s, v4.8h, v1.h[5] \n" - "smlal2 v10.4s, v4.8h, v1.h[6] \n" - "smlal2 v11.4s, v4.8h, v1.h[7] \n" - "smlal2 v12.4s, v4.8h, v2.h[0] \n" - "smlal2 v13.4s, v4.8h, v2.h[1] \n" - "smlal2 v14.4s, v4.8h, v2.h[2] \n" - "smlal2 v15.4s, v4.8h, v2.h[3] \n" - "smlal2 v16.4s, v4.8h, v2.h[4] \n" - "smlal2 v17.4s, v4.8h, v2.h[5] \n" - "smlal2 v18.4s, v4.8h, v2.h[6] \n" - "smlal2 v19.4s, v4.8h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r45 - - "smlal v8.4s, v5.4h, v3.h[0] \n" - "smlal v9.4s, v5.4h, v3.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal v10.4s, v5.4h, v3.h[2] \n" - "smlal v11.4s, v5.4h, v3.h[3] \n" - "smlal v12.4s, v5.4h, v3.h[4] \n" - "smlal v13.4s, v5.4h, v3.h[5] \n" - "smlal v14.4s, v5.4h, v3.h[6] \n" - "smlal v15.4s, v5.4h, v3.h[7] \n" - "smlal v16.4s, v5.4h, v0.h[0] \n" - "smlal v17.4s, v5.4h, v0.h[1] \n" - "smlal v18.4s, v5.4h, v0.h[2] \n" - "smlal v19.4s, v5.4h, v0.h[3] \n" - - "ld1 {v6.8h, v7.8h}, [%3], #32 \n" // w23 - - "smlal2 v8.4s, v5.8h, v0.h[4] \n" - "smlal2 v9.4s, v5.8h, v0.h[5] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal2 v10.4s, v5.8h, v0.h[6] \n" - "smlal2 v11.4s, v5.8h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r67 - - "smlal2 v12.4s, v5.8h, v1.h[0] \n" - "smlal2 v13.4s, v5.8h, v1.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal2 v14.4s, v5.8h, v1.h[2] \n" - "smlal2 v15.4s, v5.8h, v1.h[3] \n" - "smlal2 v16.4s, v5.8h, v1.h[4] \n" - "smlal2 v17.4s, v5.8h, v1.h[5] \n" - "smlal2 v18.4s, v5.8h, v1.h[6] \n" - "smlal2 v19.4s, v5.8h, v1.h[7] \n" - - "smlal v8.4s, v6.4h, v2.h[0] \n" - "smlal v9.4s, v6.4h, v2.h[1] \n" - "smlal v10.4s, v6.4h, v2.h[2] \n" - "smlal v11.4s, v6.4h, v2.h[3] \n" - "smlal v12.4s, v6.4h, v2.h[4] \n" - "smlal v13.4s, v6.4h, v2.h[5] \n" - "smlal v14.4s, v6.4h, v2.h[6] \n" - "smlal v15.4s, v6.4h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r89 - - "smlal v16.4s, v6.4h, v3.h[0] \n" - "smlal v17.4s, v6.4h, v3.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal v18.4s, v6.4h, v3.h[2] \n" - "smlal v19.4s, v6.4h, v3.h[3] \n" - - "smlal2 v8.4s, v6.8h, v3.h[4] \n" - "smlal2 v9.4s, v6.8h, v3.h[5] \n" - "smlal2 v10.4s, v6.8h, v3.h[6] \n" - "smlal2 v11.4s, v6.8h, v3.h[7] \n" - "smlal2 v12.4s, v6.8h, v0.h[0] \n" - "smlal2 v13.4s, v6.8h, v0.h[1] \n" - "smlal2 v14.4s, v6.8h, v0.h[2] \n" - "smlal2 v15.4s, v6.8h, v0.h[3] \n" - "smlal2 v16.4s, v6.8h, v0.h[4] \n" - "smlal2 v17.4s, v6.8h, v0.h[5] \n" - "smlal2 v18.4s, v6.8h, v0.h[6] \n" - "smlal2 v19.4s, v6.8h, v0.h[7] \n" - - "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r1011 - - "smlal v8.4s, v7.4h, v1.h[0] \n" - "smlal v9.4s, v7.4h, v1.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal v10.4s, v7.4h, v1.h[2] \n" - "smlal v11.4s, v7.4h, v1.h[3] \n" - "smlal v12.4s, v7.4h, v1.h[4] \n" - "smlal v13.4s, v7.4h, v1.h[5] \n" - "smlal v14.4s, v7.4h, v1.h[6] \n" - "smlal v15.4s, v7.4h, v1.h[7] \n" - "smlal v16.4s, v7.4h, v2.h[0] \n" - "smlal v17.4s, v7.4h, v2.h[1] \n" - "smlal v18.4s, v7.4h, v2.h[2] \n" - "smlal v19.4s, v7.4h, v2.h[3] \n" - - "ld1 {v4.8h, v5.8h}, [%3], #32 \n" // w01 - - "smlal2 v8.4s, v7.8h, v2.h[4] \n" - "smlal2 v9.4s, v7.8h, v2.h[5] \n" - - "prfm pldl1keep, [%3, #256] \n" - - "smlal2 v10.4s, v7.8h, v2.h[6] \n" - "smlal2 v11.4s, v7.8h, v2.h[7] \n" - - "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 - - "smlal2 v12.4s, v7.8h, v3.h[0] \n" - "smlal2 v13.4s, v7.8h, v3.h[1] \n" - - "prfm pldl1keep, [%2, #256] \n" - - "smlal2 v14.4s, v7.8h, v3.h[2] \n" - "smlal2 v15.4s, v7.8h, v3.h[3] \n" - "smlal2 v16.4s, v7.8h, v3.h[4] \n" - "smlal2 v17.4s, v7.8h, v3.h[5] \n" - - "subs %w0, %w0, #1 \n" - - "smlal2 v18.4s, v7.8h, v3.h[6] \n" - "smlal2 v19.4s, v7.8h, v3.h[7] \n" - - "bne 0b \n" - - "sub %2, %2, #32 \n" - "sub %3, %3, #32 \n" - - "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" - "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" - "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" - - : "=r"(nn), // %0 - "=r"(output0_tm), // %1 - "=r"(r0), // %2 - "=r"(k0) // %3 - : "0"(nn), - "1"(output0_tm), - "2"(r0), - "3"(k0) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); - } - for (; i + 7 < tiles; i += 8) - { - const short* r0 = bb2.row(i / 12 + (i % 12) / 8); - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - int32x4_t _sum4 = vdupq_n_s32(0); - int32x4_t _sum5 = vdupq_n_s32(0); - int32x4_t _sum6 = vdupq_n_s32(0); - int32x4_t _sum7 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - int16x8_t _val2 = vld1q_s16(r0 + 16); - int16x8_t _val3 = vld1q_s16(r0 + 24); - int16x8_t _val4 = vld1q_s16(r0 + 32); - int16x8_t _val5 = vld1q_s16(r0 + 40); - int16x8_t _val6 = vld1q_s16(r0 + 48); - int16x8_t _val7 = vld1q_s16(r0 + 56); - - int16x8_t _w0 = vld1q_s16(k0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w0), vget_low_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val0), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w0), vget_low_s16(_val0), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_high_s16(_val0), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w0), vget_high_s16(_val0), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_high_s16(_val0), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w0), vget_high_s16(_val0), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w0), vget_low_s16(_val1), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val1), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w0), vget_low_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w0), vget_high_s16(_val1), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_high_s16(_val1), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w0), vget_high_s16(_val1), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_high_s16(_val1), 3); - - int16x8_t _w1 = vld1q_s16(k0 + 8); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val2), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w1), vget_low_s16(_val2), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val2), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w1), vget_low_s16(_val2), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_high_s16(_val2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w1), vget_high_s16(_val2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_high_s16(_val2), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w1), vget_high_s16(_val2), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w1), vget_low_s16(_val3), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val3), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w1), vget_low_s16(_val3), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val3), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w1), vget_high_s16(_val3), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_high_s16(_val3), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w1), vget_high_s16(_val3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_high_s16(_val3), 3); - - int16x8_t _w2 = vld1q_s16(k0 + 16); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val4), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w2), vget_low_s16(_val4), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val4), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w2), vget_low_s16(_val4), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_high_s16(_val4), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w2), vget_high_s16(_val4), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_high_s16(_val4), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w2), vget_high_s16(_val4), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w2), vget_low_s16(_val5), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val5), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w2), vget_low_s16(_val5), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val5), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w2), vget_high_s16(_val5), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_high_s16(_val5), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w2), vget_high_s16(_val5), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_high_s16(_val5), 3); - - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val6), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w3), vget_low_s16(_val6), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val6), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w3), vget_low_s16(_val6), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_high_s16(_val6), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w3), vget_high_s16(_val6), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_high_s16(_val6), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w3), vget_high_s16(_val6), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w3), vget_low_s16(_val7), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val7), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w3), vget_low_s16(_val7), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val7), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w3), vget_high_s16(_val7), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_high_s16(_val7), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w3), vget_high_s16(_val7), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_high_s16(_val7), 3); - - r0 += 64; - k0 += 32; - } - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output0_tm + 4, _sum1); - vst1q_s32(output0_tm + 8, _sum2); - vst1q_s32(output0_tm + 12, _sum3); - vst1q_s32(output0_tm + 16, _sum4); - vst1q_s32(output0_tm + 20, _sum5); - vst1q_s32(output0_tm + 24, _sum6); - vst1q_s32(output0_tm + 28, _sum7); - output0_tm += 32; - } -#endif // __aarch64__ - for (; i + 3 < tiles; i += 4) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); -#else - const short* r0 = bb2.row(i / 4); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - -#if __aarch64__ - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - int32x4_t _sum4 = vdupq_n_s32(0); - int32x4_t _sum5 = vdupq_n_s32(0); - int32x4_t _sum6 = vdupq_n_s32(0); - int32x4_t _sum7 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - int16x8_t _val2 = vld1q_s16(r0 + 16); - int16x8_t _val3 = vld1q_s16(r0 + 24); - - int16x8_t _w0 = vld1q_s16(k0); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val3), 0); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val3), 1); - - int16x8_t _w1 = vld1q_s16(k0 + 8); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val2), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val2), 3); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val3), 3); - - int16x8_t _w2 = vld1q_s16(k0 + 16); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_high_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_high_s16(_val1), 1); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_high_s16(_val2), 0); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_high_s16(_val2), 1); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_high_s16(_val3), 0); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_high_s16(_val3), 1); - - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_high_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_high_s16(_val1), 3); - _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_high_s16(_val2), 2); - _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_high_s16(_val2), 3); - _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_high_s16(_val3), 2); - _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_high_s16(_val3), 3); - - r0 += 32; - k0 += 32; - } - - _sum0 = vaddq_s32(_sum0, _sum1); - _sum2 = vaddq_s32(_sum2, _sum3); - _sum4 = vaddq_s32(_sum4, _sum5); - _sum6 = vaddq_s32(_sum6, _sum7); - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output0_tm + 4, _sum2); - vst1q_s32(output0_tm + 8, _sum4); - vst1q_s32(output0_tm + 12, _sum6); - output0_tm += 16; -#else - asm volatile( - "veor q8, q8 \n" - "veor q9, q9 \n" - "veor q10, q10 \n" - "veor q11, q11 \n" - "veor q12, q12 \n" - "veor q13, q13 \n" - "veor q14, q14 \n" - "veor q15, q15 \n" - - "0: \n" - - "pld [%2, #256] \n" - "pld [%2, #512] \n" - "vldm %2!, {d0-d7} \n" - - "pld [%3, #256] \n" - "vld1.s16 {d8-d11}, [%3 :128]! \n" - - "vmlal.s16 q8, d8, d0[0] \n" - "vmlal.s16 q12, d9, d0[1] \n" - "vmlal.s16 q9, d8, d2[0] \n" - "vmlal.s16 q13, d9, d2[1] \n" - "vmlal.s16 q10, d8, d4[0] \n" - "vmlal.s16 q14, d9, d4[1] \n" - "vmlal.s16 q11, d8, d6[0] \n" - "vmlal.s16 q15, d9, d6[1] \n" - - "pld [%3, #128] \n" - "vld1.s16 {d8-d9}, [%3 :128]! \n" - - "vmlal.s16 q8, d10, d0[2] \n" - "vmlal.s16 q12, d11, d0[3] \n" - "vmlal.s16 q9, d10, d2[2] \n" - "vmlal.s16 q13, d11, d2[3] \n" - "vmlal.s16 q10, d10, d4[2] \n" - "vmlal.s16 q14, d11, d4[3] \n" - "vmlal.s16 q11, d10, d6[2] \n" - "vmlal.s16 q15, d11, d6[3] \n" - - "pld [%3, #128] \n" - "vld1.s16 {d10-d11}, [%3 :128]! \n" - - "vmlal.s16 q8, d8, d1[0] \n" - "vmlal.s16 q12, d9, d1[1] \n" - "vmlal.s16 q9, d8, d3[0] \n" - "vmlal.s16 q13, d9, d3[1] \n" - "vmlal.s16 q10, d8, d5[0] \n" - "vmlal.s16 q14, d9, d5[1] \n" - "vmlal.s16 q11, d8, d7[0] \n" - "vmlal.s16 q15, d9, d7[1] \n" - - "subs %0, %0, #1 \n" - - "vmlal.s16 q8, d10, d1[2] \n" - "vmlal.s16 q12, d11, d1[3] \n" - "vmlal.s16 q9, d10, d3[2] \n" - "vmlal.s16 q13, d11, d3[3] \n" - "vmlal.s16 q10, d10, d5[2] \n" - "vmlal.s16 q14, d11, d5[3] \n" - "vmlal.s16 q11, d10, d7[2] \n" - "vmlal.s16 q15, d11, d7[3] \n" - - "bne 0b \n" - - "vadd.s32 q8, q8, q12 \n" - "vadd.s32 q9, q9, q13 \n" - "vadd.s32 q10, q10, q14 \n" - "vadd.s32 q11, q11, q15 \n" - - "vstm %1!, {d16-d23} \n" - - : "=r"(nn), - "=r"(output0_tm), - "=r"(r0), - "=r"(k0) - : "0"(nn), - "1"(output0_tm), - "2"(r0), - "3"(k0) - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); -#endif - } - for (; i + 1 < tiles; i += 2) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); -#else - const short* r0 = bb2.row(i / 4 + (i % 4) / 2); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - int32x4_t _sum2 = vdupq_n_s32(0); - int32x4_t _sum3 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - int16x8_t _val1 = vld1q_s16(r0 + 8); - - int16x8_t _w0 = vld1q_s16(k0); - int16x8_t _w1 = vld1q_s16(k0 + 8); - int16x8_t _w2 = vld1q_s16(k0 + 16); - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_high_s16(_val1), 0); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_high_s16(_val1), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); - _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_high_s16(_val1), 2); - _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_high_s16(_val1), 3); - - r0 += 16; - k0 += 32; - } - - _sum0 = vaddq_s32(_sum0, _sum1); - _sum2 = vaddq_s32(_sum2, _sum3); - - vst1q_s32(output0_tm, _sum0); - vst1q_s32(output0_tm + 4, _sum2); - output0_tm += 8; - } - for (; i < tiles; i++) - { -#if __aarch64__ - const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); -#else - const short* r0 = bb2.row(i / 4 + (i % 4) / 2 + i % 2); -#endif - const short* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); - - for (int j = 0; j < nn; j++) - { - int16x8_t _val0 = vld1q_s16(r0); - - int16x8_t _w0 = vld1q_s16(k0); - int16x8_t _w1 = vld1q_s16(k0 + 8); - int16x8_t _w2 = vld1q_s16(k0 + 16); - int16x8_t _w3 = vld1q_s16(k0 + 24); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); - - _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); - _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); - - r0 += 8; - k0 += 32; - } - - _sum0 = vaddq_s32(_sum0, _sum1); - - vst1q_s32(output0_tm, _sum0); - output0_tm += 4; - } - } - } - } - bottom_blob_tm = Mat(); + convolution_winograd_dot_pack8to4_int8_neon(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output @@ -2134,166 +196,7 @@ static void conv3x3s1_winograd43_pack8to4_int8_neon(const Mat& bottom_blob, Mat& top_blob_bordered.create(outw, outh, outch, 4u * 4, 4, opt.workspace_allocator); } { - // const float otm[4][6] = { - // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, - // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, - // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} - // }; - - // 0 = r00 + (r01 + r02) + (r03 + r04) - // 1 = (r01 - r02) + (r03 - r04) * 2 - // 2 = (r01 + r02) + (r03 + r04) * 4 - // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 - - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - const int tiles = w_tm / 6 * h_tm / 6; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - const Mat out0_tm = top_blob_tm.channel(p); - Mat out0 = top_blob_bordered.channel(p); - - int tmp[4][6][4]; - - // tile - for (int i = 0; i < outh / 4; i++) - { - for (int j = 0; j < outw / 4; j++) - { - // top_blob_tm.create(tiles, 36, outch, elemsize, elempack); - - const int* output0_tm_0 = (const int*)out0_tm + (i * w_tm / 6 + j) * 4; - const int* output0_tm_1 = output0_tm_0 + tiles * 4; - const int* output0_tm_2 = output0_tm_0 + tiles * 8; - const int* output0_tm_3 = output0_tm_0 + tiles * 12; - const int* output0_tm_4 = output0_tm_0 + tiles * 16; - const int* output0_tm_5 = output0_tm_0 + tiles * 20; - - int* output0 = out0.row(i * 4) + (j * 4) * 4; - - // TODO neon optimize - for (int m = 0; m < 5; m++) - { - int32x4_t _out0tm0 = vld1q_s32(output0_tm_0); - int32x4_t _out0tm1 = vld1q_s32(output0_tm_1); - int32x4_t _out0tm2 = vld1q_s32(output0_tm_2); - int32x4_t _out0tm3 = vld1q_s32(output0_tm_3); - int32x4_t _out0tm4 = vld1q_s32(output0_tm_4); - int32x4_t _out0tm5 = vld1q_s32(output0_tm_5); - - int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2); - int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2); - - int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4); - int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4); - - int32x4_t _v2 = vdupq_n_s32(2); - int32x4_t _v4 = vdupq_n_s32(4); - int32x4_t _v8 = vdupq_n_s32(8); - - int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b); - int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2); - int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4); - int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8); - - vst1q_s32(tmp[0][m], _tmp0m); - vst1q_s32(tmp[1][m], _tmp1m); - vst1q_s32(tmp[2][m], _tmp2m); - vst1q_s32(tmp[3][m], _tmp3m); - - output0_tm_0 += tiles * 24; - output0_tm_1 += tiles * 24; - output0_tm_2 += tiles * 24; - output0_tm_3 += tiles * 24; - output0_tm_4 += tiles * 24; - output0_tm_5 += tiles * 24; - } - for (int m = 5; m < 6; m++) - { - int32x4_t _out0tm0 = vld1q_s32(output0_tm_0); - int32x4_t _out0tm1 = vld1q_s32(output0_tm_1); - int32x4_t _out0tm2 = vld1q_s32(output0_tm_2); - int32x4_t _out0tm3 = vld1q_s32(output0_tm_3); - int32x4_t _out0tm4 = vld1q_s32(output0_tm_4); - int32x4_t _out0tm5 = vld1q_s32(output0_tm_5); - - int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2); - int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2); - - int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4); - int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4); - - int32x4_t _v2 = vdupq_n_s32(2); - int32x4_t _v4 = vdupq_n_s32(4); - int32x4_t _v8 = vdupq_n_s32(8); - - int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b); - int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2); - int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4); - int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8); - - _tmp0m = vmulq_s32(_tmp0m, _v4); - _tmp1m = vmulq_s32(_tmp1m, _v4); - _tmp2m = vmulq_s32(_tmp2m, _v4); - _tmp3m = vmulq_s32(_tmp3m, _v4); - - vst1q_s32(tmp[0][m], _tmp0m); - vst1q_s32(tmp[1][m], _tmp1m); - vst1q_s32(tmp[2][m], _tmp2m); - vst1q_s32(tmp[3][m], _tmp3m); - - output0_tm_0 += tiles * 24; - output0_tm_1 += tiles * 24; - output0_tm_2 += tiles * 24; - output0_tm_3 += tiles * 24; - output0_tm_4 += tiles * 24; - output0_tm_5 += tiles * 24; - } - - for (int m = 0; m < 4; m++) - { - int32x4_t _tmp00 = vld1q_s32(tmp[m][0]); - int32x4_t _tmp01 = vld1q_s32(tmp[m][1]); - int32x4_t _tmp02 = vld1q_s32(tmp[m][2]); - int32x4_t _tmp03 = vld1q_s32(tmp[m][3]); - int32x4_t _tmp04 = vld1q_s32(tmp[m][4]); - int32x4_t _tmp05 = vld1q_s32(tmp[m][5]); - - int32x4_t _tmp02a = vaddq_s32(_tmp01, _tmp02); - int32x4_t _tmp13a = vsubq_s32(_tmp01, _tmp02); - - int32x4_t _tmp02b = vaddq_s32(_tmp03, _tmp04); - int32x4_t _tmp13b = vsubq_s32(_tmp03, _tmp04); - - int32x4_t _v2 = vdupq_n_s32(2); - int32x4_t _v4 = vdupq_n_s32(4); - int32x4_t _v8 = vdupq_n_s32(8); - - int32x4_t _out00 = vaddq_s32(vaddq_s32(_tmp00, _tmp02a), _tmp02b); - int32x4_t _out01 = vmlaq_s32(_tmp13a, _tmp13b, _v2); - int32x4_t _out02 = vmlaq_s32(_tmp02a, _tmp02b, _v4); - int32x4_t _out03 = vmlaq_s32(vaddq_s32(_tmp05, _tmp13a), _tmp13b, _v8); - - // TODO use integer trick for division by 576 - float32x4_t _v576 = vdupq_n_f32(1.0 / 576); - _out00 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out00), _v576)); - _out01 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out01), _v576)); - _out02 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out02), _v576)); - _out03 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out03), _v576)); - - vst1q_s32(output0, _out00); - vst1q_s32(output0 + 4, _out01); - vst1q_s32(output0 + 8, _out02); - vst1q_s32(output0 + 12, _out03); - - output0 += outw * 4; - } - } - } - } + conv3x3s1_winograd43_transform_output_pack4_int8_neon(top_blob_tm, top_blob_bordered, opt); } // END transform output diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index e05b6c18f..1f8f0c1cb 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -46,6 +46,8 @@ namespace ncnn { #if NCNN_INT8 #include "convolution_sgemm_int8.h" +#include "convolution_winograd_transform_int8.h" +#include "convolution_winograd_dot_int8.h" #include "convolution_1x1_int8.h" #include "convolution_3x3_int8.h" #include "convolution_int8.h" @@ -94,6 +96,10 @@ namespace ncnn { #include "convolution_sgemm_pack8to4_int8.h" #include "convolution_sgemm_pack1to4_int8.h" #include "convolution_sgemm_pack8to1_int8.h" +#include "convolution_winograd_transform_pack4_int8.h" +#include "convolution_winograd_transform_pack8_int8.h" +#include "convolution_winograd_dot_pack8to4_int8.h" +#include "convolution_winograd_dot_pack8to1_int8.h" #include "convolution_1x1_pack8to4_int8.h" #include "convolution_1x1_pack1to4_int8.h" #include "convolution_1x1_pack8to1_int8.h" @@ -1646,8 +1652,7 @@ int Convolution_arm::create_pipeline_int8_arm(const Option& opt) { if (opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - // conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output, opt); - conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output, opt); + conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_winograd43_data, num_input, num_output, opt); } /* if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) @@ -1674,6 +1679,19 @@ int Convolution_arm::create_pipeline_int8_arm(const Option& opt) } } + scale_in_data.create(num_output); + for (int p = 0; p < num_output; p++) + { + // requantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); + + scale_in_data[p] = scale_in; + } + if (opt.lightmode) { weight_data.release(); @@ -1783,33 +1801,6 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con { convolution_pack8to4_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - } } if (elempack == 1 && out_elempack_int32 == 4) @@ -1842,33 +1833,6 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con { convolution_pack1to4_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - } } if (elempack == 8 && out_elempack_int32 == 1) @@ -1893,33 +1857,6 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con { convolution_pack8to1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - } } #endif // __ARM_NEON @@ -1935,8 +1872,7 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con } else if (opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - // conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_int32, weight_3x3_winograd23_data_int8, opt); - conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_int32, weight_3x3_winograd23_data_int8, opt); + conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_int32, weight_winograd43_data, opt); } else if (opt.use_sgemm_convolution) { @@ -1946,32 +1882,19 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con { convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } + } - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } + if (use_int8_requantize) + { + requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); + } + else + { + dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - if (use_int8_requantize) + if (activation) { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + activation->forward_inplace(top_blob, opt); } } diff --git a/src/layer/arm/convolution_arm.h b/src/layer/arm/convolution_arm.h index fe5da2ca4..c22a7f65f 100644 --- a/src/layer/arm/convolution_arm.h +++ b/src/layer/arm/convolution_arm.h @@ -65,8 +65,7 @@ public: Mat bias_data_fp16; #if NCNN_INT8 - // Mat weight_3x3s2_data_int8; - std::vector weight_3x3_winograd23_data_int8; + Mat scale_in_data; #endif }; diff --git a/src/layer/arm/convolution_winograd_dot_int8.h b/src/layer/arm/convolution_winograd_dot_int8.h new file mode 100644 index 000000000..d5cf1bcd8 --- /dev/null +++ b/src/layer/arm/convolution_winograd_dot_int8.h @@ -0,0 +1,1005 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_winograd_dot_int8_neon(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt) +{ + // Mat bottom_blob_tm(tiles, 16/36/64, inch, 2u, 1, opt.workspace_allocator); + + const int tiles = bottom_blob_tm.w; + const int batch = bottom_blob_tm.h; + const int inch = bottom_blob_tm.c; + + // permute + Mat bottom_blob_tm2; +#if __ARM_NEON +#if __aarch64__ + if (tiles >= 8) + bottom_blob_tm2.create(inch, tiles / 8 + (tiles % 8) / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 4) + bottom_blob_tm2.create(inch, tiles / 4 + tiles % 4, batch, 8u, 4, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(inch, tiles, batch, 2u, 1, opt.workspace_allocator); +#else + if (tiles >= 4) + bottom_blob_tm2.create(inch, tiles / 4 + tiles % 4, batch, 8u, 4, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(inch, tiles, batch, 2u, 1, opt.workspace_allocator); +#endif +#else // __ARM_NEON + if (tiles >= 2) + bottom_blob_tm2.create(inch, tiles / 2 + tiles % 2, batch, 4u, 2, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(inch, tiles, batch, 2u, 1, opt.workspace_allocator); +#endif // __ARM_NEON + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < batch; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; +#if __ARM_NEON +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + short* tmpptr = tm2.row(i / 8); + const short* r0 = (const short*)bottom_blob_tm + r * tiles + i; + + int q = 0; + for (; q < inch; q++) + { + int16x8_t _r0 = vld1q_s16(r0); + vst1q_s16(tmpptr, _r0); + r0 += bottom_blob_tm.cstep; + tmpptr += 8; + } + } +#endif + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + short* tmpptr = tm2.row(i / 8 + (i % 8) / 4); +#else + short* tmpptr = tm2.row(i / 4); +#endif + const short* r0 = (const short*)bottom_blob_tm + r * tiles + i; + + int q = 0; + for (; q < inch; q++) + { + int16x4_t _r0 = vld1_s16(r0); + vst1_s16(tmpptr, _r0); + r0 += bottom_blob_tm.cstep; + tmpptr += 4; + } + } +#else // __ARM_NEON + for (; i + 1 < tiles; i += 2) + { + short* tmpptr = tm2.row(i / 2); + const short* r0 = (const short*)bottom_blob_tm + r * tiles + i; + + int q = 0; +#if __ARM_FEATURE_SIMD32 + for (; q + 1 < inch; q += 2) + { + tmpptr[0] = r0[0]; + tmpptr[2] = r0[1]; + r0 += bottom_blob_tm.cstep; + tmpptr[1] = r0[0]; + tmpptr[3] = r0[1]; + r0 += bottom_blob_tm.cstep; + tmpptr += 4; + } +#endif // __ARM_FEATURE_SIMD32 + for (; q < inch; q++) + { + tmpptr[0] = r0[0]; + tmpptr[1] = r0[1]; + r0 += bottom_blob_tm.cstep; + tmpptr += 2; + } + } +#endif // __ARM_NEON + for (; i < tiles; i++) + { +#if __ARM_NEON +#if __aarch64__ + short* tmpptr = tm2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + short* tmpptr = tm2.row(i / 4 + i % 4); +#endif +#else + short* tmpptr = tm2.row(i / 2 + i % 2); +#endif + const short* r0 = (const short*)bottom_blob_tm + r * tiles + i; + + int q = 0; + for (; q < inch; q++) + { + tmpptr[0] = r0[0]; + r0 += bottom_blob_tm.cstep; + tmpptr += 1; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, batch, outch, 4u, 1, opt.workspace_allocator); + +#if __ARM_NEON + int nn_outch = outch >> 3; + int remain_outch_start = nn_outch << 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 8; + + int* output0_tm = top_blob_tm.channel(p); + int* output1_tm = top_blob_tm.channel(p + 1); + int* output2_tm = top_blob_tm.channel(p + 2); + int* output3_tm = top_blob_tm.channel(p + 3); + int* output4_tm = top_blob_tm.channel(p + 4); + int* output5_tm = top_blob_tm.channel(p + 5); + int* output6_tm = top_blob_tm.channel(p + 6); + int* output7_tm = top_blob_tm.channel(p + 7); + + const Mat kernel0_tm = kernel_tm.channel(p / 8); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 8); + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum00 = vdupq_n_s32(0); + int32x4_t _sum10 = vdupq_n_s32(0); + int32x4_t _sum20 = vdupq_n_s32(0); + int32x4_t _sum30 = vdupq_n_s32(0); + int32x4_t _sum40 = vdupq_n_s32(0); + int32x4_t _sum50 = vdupq_n_s32(0); + int32x4_t _sum60 = vdupq_n_s32(0); + int32x4_t _sum70 = vdupq_n_s32(0); + int32x4_t _sum01 = vdupq_n_s32(0); + int32x4_t _sum11 = vdupq_n_s32(0); + int32x4_t _sum21 = vdupq_n_s32(0); + int32x4_t _sum31 = vdupq_n_s32(0); + int32x4_t _sum41 = vdupq_n_s32(0); + int32x4_t _sum51 = vdupq_n_s32(0); + int32x4_t _sum61 = vdupq_n_s32(0); + int32x4_t _sum71 = vdupq_n_s32(0); + + int j = 0; + for (; j < inch; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _w0 = vld1q_s16(k0); + + _sum00 = vmlal_lane_s16(_sum00, vget_low_s16(_val0), vget_low_s16(_w0), 0); + _sum10 = vmlal_lane_s16(_sum10, vget_low_s16(_val0), vget_low_s16(_w0), 1); + _sum20 = vmlal_lane_s16(_sum20, vget_low_s16(_val0), vget_low_s16(_w0), 2); + _sum30 = vmlal_lane_s16(_sum30, vget_low_s16(_val0), vget_low_s16(_w0), 3); + _sum40 = vmlal_lane_s16(_sum40, vget_low_s16(_val0), vget_high_s16(_w0), 0); + _sum50 = vmlal_lane_s16(_sum50, vget_low_s16(_val0), vget_high_s16(_w0), 1); + _sum60 = vmlal_lane_s16(_sum60, vget_low_s16(_val0), vget_high_s16(_w0), 2); + _sum70 = vmlal_lane_s16(_sum70, vget_low_s16(_val0), vget_high_s16(_w0), 3); + + _sum01 = vmlal_lane_s16(_sum01, vget_high_s16(_val0), vget_low_s16(_w0), 0); + _sum11 = vmlal_lane_s16(_sum11, vget_high_s16(_val0), vget_low_s16(_w0), 1); + _sum21 = vmlal_lane_s16(_sum21, vget_high_s16(_val0), vget_low_s16(_w0), 2); + _sum31 = vmlal_lane_s16(_sum31, vget_high_s16(_val0), vget_low_s16(_w0), 3); + _sum41 = vmlal_lane_s16(_sum41, vget_high_s16(_val0), vget_high_s16(_w0), 0); + _sum51 = vmlal_lane_s16(_sum51, vget_high_s16(_val0), vget_high_s16(_w0), 1); + _sum61 = vmlal_lane_s16(_sum61, vget_high_s16(_val0), vget_high_s16(_w0), 2); + _sum71 = vmlal_lane_s16(_sum71, vget_high_s16(_val0), vget_high_s16(_w0), 3); + + r0 += 8; + k0 += 8; + } + + vst1q_s32(output0_tm, _sum00); + vst1q_s32(output0_tm + 4, _sum01); + vst1q_s32(output1_tm, _sum10); + vst1q_s32(output1_tm + 4, _sum11); + vst1q_s32(output2_tm, _sum20); + vst1q_s32(output2_tm + 4, _sum21); + vst1q_s32(output3_tm, _sum30); + vst1q_s32(output3_tm + 4, _sum31); + vst1q_s32(output4_tm, _sum40); + vst1q_s32(output4_tm + 4, _sum41); + vst1q_s32(output5_tm, _sum50); + vst1q_s32(output5_tm + 4, _sum51); + vst1q_s32(output6_tm, _sum60); + vst1q_s32(output6_tm + 4, _sum61); + vst1q_s32(output7_tm, _sum70); + vst1q_s32(output7_tm + 4, _sum71); + + output0_tm += 8; + output1_tm += 8; + output2_tm += 8; + output3_tm += 8; + output4_tm += 8; + output5_tm += 8; + output6_tm += 8; + output7_tm += 8; + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + + int j = 0; + for (; j + 1 < inch; j += 2) + { + int16x8_t _val01 = vld1q_s16(r0); + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val01), vget_low_s16(_w0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val01), vget_low_s16(_w0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val01), vget_low_s16(_w0), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val01), vget_low_s16(_w0), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val01), vget_high_s16(_w0), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val01), vget_high_s16(_w0), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val01), vget_high_s16(_w0), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val01), vget_high_s16(_w0), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val01), vget_low_s16(_w1), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val01), vget_low_s16(_w1), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val01), vget_low_s16(_w1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val01), vget_low_s16(_w1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val01), vget_high_s16(_w1), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val01), vget_high_s16(_w1), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val01), vget_high_s16(_w1), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val01), vget_high_s16(_w1), 3); + + r0 += 8; + k0 += 16; + } + for (; j < inch; j++) + { + int16x4_t _val0 = vld1_s16(r0); + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, _val0, vget_low_s16(_w0), 0); + _sum1 = vmlal_lane_s16(_sum1, _val0, vget_low_s16(_w0), 1); + _sum2 = vmlal_lane_s16(_sum2, _val0, vget_low_s16(_w0), 2); + _sum3 = vmlal_lane_s16(_sum3, _val0, vget_low_s16(_w0), 3); + _sum4 = vmlal_lane_s16(_sum4, _val0, vget_high_s16(_w0), 0); + _sum5 = vmlal_lane_s16(_sum5, _val0, vget_high_s16(_w0), 1); + _sum6 = vmlal_lane_s16(_sum6, _val0, vget_high_s16(_w0), 2); + _sum7 = vmlal_lane_s16(_sum7, _val0, vget_high_s16(_w0), 3); + + r0 += 4; + k0 += 8; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output2_tm, _sum2); + vst1q_s32(output3_tm, _sum3); + vst1q_s32(output4_tm, _sum4); + vst1q_s32(output5_tm, _sum5); + vst1q_s32(output6_tm, _sum6); + vst1q_s32(output7_tm, _sum7); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + output4_tm += 4; + output5_tm += 4; + output6_tm += 4; + output7_tm += 4; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + const short* r0 = bb2.row(i / 4 + i % 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + int j = 0; + for (; j + 3 < inch; j += 4) + { + int16x4_t _val0123 = vld1_s16(r0); + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), _val0123, 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), _val0123, 0); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), _val0123, 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), _val0123, 1); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), _val0123, 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), _val0123, 2); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), _val0123, 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), _val0123, 3); + + r0 += 4; + k0 += 32; + } + for (; j < inch; j++) + { + int16x4_t _val0 = vld1_dup_s16(r0); + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_s16(_sum0, _val0, vget_low_s16(_w0)); + _sum1 = vmlal_s16(_sum1, _val0, vget_high_s16(_w0)); + + r0 += 1; + k0 += 8; + } + + output0_tm[0] = vgetq_lane_s32(_sum0, 0); + output1_tm[0] = vgetq_lane_s32(_sum0, 1); + output2_tm[0] = vgetq_lane_s32(_sum0, 2); + output3_tm[0] = vgetq_lane_s32(_sum0, 3); + output4_tm[0] = vgetq_lane_s32(_sum1, 0); + output5_tm[0] = vgetq_lane_s32(_sum1, 1); + output6_tm[0] = vgetq_lane_s32(_sum1, 2); + output7_tm[0] = vgetq_lane_s32(_sum1, 3); + output0_tm += 1; + output1_tm += 1; + output2_tm += 1; + output3_tm += 1; + output4_tm += 1; + output5_tm += 1; + output6_tm += 1; + output7_tm += 1; + } + } + } + + nn_outch = (outch - remain_outch_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = remain_outch_start + pp * 4; + + int* output0_tm = top_blob_tm.channel(p); + int* output1_tm = top_blob_tm.channel(p + 1); + int* output2_tm = top_blob_tm.channel(p + 2); + int* output3_tm = top_blob_tm.channel(p + 3); + + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 8); + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum00 = vdupq_n_s32(0); + int32x4_t _sum10 = vdupq_n_s32(0); + int32x4_t _sum20 = vdupq_n_s32(0); + int32x4_t _sum30 = vdupq_n_s32(0); + int32x4_t _sum01 = vdupq_n_s32(0); + int32x4_t _sum11 = vdupq_n_s32(0); + int32x4_t _sum21 = vdupq_n_s32(0); + int32x4_t _sum31 = vdupq_n_s32(0); + + int j = 0; + for (; j + 1 < inch; j += 2) + { + int16x8_t _val01 = vld1q_s16(r0); + int16x8_t _val23 = vld1q_s16(r0 + 8); + int16x8_t _w01 = vld1q_s16(k0); + + _sum00 = vmlal_lane_s16(_sum00, vget_low_s16(_val01), vget_low_s16(_w01), 0); + _sum10 = vmlal_lane_s16(_sum10, vget_low_s16(_val01), vget_low_s16(_w01), 1); + _sum20 = vmlal_lane_s16(_sum20, vget_low_s16(_val01), vget_low_s16(_w01), 2); + _sum30 = vmlal_lane_s16(_sum30, vget_low_s16(_val01), vget_low_s16(_w01), 3); + _sum01 = vmlal_lane_s16(_sum01, vget_high_s16(_val01), vget_low_s16(_w01), 0); + _sum11 = vmlal_lane_s16(_sum11, vget_high_s16(_val01), vget_low_s16(_w01), 1); + _sum21 = vmlal_lane_s16(_sum21, vget_high_s16(_val01), vget_low_s16(_w01), 2); + _sum31 = vmlal_lane_s16(_sum31, vget_high_s16(_val01), vget_low_s16(_w01), 3); + + _sum00 = vmlal_lane_s16(_sum00, vget_low_s16(_val23), vget_high_s16(_w01), 0); + _sum10 = vmlal_lane_s16(_sum10, vget_low_s16(_val23), vget_high_s16(_w01), 1); + _sum20 = vmlal_lane_s16(_sum20, vget_low_s16(_val23), vget_high_s16(_w01), 2); + _sum30 = vmlal_lane_s16(_sum30, vget_low_s16(_val23), vget_high_s16(_w01), 3); + _sum01 = vmlal_lane_s16(_sum01, vget_high_s16(_val23), vget_high_s16(_w01), 0); + _sum11 = vmlal_lane_s16(_sum11, vget_high_s16(_val23), vget_high_s16(_w01), 1); + _sum21 = vmlal_lane_s16(_sum21, vget_high_s16(_val23), vget_high_s16(_w01), 2); + _sum31 = vmlal_lane_s16(_sum31, vget_high_s16(_val23), vget_high_s16(_w01), 3); + + r0 += 16; + k0 += 8; + } + for (; j < inch; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x4_t _w0 = vld1_s16(k0); + + _sum00 = vmlal_lane_s16(_sum00, vget_low_s16(_val0), _w0, 0); + _sum10 = vmlal_lane_s16(_sum10, vget_low_s16(_val0), _w0, 1); + _sum20 = vmlal_lane_s16(_sum20, vget_low_s16(_val0), _w0, 2); + _sum30 = vmlal_lane_s16(_sum30, vget_low_s16(_val0), _w0, 3); + _sum01 = vmlal_lane_s16(_sum01, vget_high_s16(_val0), _w0, 0); + _sum11 = vmlal_lane_s16(_sum11, vget_high_s16(_val0), _w0, 1); + _sum21 = vmlal_lane_s16(_sum21, vget_high_s16(_val0), _w0, 2); + _sum31 = vmlal_lane_s16(_sum31, vget_high_s16(_val0), _w0, 3); + + r0 += 8; + k0 += 4; + } + + vst1q_s32(output0_tm, _sum00); + vst1q_s32(output0_tm + 4, _sum01); + vst1q_s32(output1_tm, _sum10); + vst1q_s32(output1_tm + 4, _sum11); + vst1q_s32(output2_tm, _sum20); + vst1q_s32(output2_tm + 4, _sum21); + vst1q_s32(output3_tm, _sum30); + vst1q_s32(output3_tm + 4, _sum31); + + output0_tm += 8; + output1_tm += 8; + output2_tm += 8; + output3_tm += 8; + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + int j = 0; + for (; j + 1 < inch; j += 2) + { + int16x8_t _val01 = vld1q_s16(r0); + int16x8_t _w01 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val01), vget_low_s16(_w01), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val01), vget_low_s16(_w01), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val01), vget_low_s16(_w01), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val01), vget_low_s16(_w01), 3); + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val01), vget_high_s16(_w01), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val01), vget_high_s16(_w01), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val01), vget_high_s16(_w01), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val01), vget_high_s16(_w01), 3); + + r0 += 8; + k0 += 8; + } + for (; j < inch; j++) + { + int16x4_t _val0 = vld1_s16(r0); + int16x4_t _w0 = vld1_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, _val0, _w0, 0); + _sum1 = vmlal_lane_s16(_sum1, _val0, _w0, 1); + _sum2 = vmlal_lane_s16(_sum2, _val0, _w0, 2); + _sum3 = vmlal_lane_s16(_sum3, _val0, _w0, 3); + + r0 += 4; + k0 += 4; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output2_tm, _sum2); + vst1q_s32(output3_tm, _sum3); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + const short* r0 = bb2.row(i / 4 + i % 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + int j = 0; + for (; j + 3 < inch; j += 4) + { + int16x4_t _val0123 = vld1_s16(r0); + int16x8_t _w01 = vld1q_s16(k0); + int16x8_t _w23 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w01), _val0123, 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w01), _val0123, 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w23), _val0123, 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w23), _val0123, 3); + + r0 += 4; + k0 += 16; + } + _sum0 = vaddq_s32(_sum0, _sum1); + _sum2 = vaddq_s32(_sum2, _sum3); + _sum0 = vaddq_s32(_sum0, _sum2); + for (; j < inch; j++) + { + int16x4_t _val0 = vld1_dup_s16(r0); + int16x4_t _w0 = vld1_s16(k0); + + _sum0 = vmlal_s16(_sum0, _val0, _w0); + + r0 += 1; + k0 += 4; + } + + output0_tm[0] = vgetq_lane_s32(_sum0, 0); + output1_tm[0] = vgetq_lane_s32(_sum0, 1); + output2_tm[0] = vgetq_lane_s32(_sum0, 2); + output3_tm[0] = vgetq_lane_s32(_sum0, 3); + output0_tm += 1; + output1_tm += 1; + output2_tm += 1; + output3_tm += 1; + } + } + } + + remain_outch_start += nn_outch << 2; +#else // __ARM_NEON + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + int* output0_tm = top_blob_tm.channel(p); + int* output1_tm = top_blob_tm.channel(p + 1); + + const Mat kernel0_tm = kernel_tm.channel(p / 2); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 1 < tiles; i += 2) + { + const short* r0 = bb2.row(i / 2); + const short* k0 = kernel0_tm.row(r); + + int sum00 = 0; + int sum10 = 0; + int sum01 = 0; + int sum11 = 0; + + int j = 0; +#if __ARM_FEATURE_SIMD32 + for (; j + 1 < inch; j += 2) + { + // fomit-frame-pointer implied in optimized flag spare one register + // let us stay away from error: ‘asm’ operand has impossible constraints --- nihui +#if __OPTIMIZE__ + asm volatile( + "ldr r2, [%0], #4 \n" // int16x2_t _val02 = *((int16x2_t*)r0); r0 += 2; + "ldr r3, [%0], #4 \n" // int16x2_t _val13 = *((int16x2_t*)r0); r0 += 2; + "ldr r4, [%1], #4 \n" // int16x2_t _w02 = *((int16x2_t*)k0); k0 += 2; + "ldr r5, [%1], #4 \n" // int16x2_t _w13 = *((int16x2_t*)k0); k0 += 2; + "smlad %2, r2, r4, %2 \n" // sum00 = __smlad(_val02, _w02, sum00); + "smlad %3, r3, r4, %3 \n" // sum01 = __smlad(_val13, _w02, sum01); + "smlad %4, r2, r5, %4 \n" // sum10 = __smlad(_val02, _w13, sum10); + "smlad %5, r3, r5, %5 \n" // sum11 = __smlad(_val13, _w13, sum11); + : "=r"(r0), + "=r"(k0), + "=r"(sum00), + "=r"(sum01), + "=r"(sum10), + "=r"(sum11) + : "0"(r0), + "1"(k0), + "2"(sum00), + "3"(sum01), + "4"(sum10), + "5"(sum11) + : "memory", "r2", "r3", "r4", "r5"); +#else + int _val02 = *((int*)r0); + int _val13 = *((int*)(r0 + 2)); + int _w02 = *((int*)k0); + int _w13 = *((int*)(k0 + 2)); + asm volatile("smlad %0, %2, %3, %0" + : "=r"(sum00) + : "0"(sum00), "r"(_val02), "r"(_w02) + :); + asm volatile("smlad %0, %2, %3, %0" + : "=r"(sum01) + : "0"(sum01), "r"(_val13), "r"(_w02) + :); + asm volatile("smlad %0, %2, %3, %0" + : "=r"(sum10) + : "0"(sum10), "r"(_val02), "r"(_w13) + :); + asm volatile("smlad %0, %2, %3, %0" + : "=r"(sum11) + : "0"(sum11), "r"(_val13), "r"(_w13) + :); + r0 += 4; + k0 += 4; +#endif + } +#endif // __ARM_FEATURE_SIMD32 + for (; j < inch; j++) + { + signed short val0 = r0[0]; + signed short val1 = r0[1]; + + signed short w0 = k0[0]; + signed short w1 = k0[1]; + + sum00 += val0 * w0; + sum10 += val0 * w1; + sum01 += val1 * w0; + sum11 += val1 * w1; + + r0 += 2; + k0 += 2; + } + + output0_tm[0] = sum00; + output1_tm[0] = sum10; + output0_tm[1] = sum01; + output1_tm[1] = sum11; + output0_tm += 2; + output1_tm += 2; + } + for (; i < tiles; i++) + { + const short* r0 = bb2.row(i / 2 + i % 2); + const short* k0 = kernel0_tm.row(r); + + int sum0 = 0; + int sum1 = 0; + + int j = 0; +#if __ARM_FEATURE_SIMD32 + for (; j + 1 < inch; j += 2) + { + asm volatile( + "ldr r2, [%0], #4 \n" // int16x2_t _val01 = *((int16x2_t*)r0); r0 += 2; + "ldr r3, [%1], #4 \n" // int16x2_t _w02 = *((int16x2_t*)k0); k0 += 2; + "ldr r4, [%1], #4 \n" // int16x2_t _w13 = *((int16x2_t*)k0); k0 += 2; + "smlad %2, r2, r3, %2 \n" // sum00 = __smlad(_val01, _w02, sum00); + "smlad %3, r2, r4, %3 \n" // sum01 = __smlad(_val01, _w02, sum01); + : "=r"(r0), + "=r"(k0), + "=r"(sum0), + "=r"(sum1) + : "0"(r0), + "1"(k0), + "2"(sum0), + "3"(sum1) + : "memory", "r2", "r3", "r4"); + } +#endif // __ARM_FEATURE_SIMD32 + for (; j < inch; j++) + { + signed short val = r0[0]; + + sum0 += val * k0[0]; + sum1 += val * k0[1]; + + r0 += 1; + k0 += 2; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output0_tm += 1; + output1_tm += 1; + } + } + } +#endif // __ARM_NEON + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + int* output0_tm = top_blob_tm.channel(p); + +#if __ARM_NEON + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); +#endif + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __ARM_NEON +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 8); + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + int j = 0; + for (; j + 3 < inch; j += 4) + { + int16x8_t _val01 = vld1q_s16(r0); + int16x8_t _val23 = vld1q_s16(r0 + 8); + int16x8_t _val45 = vld1q_s16(r0 + 16); + int16x8_t _val67 = vld1q_s16(r0 + 24); + int16x4_t _w0123 = vld1_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val01), _w0123, 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val01), _w0123, 0); + + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val23), _w0123, 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val23), _w0123, 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val45), _w0123, 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val45), _w0123, 2); + + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val67), _w0123, 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val67), _w0123, 3); + + k0 += 4; + r0 += 32; + } + _sum0 = vaddq_s32(_sum0, _sum2); + _sum1 = vaddq_s32(_sum1, _sum3); + for (; j < inch; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x4_t _w0 = vld1_dup_s16(k0); + + _sum0 = vmlal_s16(_sum0, _w0, vget_low_s16(_val0)); + _sum1 = vmlal_s16(_sum1, _w0, vget_high_s16(_val0)); + + k0 += 1; + r0 += 8; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output0_tm + 4, _sum1); + output0_tm += 8; + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + int j = 0; + for (; j + 3 < inch; j += 4) + { + int16x8_t _val01 = vld1q_s16(r0); + int16x8_t _val23 = vld1q_s16(r0 + 8); + int16x4_t _w0123 = vld1_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val01), _w0123, 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val01), _w0123, 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val23), _w0123, 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val23), _w0123, 3); + + k0 += 4; + r0 += 16; + } + _sum0 = vaddq_s32(_sum0, _sum1); + _sum2 = vaddq_s32(_sum2, _sum3); + _sum0 = vaddq_s32(_sum0, _sum2); + for (; j < inch; j++) + { + int16x4_t _val0 = vld1_s16(r0); + int16x4_t _w0 = vld1_dup_s16(k0); + + _sum0 = vmlal_s16(_sum0, _val0, _w0); + + k0 += 1; + r0 += 4; + } + + vst1q_s32(output0_tm, _sum0); + output0_tm += 4; + } +#else + for (; i + 1 < tiles; i += 2) + { + const short* r0 = bb2.row(i / 2); + const short* k0 = kernel0_tm.row(r); + + int sum0 = 0; + int sum1 = 0; + + int j = 0; +#if __ARM_FEATURE_SIMD32 + for (; j + 1 < inch; j += 2) + { + asm volatile( + "ldr r2, [%0], #4 \n" // int16x2_t _val02 = *((int16x2_t*)r0); r0 += 2; + "ldr r3, [%0], #4 \n" // int16x2_t _val13 = *((int16x2_t*)r0); r0 += 2; + "ldr r4, [%1], #4 \n" // int16x2_t _w01 = *((int16x2_t*)k0); k0 += 2; + "smlad %2, r2, r4, %2 \n" // sum00 = __smlad(_val02, _w01, sum00); + "smlad %3, r3, r4, %3 \n" // sum01 = __smlad(_val13, _w01, sum01); + : "=r"(r0), + "=r"(k0), + "=r"(sum0), + "=r"(sum1) + : "0"(r0), + "1"(k0), + "2"(sum0), + "3"(sum1) + : "memory", "r2", "r3", "r4"); + } +#endif // __ARM_FEATURE_SIMD32 + for (; j < inch; j++) + { + signed short val0 = r0[0]; + signed short val1 = r0[1]; + signed short w = k0[0]; + + sum0 += val0 * w; + sum1 += val1 * w; + + k0 += 1; + r0 += 2; + } + + output0_tm[0] = sum0; + output0_tm[1] = sum1; + output0_tm += 2; + } +#endif + for (; i < tiles; i++) + { +#if __ARM_NEON +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + const short* r0 = bb2.row(i / 4 + i % 4); +#endif +#else + const short* r0 = bb2.row(i / 2 + i % 2); +#endif + const short* k0 = kernel0_tm.row(r); + + int sum = 0; + + int j = 0; +#if __ARM_NEON + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + for (; j + 7 < inch; j += 8) + { + int16x8_t _val = vld1q_s16(r0); + int16x8_t _w = vld1q_s16(k0); + + _sum0 = vmlal_s16(_sum0, vget_low_s16(_val), vget_low_s16(_w)); + _sum1 = vmlal_s16(_sum1, vget_high_s16(_val), vget_high_s16(_w)); + + k0 += 8; + r0 += 8; + } + _sum0 = vaddq_s32(_sum0, _sum1); + for (; j + 3 < inch; j += 4) + { + int16x4_t _val = vld1_s16(r0); + int16x4_t _w = vld1_s16(k0); + + _sum0 = vmlal_s16(_sum0, _val, _w); + + k0 += 4; + r0 += 4; + } +#if __aarch64__ + sum = vaddvq_s32(_sum0); +#else + int32x2_t _ss = vadd_s32(vget_low_s32(_sum0), vget_high_s32(_sum0)); + _ss = vpadd_s32(_ss, _ss); + + sum = vget_lane_s32(_ss, 0); +#endif +#endif // __ARM_NEON +#if __ARM_FEATURE_SIMD32 + for (; j + 1 < inch; j += 2) + { + asm volatile( + "ldr r2, [%0], #4 \n" // int16x2_t _val = *((int16x2_t*)r0); r0 += 2; + "ldr r3, [%1], #4 \n" // int16x2_t _w = *((int16x2_t*)k0); k0 += 2; + "smlad %2, r2, r3, %2 \n" // sum = __smlad(_val, _w, sum); + : "=r"(r0), + "=r"(k0), + "=r"(sum) + : "0"(r0), + "1"(k0), + "2"(sum) + : "memory", "r2", "r3"); + } +#endif // __ARM_FEATURE_SIMD32 + for (; j < inch; j++) + { + signed short val = r0[0]; + signed short w = k0[0]; + + sum += val * w; + + k0 += 1; + r0 += 1; + } + + output0_tm[0] = sum; + output0_tm++; + } + } + } +} diff --git a/src/layer/arm/convolution_winograd_dot_pack8to1_int8.h b/src/layer/arm/convolution_winograd_dot_pack8to1_int8.h new file mode 100644 index 000000000..6192be128 --- /dev/null +++ b/src/layer/arm/convolution_winograd_dot_pack8to1_int8.h @@ -0,0 +1,774 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_winograd_dot_pack8to1_int8_neon(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt) +{ + // Mat bottom_blob_tm(tiles, 16/36/64, inch, 16u, 8, opt.workspace_allocator); + + const int tiles = bottom_blob_tm.w; + const int batch = bottom_blob_tm.h; + const int inch = bottom_blob_tm.c; + + // permute + Mat bottom_blob_tm2; +#if __aarch64__ + if (tiles >= 8) + bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator); +#else + if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator); +#endif // __aarch64__ + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < batch; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + short* tm2p = tm2.row(i / 8); + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { + // transpose 8x8 + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" + "sub %0, %0, #64 \n" + + "uzp1 v16.8h, v0.8h, v4.8h \n" + "uzp2 v20.8h, v0.8h, v4.8h \n" + "uzp1 v17.8h, v1.8h, v5.8h \n" + "uzp2 v21.8h, v1.8h, v5.8h \n" + "uzp1 v18.8h, v2.8h, v6.8h \n" + "uzp2 v22.8h, v2.8h, v6.8h \n" + "uzp1 v19.8h, v3.8h, v7.8h \n" + "uzp2 v23.8h, v3.8h, v7.8h \n" + + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + + r0 += bottom_blob_tm.cstep * 8; + } + } +#endif + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + short* tm2p = tm2.row(i / 8 + (i % 8) / 4); +#else + short* tm2p = tm2.row(i / 4); +#endif + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { + // transpose 8x4 +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" + "st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "v0", "v1", "v2", "v3"); +#else + asm volatile( + "pld [%0, #512] \n" + "vldm %0, {d0-d7} \n" + "vswp d1, d2 \n" + "vswp d5, d6 \n" + "vswp q1, q2 \n" + "vst4.s16 {d0-d3}, [%1 :64]! \n" + "vst4.s16 {d4-d7}, [%1 :64]! \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "q0", "q1", "q2", "q3"); +#endif // __aarch64__ + r0 += bottom_blob_tm.cstep * 8; + } + } + for (; i < tiles; i++) + { +#if __aarch64__ + short* tm2p = tm2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + short* tm2p = tm2.row(i / 4 + i % 4); +#endif + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.8h}, [%0] \n" + "st1 {v0.8h}, [%1], #16 \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "v0"); +#else + asm volatile( + "pld [%0, #128] \n" + "vld1.s16 {d0-d1}, [%0 :64] \n" + "vst1.s16 {d0-d1}, [%1 :64]! \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "q0"); +#endif // __aarch64__ + r0 += bottom_blob_tm.cstep * 8; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, batch, outch, 4u, 1, opt.workspace_allocator); + + int nn_outch = 0; + int remain_outch_start = 0; + + nn_outch = outch >> 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 8; + + int* output0_tm = top_blob_tm.channel(p); + int* output1_tm = top_blob_tm.channel(p + 1); + int* output2_tm = top_blob_tm.channel(p + 2); + int* output3_tm = top_blob_tm.channel(p + 3); + int* output4_tm = top_blob_tm.channel(p + 4); + int* output5_tm = top_blob_tm.channel(p + 5); + int* output6_tm = top_blob_tm.channel(p + 6); + int* output7_tm = top_blob_tm.channel(p + 7); + + const Mat kernel01_tm = kernel_tm.channel(p / 8); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 8); + const short* kptr = kernel01_tm.row(r); + + int nn = inch; // inch always > 0 + + asm volatile( + "eor v16.16b, v16.16b, v16.16b \n" + "eor v17.16b, v17.16b, v17.16b \n" + "eor v18.16b, v18.16b, v18.16b \n" + "eor v19.16b, v19.16b, v19.16b \n" + "eor v20.16b, v20.16b, v20.16b \n" + "eor v21.16b, v21.16b, v21.16b \n" + "eor v22.16b, v22.16b, v22.16b \n" + "eor v23.16b, v23.16b, v23.16b \n" + "eor v24.16b, v24.16b, v24.16b \n" + "eor v25.16b, v25.16b, v25.16b \n" + "eor v26.16b, v26.16b, v26.16b \n" + "eor v27.16b, v27.16b, v27.16b \n" + "eor v28.16b, v28.16b, v28.16b \n" + "eor v29.16b, v29.16b, v29.16b \n" + "eor v30.16b, v30.16b, v30.16b \n" + "eor v31.16b, v31.16b, v31.16b \n" + + "0: \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%9], #64 \n" + + "prfm pldl1keep, [%10, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%10], #64 \n" + + "smlal v16.4s, v8.4h, v0.h[0] \n" + "smlal2 v17.4s, v8.8h, v0.h[0] \n" + "smlal v18.4s, v8.4h, v0.h[1] \n" + "smlal2 v19.4s, v8.8h, v0.h[1] \n" + "smlal v20.4s, v8.4h, v0.h[2] \n" + "smlal2 v21.4s, v8.8h, v0.h[2] \n" + "smlal v22.4s, v8.4h, v0.h[3] \n" + "smlal2 v23.4s, v8.8h, v0.h[3] \n" + "smlal v24.4s, v8.4h, v0.h[4] \n" + "smlal2 v25.4s, v8.8h, v0.h[4] \n" + "smlal v26.4s, v8.4h, v0.h[5] \n" + "smlal2 v27.4s, v8.8h, v0.h[5] \n" + "smlal v28.4s, v8.4h, v0.h[6] \n" + "smlal2 v29.4s, v8.8h, v0.h[6] \n" + "smlal v30.4s, v8.4h, v0.h[7] \n" + "smlal2 v31.4s, v8.8h, v0.h[7] \n" + + "smlal v16.4s, v9.4h, v1.h[0] \n" + "smlal2 v17.4s, v9.8h, v1.h[0] \n" + "smlal v18.4s, v9.4h, v1.h[1] \n" + "smlal2 v19.4s, v9.8h, v1.h[1] \n" + "smlal v20.4s, v9.4h, v1.h[2] \n" + "smlal2 v21.4s, v9.8h, v1.h[2] \n" + "smlal v22.4s, v9.4h, v1.h[3] \n" + "smlal2 v23.4s, v9.8h, v1.h[3] \n" + "smlal v24.4s, v9.4h, v1.h[4] \n" + "smlal2 v25.4s, v9.8h, v1.h[4] \n" + "smlal v26.4s, v9.4h, v1.h[5] \n" + "smlal2 v27.4s, v9.8h, v1.h[5] \n" + "smlal v28.4s, v9.4h, v1.h[6] \n" + "smlal2 v29.4s, v9.8h, v1.h[6] \n" + "smlal v30.4s, v9.4h, v1.h[7] \n" + "smlal2 v31.4s, v9.8h, v1.h[7] \n" + + "prfm pldl1keep, [%9, #512] \n" + "ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%9], #64 \n" + + "smlal v16.4s, v10.4h, v2.h[0] \n" + "smlal2 v17.4s, v10.8h, v2.h[0] \n" + "smlal v18.4s, v10.4h, v2.h[1] \n" + "smlal2 v19.4s, v10.8h, v2.h[1] \n" + "smlal v20.4s, v10.4h, v2.h[2] \n" + "smlal2 v21.4s, v10.8h, v2.h[2] \n" + "smlal v22.4s, v10.4h, v2.h[3] \n" + "smlal2 v23.4s, v10.8h, v2.h[3] \n" + "smlal v24.4s, v10.4h, v2.h[4] \n" + "smlal2 v25.4s, v10.8h, v2.h[4] \n" + "smlal v26.4s, v10.4h, v2.h[5] \n" + "smlal2 v27.4s, v10.8h, v2.h[5] \n" + "smlal v28.4s, v10.4h, v2.h[6] \n" + "smlal2 v29.4s, v10.8h, v2.h[6] \n" + "smlal v30.4s, v10.4h, v2.h[7] \n" + "smlal2 v31.4s, v10.8h, v2.h[7] \n" + + "prfm pldl1keep, [%10, #512] \n" + "ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%10], #64 \n" + + "smlal v16.4s, v11.4h, v3.h[0] \n" + "smlal2 v17.4s, v11.8h, v3.h[0] \n" + "smlal v18.4s, v11.4h, v3.h[1] \n" + "smlal2 v19.4s, v11.8h, v3.h[1] \n" + "smlal v20.4s, v11.4h, v3.h[2] \n" + "smlal2 v21.4s, v11.8h, v3.h[2] \n" + "smlal v22.4s, v11.4h, v3.h[3] \n" + "smlal2 v23.4s, v11.8h, v3.h[3] \n" + "smlal v24.4s, v11.4h, v3.h[4] \n" + "smlal2 v25.4s, v11.8h, v3.h[4] \n" + "smlal v26.4s, v11.4h, v3.h[5] \n" + "smlal2 v27.4s, v11.8h, v3.h[5] \n" + "smlal v28.4s, v11.4h, v3.h[6] \n" + "smlal2 v29.4s, v11.8h, v3.h[6] \n" + "smlal v30.4s, v11.4h, v3.h[7] \n" + "smlal2 v31.4s, v11.8h, v3.h[7] \n" + + "smlal v16.4s, v12.4h, v4.h[0] \n" + "smlal2 v17.4s, v12.8h, v4.h[0] \n" + "smlal v18.4s, v12.4h, v4.h[1] \n" + "smlal2 v19.4s, v12.8h, v4.h[1] \n" + "smlal v20.4s, v12.4h, v4.h[2] \n" + "smlal2 v21.4s, v12.8h, v4.h[2] \n" + "smlal v22.4s, v12.4h, v4.h[3] \n" + "smlal2 v23.4s, v12.8h, v4.h[3] \n" + "smlal v24.4s, v12.4h, v4.h[4] \n" + "smlal2 v25.4s, v12.8h, v4.h[4] \n" + "smlal v26.4s, v12.4h, v4.h[5] \n" + "smlal2 v27.4s, v12.8h, v4.h[5] \n" + "smlal v28.4s, v12.4h, v4.h[6] \n" + "smlal2 v29.4s, v12.8h, v4.h[6] \n" + "smlal v30.4s, v12.4h, v4.h[7] \n" + "smlal2 v31.4s, v12.8h, v4.h[7] \n" + + "smlal v16.4s, v13.4h, v5.h[0] \n" + "smlal2 v17.4s, v13.8h, v5.h[0] \n" + "smlal v18.4s, v13.4h, v5.h[1] \n" + "smlal2 v19.4s, v13.8h, v5.h[1] \n" + "smlal v20.4s, v13.4h, v5.h[2] \n" + "smlal2 v21.4s, v13.8h, v5.h[2] \n" + "smlal v22.4s, v13.4h, v5.h[3] \n" + "smlal2 v23.4s, v13.8h, v5.h[3] \n" + "smlal v24.4s, v13.4h, v5.h[4] \n" + "smlal2 v25.4s, v13.8h, v5.h[4] \n" + "smlal v26.4s, v13.4h, v5.h[5] \n" + "smlal2 v27.4s, v13.8h, v5.h[5] \n" + "smlal v28.4s, v13.4h, v5.h[6] \n" + "smlal2 v29.4s, v13.8h, v5.h[6] \n" + "smlal v30.4s, v13.4h, v5.h[7] \n" + "smlal2 v31.4s, v13.8h, v5.h[7] \n" + + "smlal v16.4s, v14.4h, v6.h[0] \n" + "smlal2 v17.4s, v14.8h, v6.h[0] \n" + "smlal v18.4s, v14.4h, v6.h[1] \n" + "smlal2 v19.4s, v14.8h, v6.h[1] \n" + "smlal v20.4s, v14.4h, v6.h[2] \n" + "smlal2 v21.4s, v14.8h, v6.h[2] \n" + "smlal v22.4s, v14.4h, v6.h[3] \n" + "smlal2 v23.4s, v14.8h, v6.h[3] \n" + "smlal v24.4s, v14.4h, v6.h[4] \n" + "smlal2 v25.4s, v14.8h, v6.h[4] \n" + "smlal v26.4s, v14.4h, v6.h[5] \n" + "smlal2 v27.4s, v14.8h, v6.h[5] \n" + "smlal v28.4s, v14.4h, v6.h[6] \n" + "smlal2 v29.4s, v14.8h, v6.h[6] \n" + "smlal v30.4s, v14.4h, v6.h[7] \n" + "smlal2 v31.4s, v14.8h, v6.h[7] \n" + + "subs %w0, %w0, #1 \n" + + "smlal v16.4s, v15.4h, v7.h[0] \n" + "smlal2 v17.4s, v15.8h, v7.h[0] \n" + "smlal v18.4s, v15.4h, v7.h[1] \n" + "smlal2 v19.4s, v15.8h, v7.h[1] \n" + "smlal v20.4s, v15.4h, v7.h[2] \n" + "smlal2 v21.4s, v15.8h, v7.h[2] \n" + "smlal v22.4s, v15.4h, v7.h[3] \n" + "smlal2 v23.4s, v15.8h, v7.h[3] \n" + "smlal v24.4s, v15.4h, v7.h[4] \n" + "smlal2 v25.4s, v15.8h, v7.h[4] \n" + "smlal v26.4s, v15.4h, v7.h[5] \n" + "smlal2 v27.4s, v15.8h, v7.h[5] \n" + "smlal v28.4s, v15.4h, v7.h[6] \n" + "smlal2 v29.4s, v15.8h, v7.h[6] \n" + "smlal v30.4s, v15.4h, v7.h[7] \n" + "smlal2 v31.4s, v15.8h, v7.h[7] \n" + + "bne 0b \n" + + "st1 {v16.4s, v17.4s}, [%1], #32 \n" + "st1 {v18.4s, v19.4s}, [%2], #32 \n" + "st1 {v20.4s, v21.4s}, [%3], #32 \n" + "st1 {v22.4s, v23.4s}, [%4], #32 \n" + "st1 {v24.4s, v25.4s}, [%5], #32 \n" + "st1 {v26.4s, v27.4s}, [%6], #32 \n" + "st1 {v28.4s, v29.4s}, [%7], #32 \n" + "st1 {v30.4s, v31.4s}, [%8], #32 \n" + + : "=r"(nn), // %0 + "=r"(output0_tm), // %1 + "=r"(output1_tm), // %2 + "=r"(output2_tm), // %3 + "=r"(output3_tm), // %4 + "=r"(output4_tm), // %5 + "=r"(output5_tm), // %6 + "=r"(output6_tm), // %7 + "=r"(output7_tm), // %8 + "=r"(r0), // %9 + "=r"(kptr) // %10 + : "0"(nn), + "1"(output0_tm), + "2"(output1_tm), + "3"(output2_tm), + "4"(output3_tm), + "5"(output4_tm), + "6"(output5_tm), + "7"(output6_tm), + "8"(output7_tm), + "9"(r0), + "10"(kptr) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } +#endif + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel01_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + int16x8_t _val2 = vld1q_s16(r0 + 16); + int16x8_t _val3 = vld1q_s16(r0 + 24); + + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val0), vget_low_s16(_w0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val0), vget_low_s16(_w0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val0), vget_low_s16(_w0), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val0), vget_low_s16(_w0), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val0), vget_high_s16(_w0), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val0), vget_high_s16(_w0), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val0), vget_high_s16(_w0), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val0), vget_high_s16(_w0), 3); + + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val0), vget_low_s16(_w1), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val0), vget_low_s16(_w1), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val0), vget_low_s16(_w1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val0), vget_low_s16(_w1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val0), vget_high_s16(_w1), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val0), vget_high_s16(_w1), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val0), vget_high_s16(_w1), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val0), vget_high_s16(_w1), 3); + + int16x8_t _w2 = vld1q_s16(k0 + 16); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val1), vget_low_s16(_w2), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val1), vget_low_s16(_w2), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val1), vget_low_s16(_w2), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val1), vget_low_s16(_w2), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val1), vget_high_s16(_w2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val1), vget_high_s16(_w2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val1), vget_high_s16(_w2), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val1), vget_high_s16(_w2), 3); + + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val1), vget_low_s16(_w3), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val1), vget_low_s16(_w3), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val1), vget_low_s16(_w3), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val1), vget_low_s16(_w3), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val1), vget_high_s16(_w3), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val1), vget_high_s16(_w3), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val1), vget_high_s16(_w3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val1), vget_high_s16(_w3), 3); + + int16x8_t _w4 = vld1q_s16(k0 + 32); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val2), vget_low_s16(_w4), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val2), vget_low_s16(_w4), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val2), vget_low_s16(_w4), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val2), vget_low_s16(_w4), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val2), vget_high_s16(_w4), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val2), vget_high_s16(_w4), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val2), vget_high_s16(_w4), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val2), vget_high_s16(_w4), 3); + + int16x8_t _w5 = vld1q_s16(k0 + 40); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val2), vget_low_s16(_w5), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val2), vget_low_s16(_w5), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val2), vget_low_s16(_w5), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val2), vget_low_s16(_w5), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val2), vget_high_s16(_w5), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val2), vget_high_s16(_w5), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val2), vget_high_s16(_w5), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val2), vget_high_s16(_w5), 3); + + int16x8_t _w6 = vld1q_s16(k0 + 48); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val3), vget_low_s16(_w6), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val3), vget_low_s16(_w6), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val3), vget_low_s16(_w6), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val3), vget_low_s16(_w6), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val3), vget_high_s16(_w6), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val3), vget_high_s16(_w6), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val3), vget_high_s16(_w6), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val3), vget_high_s16(_w6), 3); + + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val3), vget_low_s16(_w7), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val3), vget_low_s16(_w7), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val3), vget_low_s16(_w7), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val3), vget_low_s16(_w7), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val3), vget_high_s16(_w7), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val3), vget_high_s16(_w7), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val3), vget_high_s16(_w7), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val3), vget_high_s16(_w7), 3); + + r0 += 32; + k0 += 64; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output2_tm, _sum2); + vst1q_s32(output3_tm, _sum3); + vst1q_s32(output4_tm, _sum4); + vst1q_s32(output5_tm, _sum5); + vst1q_s32(output6_tm, _sum6); + vst1q_s32(output7_tm, _sum7); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + output4_tm += 4; + output5_tm += 4; + output6_tm += 4; + output7_tm += 4; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + const short* r0 = bb2.row(i / 4 + i % 4); +#endif + const short* k0 = kernel01_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + int16x8_t _w4 = vld1q_s16(k0 + 32); + int16x8_t _w5 = vld1q_s16(k0 + 40); + int16x8_t _w6 = vld1q_s16(k0 + 48); + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); + + r0 += 8; + k0 += 64; + } + + output0_tm[0] = vgetq_lane_s32(_sum0, 0); + output1_tm[0] = vgetq_lane_s32(_sum0, 1); + output2_tm[0] = vgetq_lane_s32(_sum0, 2); + output3_tm[0] = vgetq_lane_s32(_sum0, 3); + output4_tm[0] = vgetq_lane_s32(_sum1, 0); + output5_tm[0] = vgetq_lane_s32(_sum1, 1); + output6_tm[0] = vgetq_lane_s32(_sum1, 2); + output7_tm[0] = vgetq_lane_s32(_sum1, 3); + output0_tm += 1; + output1_tm += 1; + output2_tm += 1; + output3_tm += 1; + output4_tm += 1; + output5_tm += 1; + output6_tm += 1; + output7_tm += 1; + } + } + } + + remain_outch_start += nn_outch << 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + int* output0_tm = top_blob_tm.channel(p); + + const Mat kernel0_tm = kernel_tm.channel(p / 8 + p % 8); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 8); + + const short* kptr = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + for (int q = 0; q < inch; q++) + { + int16x8_t _r0 = vld1q_s16(r0); + int16x8_t _r1 = vld1q_s16(r0 + 8); + int16x8_t _r2 = vld1q_s16(r0 + 16); + int16x8_t _r3 = vld1q_s16(r0 + 24); + int16x8_t _r4 = vld1q_s16(r0 + 32); + int16x8_t _r5 = vld1q_s16(r0 + 40); + int16x8_t _r6 = vld1q_s16(r0 + 48); + int16x8_t _r7 = vld1q_s16(r0 + 56); + + int16x8_t _k0 = vld1q_s16(kptr); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r1), vget_low_s16(_k0), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r1), vget_low_s16(_k0), 1); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_low_s16(_k0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_low_s16(_k0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r3), vget_low_s16(_k0), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r3), vget_low_s16(_k0), 3); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r4), vget_high_s16(_k0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r4), vget_high_s16(_k0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r5), vget_high_s16(_k0), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r5), vget_high_s16(_k0), 1); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r6), vget_high_s16(_k0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r6), vget_high_s16(_k0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r7), vget_high_s16(_k0), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r7), vget_high_s16(_k0), 3); + + kptr += 8; + r0 += 64; + } + + _sum0 = vaddq_s32(_sum0, _sum2); + _sum1 = vaddq_s32(_sum1, _sum3); + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output0_tm + 4, _sum1); + + output0_tm += 8; + } +#endif + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* kptr = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + for (int q = 0; q < inch; q++) + { + int16x8_t _r0 = vld1q_s16(r0); + int16x8_t _r1 = vld1q_s16(r0 + 8); + int16x8_t _r2 = vld1q_s16(r0 + 16); + int16x8_t _r3 = vld1q_s16(r0 + 24); + + int16x8_t _k0 = vld1q_s16(kptr); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 1); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r1), vget_low_s16(_k0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r1), vget_low_s16(_k0), 3); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_high_s16(_k0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_high_s16(_k0), 1); + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r3), vget_high_s16(_k0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r3), vget_high_s16(_k0), 3); + + kptr += 8; + r0 += 32; + } + + int32x4_t _sum01 = vaddq_s32(_sum0, _sum1); + + vst1q_s32(output0_tm, _sum01); + + output0_tm += 4; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 8 + (i % 8) / 4 + i % 4); +#else + const short* r0 = bb2.row(i / 4 + i % 4); +#endif + const short* kptr = kernel0_tm.row(r); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + for (int q = 0; q < inch; q++) + { + int16x8_t _r0 = vld1q_s16(r0); + + int16x8_t _k0 = vld1q_s16(kptr); + + _sum0 = vmlal_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0)); + _sum1 = vmlal_s16(_sum1, vget_high_s16(_r0), vget_high_s16(_k0)); + + kptr += 8; + r0 += 8; + } + + int32x4_t _sum = vaddq_s32(_sum0, _sum1); +#if __aarch64__ + int sum = vaddvq_s32(_sum); // dot +#else + int32x2_t _ss = vadd_s32(vget_low_s32(_sum), vget_high_s32(_sum)); + _ss = vpadd_s32(_ss, _ss); + int sum = vget_lane_s32(_ss, 0); +#endif + + output0_tm[0] = sum; + + output0_tm++; + } + } + } +} diff --git a/src/layer/arm/convolution_winograd_dot_pack8to4_int8.h b/src/layer/arm/convolution_winograd_dot_pack8to4_int8.h new file mode 100644 index 000000000..a17559f6c --- /dev/null +++ b/src/layer/arm/convolution_winograd_dot_pack8to4_int8.h @@ -0,0 +1,1835 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_winograd_dot_pack8to4_int8_neon(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt) +{ + // Mat bottom_blob_tm(tiles, 16/36/64, inch, 16u, 8, opt.workspace_allocator); + + const int tiles = bottom_blob_tm.w; + const int batch = bottom_blob_tm.h; + const int inch = bottom_blob_tm.c; + + // permute + Mat bottom_blob_tm2; +#if __aarch64__ + if (tiles >= 12) + bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 8) + bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 2) + bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, batch, 16u, 8, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator); +#else + if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 8, opt.workspace_allocator); + else if (tiles >= 2) + bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, batch, 16u, 8, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator); +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < batch; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; +#if __aarch64__ + for (; i + 11 < tiles; i += 12) + { + short* tm2p = tm2.row(i / 12); + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { + // transpose 12x8 + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n" + "ld4 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n" + + "sub %0, %0, #128 \n" + + "uzp1 v20.8h, v0.8h, v4.8h \n" // 0 + "uzp1 v21.8h, v16.8h, v1.8h \n" // 1 + "uzp1 v22.8h, v5.8h, v17.8h \n" // 2 + "uzp1 v23.8h, v2.8h, v6.8h \n" // 3 + "uzp1 v24.8h, v18.8h, v3.8h \n" // 4 + "uzp1 v25.8h, v7.8h, v19.8h \n" // 5 + "uzp2 v26.8h, v0.8h, v4.8h \n" // 6 + "uzp2 v27.8h, v16.8h, v1.8h \n" // 7 + "uzp2 v28.8h, v5.8h, v17.8h \n" // 8 + "uzp2 v29.8h, v2.8h, v6.8h \n" // 9 + "uzp2 v30.8h, v18.8h, v3.8h \n" // 10 + "uzp2 v31.8h, v7.8h, v19.8h \n" // 11 + + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + "st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n" + "st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n" + : "=r"(r0), // %0 + "=r"(tm2p) // %1 + : "0"(r0), + "1"(tm2p) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + + r0 += bottom_blob_tm.cstep * 8; + } + } + for (; i + 7 < tiles; i += 8) + { + short* tmpptr = tm2.row(i / 12 + (i % 12) / 8); + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { + // transpose 8x8 + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n" + "ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n" + "sub %0, %0, #64 \n" + + "uzp1 v16.8h, v0.8h, v4.8h \n" + "uzp2 v20.8h, v0.8h, v4.8h \n" + "uzp1 v17.8h, v1.8h, v5.8h \n" + "uzp2 v21.8h, v1.8h, v5.8h \n" + "uzp1 v18.8h, v2.8h, v6.8h \n" + "uzp2 v22.8h, v2.8h, v6.8h \n" + "uzp1 v19.8h, v3.8h, v7.8h \n" + "uzp2 v23.8h, v3.8h, v7.8h \n" + + "st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n" + "st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + + r0 += bottom_blob_tm.cstep * 8; + } + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); +#else + short* tmpptr = tm2.row(i / 4); +#endif + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #512] \n" + "ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n" + "st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "v0", "v1", "v2", "v3"); +#else + asm volatile( + "pld [%0, #512] \n" + "vldm %0, {d0-d7} \n" + "vstm %1!, {d0-d7} \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "q0", "q1", "q2", "q3"); +#endif + r0 += bottom_blob_tm.cstep * 8; + } + } + for (; i + 1 < tiles; i += 2) + { +#if __aarch64__ + short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); +#else + short* tmpptr = tm2.row(i / 4 + (i % 4) / 2); +#endif + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.8h, v1.8h}, [%0] \n" + "st1 {v0.8h, v1.8h}, [%1], #32 \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "v0", "v1"); +#else + asm volatile( + "pld [%0, #256] \n" + "vld1.s16 {d0-d3}, [%0 :128] \n" + "vst1.s16 {d0-d3}, [%1 :128]! \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "q0", "q1"); +#endif + r0 += bottom_blob_tm.cstep * 8; + } + } + for (; i < tiles; i++) + { +#if __aarch64__ + short* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); +#else + short* tmpptr = tm2.row(i / 4 + (i % 4) / 2 + i % 2); +#endif + + const short* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 8; + + for (int q = 0; q < inch; q++) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #128] \n" + "ld1 {v0.8h}, [%0] \n" + "st1 {v0.8h}, [%1], #16 \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "v0"); +#else + asm volatile( + "pld [%0, #128] \n" + "vld1.s16 {d0-d1}, [%0 :128] \n" + "vst1.s16 {d0-d1}, [%1 :128]! \n" + : "=r"(r0), // %0 + "=r"(tmpptr) // %1 + : "0"(r0), + "1"(tmpptr) + : "memory", "q0"); +#endif + r0 += bottom_blob_tm.cstep * 8; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, batch, outch, 16u, 4, opt.workspace_allocator); + + int nn_outch = 0; + int remain_outch_start = 0; + + nn_outch = outch >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + int* output0_tm = top_blob_tm.channel(p); + int* output1_tm = top_blob_tm.channel(p + 1); + + const Mat kernel0_tm = kernel_tm.channel(p / 2); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 11 < tiles; i += 12) + { + const short* r0 = bb2.row(i / 12); + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + asm volatile( + "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r01 + + "eor v8.16b, v8.16b, v8.16b \n" + "eor v9.16b, v9.16b, v9.16b \n" + + "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w01 + + "eor v10.16b, v10.16b, v10.16b \n" + "eor v11.16b, v11.16b, v11.16b \n" + + "prfm pldl1keep, [%3, #256] \n" + + "eor v12.16b, v12.16b, v12.16b \n" + "eor v13.16b, v13.16b, v13.16b \n" + + "prfm pldl1keep, [%4, #256] \n" + + "eor v14.16b, v14.16b, v14.16b \n" + "eor v15.16b, v15.16b, v15.16b \n" + "eor v16.16b, v16.16b, v16.16b \n" + "eor v17.16b, v17.16b, v17.16b \n" + "eor v18.16b, v18.16b, v18.16b \n" + "eor v19.16b, v19.16b, v19.16b \n" + "eor v20.16b, v20.16b, v20.16b \n" + "eor v21.16b, v21.16b, v21.16b \n" + "eor v22.16b, v22.16b, v22.16b \n" + "eor v23.16b, v23.16b, v23.16b \n" + "eor v24.16b, v24.16b, v24.16b \n" + "eor v25.16b, v25.16b, v25.16b \n" + "eor v26.16b, v26.16b, v26.16b \n" + "eor v27.16b, v27.16b, v27.16b \n" + "eor v28.16b, v28.16b, v28.16b \n" + "eor v29.16b, v29.16b, v29.16b \n" + "eor v30.16b, v30.16b, v30.16b \n" + "eor v31.16b, v31.16b, v31.16b \n" + + "0: \n" + + "smlal v8.4s, v4.4h, v0.h[0] \n" + "smlal2 v20.4s, v4.8h, v0.h[0] \n" + "smlal v9.4s, v4.4h, v0.h[1] \n" + "smlal2 v21.4s, v4.8h, v0.h[1] \n" + "smlal v10.4s, v4.4h, v0.h[2] \n" + "smlal2 v22.4s, v4.8h, v0.h[2] \n" + "smlal v11.4s, v4.4h, v0.h[3] \n" + "smlal2 v23.4s, v4.8h, v0.h[3] \n" + "smlal v12.4s, v4.4h, v0.h[4] \n" + "smlal2 v24.4s, v4.8h, v0.h[4] \n" + "smlal v13.4s, v4.4h, v0.h[5] \n" + "smlal2 v25.4s, v4.8h, v0.h[5] \n" + "smlal v14.4s, v4.4h, v0.h[6] \n" + "smlal2 v26.4s, v4.8h, v0.h[6] \n" + "smlal v15.4s, v4.4h, v0.h[7] \n" + "smlal2 v27.4s, v4.8h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r23 + + "smlal v16.4s, v4.4h, v1.h[0] \n" + "smlal2 v28.4s, v4.8h, v1.h[0] \n" + "smlal v17.4s, v4.4h, v1.h[1] \n" + "smlal2 v29.4s, v4.8h, v1.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v18.4s, v4.4h, v1.h[2] \n" + "smlal2 v30.4s, v4.8h, v1.h[2] \n" + "smlal v19.4s, v4.4h, v1.h[3] \n" + "smlal2 v31.4s, v4.8h, v1.h[3] \n" + + "ld1 {v6.8h, v7.8h}, [%4], #32 \n" // w23 + + "smlal v8.4s, v5.4h, v1.h[4] \n" + "smlal2 v20.4s, v5.8h, v1.h[4] \n" + "smlal v9.4s, v5.4h, v1.h[5] \n" + "smlal2 v21.4s, v5.8h, v1.h[5] \n" + + "prfm pldl1keep, [%4, #256] \n" + + "smlal v10.4s, v5.4h, v1.h[6] \n" + "smlal2 v22.4s, v5.8h, v1.h[6] \n" + "smlal v11.4s, v5.4h, v1.h[7] \n" + "smlal2 v23.4s, v5.8h, v1.h[7] \n" + "smlal v12.4s, v5.4h, v2.h[0] \n" + "smlal2 v24.4s, v5.8h, v2.h[0] \n" + "smlal v13.4s, v5.4h, v2.h[1] \n" + "smlal2 v25.4s, v5.8h, v2.h[1] \n" + "smlal v14.4s, v5.4h, v2.h[2] \n" + "smlal2 v26.4s, v5.8h, v2.h[2] \n" + "smlal v15.4s, v5.4h, v2.h[3] \n" + "smlal2 v27.4s, v5.8h, v2.h[3] \n" + "smlal v16.4s, v5.4h, v2.h[4] \n" + "smlal2 v28.4s, v5.8h, v2.h[4] \n" + "smlal v17.4s, v5.4h, v2.h[5] \n" + "smlal2 v29.4s, v5.8h, v2.h[5] \n" + "smlal v18.4s, v5.4h, v2.h[6] \n" + "smlal2 v30.4s, v5.8h, v2.h[6] \n" + "smlal v19.4s, v5.4h, v2.h[7] \n" + "smlal2 v31.4s, v5.8h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r45 + + "smlal v8.4s, v6.4h, v3.h[0] \n" + "smlal2 v20.4s, v6.8h, v3.h[0] \n" + "smlal v9.4s, v6.4h, v3.h[1] \n" + "smlal2 v21.4s, v6.8h, v3.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v10.4s, v6.4h, v3.h[2] \n" + "smlal2 v22.4s, v6.8h, v3.h[2] \n" + "smlal v11.4s, v6.4h, v3.h[3] \n" + "smlal2 v23.4s, v6.8h, v3.h[3] \n" + "smlal v12.4s, v6.4h, v3.h[4] \n" + "smlal2 v24.4s, v6.8h, v3.h[4] \n" + "smlal v13.4s, v6.4h, v3.h[5] \n" + "smlal2 v25.4s, v6.8h, v3.h[5] \n" + "smlal v14.4s, v6.4h, v3.h[6] \n" + "smlal2 v26.4s, v6.8h, v3.h[6] \n" + "smlal v15.4s, v6.4h, v3.h[7] \n" + "smlal2 v27.4s, v6.8h, v3.h[7] \n" + + "smlal v16.4s, v6.4h, v0.h[0] \n" + "smlal2 v28.4s, v6.8h, v0.h[0] \n" + "smlal v17.4s, v6.4h, v0.h[1] \n" + "smlal2 v29.4s, v6.8h, v0.h[1] \n" + "smlal v18.4s, v6.4h, v0.h[2] \n" + "smlal2 v30.4s, v6.8h, v0.h[2] \n" + "smlal v19.4s, v6.4h, v0.h[3] \n" + "smlal2 v31.4s, v6.8h, v0.h[3] \n" + + "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w45 + + "smlal v8.4s, v7.4h, v0.h[4] \n" + "smlal2 v20.4s, v7.8h, v0.h[4] \n" + "smlal v9.4s, v7.4h, v0.h[5] \n" + "smlal2 v21.4s, v7.8h, v0.h[5] \n" + + "prfm pldl1keep, [%4, #256] \n" + + "smlal v10.4s, v7.4h, v0.h[6] \n" + "smlal2 v22.4s, v7.8h, v0.h[6] \n" + "smlal v11.4s, v7.4h, v0.h[7] \n" + "smlal2 v23.4s, v7.8h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r67 + + "smlal v12.4s, v7.4h, v1.h[0] \n" + "smlal2 v24.4s, v7.8h, v1.h[0] \n" + "smlal v13.4s, v7.4h, v1.h[1] \n" + "smlal2 v25.4s, v7.8h, v1.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v14.4s, v7.4h, v1.h[2] \n" + "smlal2 v26.4s, v7.8h, v1.h[2] \n" + "smlal v15.4s, v7.4h, v1.h[3] \n" + "smlal2 v27.4s, v7.8h, v1.h[3] \n" + "smlal v16.4s, v7.4h, v1.h[4] \n" + "smlal2 v28.4s, v7.8h, v1.h[4] \n" + "smlal v17.4s, v7.4h, v1.h[5] \n" + "smlal2 v29.4s, v7.8h, v1.h[5] \n" + "smlal v18.4s, v7.4h, v1.h[6] \n" + "smlal2 v30.4s, v7.8h, v1.h[6] \n" + "smlal v19.4s, v7.4h, v1.h[7] \n" + "smlal2 v31.4s, v7.8h, v1.h[7] \n" + + "smlal v8.4s, v4.4h, v2.h[0] \n" + "smlal2 v20.4s, v4.8h, v2.h[0] \n" + "smlal v9.4s, v4.4h, v2.h[1] \n" + "smlal2 v21.4s, v4.8h, v2.h[1] \n" + "smlal v10.4s, v4.4h, v2.h[2] \n" + "smlal2 v22.4s, v4.8h, v2.h[2] \n" + "smlal v11.4s, v4.4h, v2.h[3] \n" + "smlal2 v23.4s, v4.8h, v2.h[3] \n" + "smlal v12.4s, v4.4h, v2.h[4] \n" + "smlal2 v24.4s, v4.8h, v2.h[4] \n" + "smlal v13.4s, v4.4h, v2.h[5] \n" + "smlal2 v25.4s, v4.8h, v2.h[5] \n" + "smlal v14.4s, v4.4h, v2.h[6] \n" + "smlal2 v26.4s, v4.8h, v2.h[6] \n" + "smlal v15.4s, v4.4h, v2.h[7] \n" + "smlal2 v27.4s, v4.8h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r89 + + "smlal v16.4s, v4.4h, v3.h[0] \n" + "smlal2 v28.4s, v4.8h, v3.h[0] \n" + "smlal v17.4s, v4.4h, v3.h[1] \n" + "smlal2 v29.4s, v4.8h, v3.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v18.4s, v4.4h, v3.h[2] \n" + "smlal2 v30.4s, v4.8h, v3.h[2] \n" + "smlal v19.4s, v4.4h, v3.h[3] \n" + "smlal2 v31.4s, v4.8h, v3.h[3] \n" + + "ld1 {v6.8h, v7.8h}, [%4], #32 \n" // w67 + + "smlal v8.4s, v5.4h, v3.h[4] \n" + "smlal2 v20.4s, v5.8h, v3.h[4] \n" + "smlal v9.4s, v5.4h, v3.h[5] \n" + "smlal2 v21.4s, v5.8h, v3.h[5] \n" + + "prfm pldl1keep, [%4, #256] \n" + + "smlal v10.4s, v5.4h, v3.h[6] \n" + "smlal2 v22.4s, v5.8h, v3.h[6] \n" + "smlal v11.4s, v5.4h, v3.h[7] \n" + "smlal2 v23.4s, v5.8h, v3.h[7] \n" + + "smlal v12.4s, v5.4h, v0.h[0] \n" + "smlal2 v24.4s, v5.8h, v0.h[0] \n" + "smlal v13.4s, v5.4h, v0.h[1] \n" + "smlal2 v25.4s, v5.8h, v0.h[1] \n" + "smlal v14.4s, v5.4h, v0.h[2] \n" + "smlal2 v26.4s, v5.8h, v0.h[2] \n" + "smlal v15.4s, v5.4h, v0.h[3] \n" + "smlal2 v27.4s, v5.8h, v0.h[3] \n" + "smlal v16.4s, v5.4h, v0.h[4] \n" + "smlal2 v28.4s, v5.8h, v0.h[4] \n" + "smlal v17.4s, v5.4h, v0.h[5] \n" + "smlal2 v29.4s, v5.8h, v0.h[5] \n" + "smlal v18.4s, v5.4h, v0.h[6] \n" + "smlal2 v30.4s, v5.8h, v0.h[6] \n" + "smlal v19.4s, v5.4h, v0.h[7] \n" + "smlal2 v31.4s, v5.8h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%3], #32 \n" // r1011 + + "smlal v8.4s, v6.4h, v1.h[0] \n" + "smlal2 v20.4s, v6.8h, v1.h[0] \n" + "smlal v9.4s, v6.4h, v1.h[1] \n" + "smlal2 v21.4s, v6.8h, v1.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v10.4s, v6.4h, v1.h[2] \n" + "smlal2 v22.4s, v6.8h, v1.h[2] \n" + "smlal v11.4s, v6.4h, v1.h[3] \n" + "smlal2 v23.4s, v6.8h, v1.h[3] \n" + "smlal v12.4s, v6.4h, v1.h[4] \n" + "smlal2 v24.4s, v6.8h, v1.h[4] \n" + "smlal v13.4s, v6.4h, v1.h[5] \n" + "smlal2 v25.4s, v6.8h, v1.h[5] \n" + "smlal v14.4s, v6.4h, v1.h[6] \n" + "smlal2 v26.4s, v6.8h, v1.h[6] \n" + "smlal v15.4s, v6.4h, v1.h[7] \n" + "smlal2 v27.4s, v6.8h, v1.h[7] \n" + "smlal v16.4s, v6.4h, v2.h[0] \n" + "smlal2 v28.4s, v6.8h, v2.h[0] \n" + "smlal v17.4s, v6.4h, v2.h[1] \n" + "smlal2 v29.4s, v6.8h, v2.h[1] \n" + "smlal v18.4s, v6.4h, v2.h[2] \n" + "smlal2 v30.4s, v6.8h, v2.h[2] \n" + "smlal v19.4s, v6.4h, v2.h[3] \n" + "smlal2 v31.4s, v6.8h, v2.h[3] \n" + + "ld1 {v4.8h, v5.8h}, [%4], #32 \n" // w01 + + "smlal v8.4s, v7.4h, v2.h[4] \n" + "smlal2 v20.4s, v7.8h, v2.h[4] \n" + "smlal v9.4s, v7.4h, v2.h[5] \n" + "smlal2 v21.4s, v7.8h, v2.h[5] \n" + + "prfm pldl1keep, [%4, #256] \n" + + "smlal v10.4s, v7.4h, v2.h[6] \n" + "smlal2 v22.4s, v7.8h, v2.h[6] \n" + "smlal v11.4s, v7.4h, v2.h[7] \n" + "smlal2 v23.4s, v7.8h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%3], #32 \n" // r01 + + "smlal v12.4s, v7.4h, v3.h[0] \n" + "smlal2 v24.4s, v7.8h, v3.h[0] \n" + "smlal v13.4s, v7.4h, v3.h[1] \n" + "smlal2 v25.4s, v7.8h, v3.h[1] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal v14.4s, v7.4h, v3.h[2] \n" + "smlal2 v26.4s, v7.8h, v3.h[2] \n" + "smlal v15.4s, v7.4h, v3.h[3] \n" + "smlal2 v27.4s, v7.8h, v3.h[3] \n" + "smlal v16.4s, v7.4h, v3.h[4] \n" + "smlal2 v28.4s, v7.8h, v3.h[4] \n" + "smlal v17.4s, v7.4h, v3.h[5] \n" + "smlal2 v29.4s, v7.8h, v3.h[5] \n" + + "subs %w0, %w0, #1 \n" + + "smlal v18.4s, v7.4h, v3.h[6] \n" + "smlal2 v30.4s, v7.8h, v3.h[6] \n" + "smlal v19.4s, v7.4h, v3.h[7] \n" + "smlal2 v31.4s, v7.8h, v3.h[7] \n" + + "bne 0b \n" + + "sub %3, %3, #32 \n" + "sub %4, %4, #32 \n" + + "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" + "st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [%2], #64 \n" + "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" + "st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [%2], #64 \n" + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + "st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [%2], #64 \n" + + : "=r"(nn), // %0 + "=r"(output0_tm), // %1 + "=r"(output1_tm), // %2 + "=r"(r0), // %3 + "=r"(k0) // %4 + : "0"(nn), + "1"(output0_tm), + "2"(output1_tm), + "3"(r0), + "4"(k0) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 12 + (i % 12) / 8); + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + int32x4_t _sum8 = vdupq_n_s32(0); + int32x4_t _sum9 = vdupq_n_s32(0); + int32x4_t _suma = vdupq_n_s32(0); + int32x4_t _sumb = vdupq_n_s32(0); + int32x4_t _sumc = vdupq_n_s32(0); + int32x4_t _sumd = vdupq_n_s32(0); + int32x4_t _sume = vdupq_n_s32(0); + int32x4_t _sumf = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + int16x8_t _val2 = vld1q_s16(r0 + 16); + int16x8_t _val3 = vld1q_s16(r0 + 24); + int16x8_t _val4 = vld1q_s16(r0 + 32); + int16x8_t _val5 = vld1q_s16(r0 + 40); + int16x8_t _val6 = vld1q_s16(r0 + 48); + int16x8_t _val7 = vld1q_s16(r0 + 56); + + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val0), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val0), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val0), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val0), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val0), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val0), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w0), vget_high_s16(_val0), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w0), vget_high_s16(_val0), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w0), vget_high_s16(_val0), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w0), vget_high_s16(_val0), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w0), vget_high_s16(_val0), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w0), vget_high_s16(_val0), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w0), vget_high_s16(_val0), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w0), vget_high_s16(_val0), 3); + + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val1), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val1), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val1), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val1), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val1), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val1), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w1), vget_high_s16(_val1), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w1), vget_high_s16(_val1), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w1), vget_high_s16(_val1), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w1), vget_high_s16(_val1), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w1), vget_high_s16(_val1), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w1), vget_high_s16(_val1), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w1), vget_high_s16(_val1), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w1), vget_high_s16(_val1), 3); + + int16x8_t _w2 = vld1q_s16(k0 + 16); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val2), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val2), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val2), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val2), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_low_s16(_val2), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_low_s16(_val2), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_low_s16(_val2), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_low_s16(_val2), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w2), vget_high_s16(_val2), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w2), vget_high_s16(_val2), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w2), vget_high_s16(_val2), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w2), vget_high_s16(_val2), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w2), vget_high_s16(_val2), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w2), vget_high_s16(_val2), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w2), vget_high_s16(_val2), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w2), vget_high_s16(_val2), 3); + + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val3), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val3), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val3), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val3), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_low_s16(_val3), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_low_s16(_val3), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_low_s16(_val3), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_low_s16(_val3), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w3), vget_high_s16(_val3), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w3), vget_high_s16(_val3), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w3), vget_high_s16(_val3), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w3), vget_high_s16(_val3), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w3), vget_high_s16(_val3), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w3), vget_high_s16(_val3), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w3), vget_high_s16(_val3), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w3), vget_high_s16(_val3), 3); + + int16x8_t _w4 = vld1q_s16(k0 + 32); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_low_s16(_val4), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_low_s16(_val4), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_low_s16(_val4), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_low_s16(_val4), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w4), vget_low_s16(_val4), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w4), vget_low_s16(_val4), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w4), vget_low_s16(_val4), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w4), vget_low_s16(_val4), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w4), vget_high_s16(_val4), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w4), vget_high_s16(_val4), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w4), vget_high_s16(_val4), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w4), vget_high_s16(_val4), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w4), vget_high_s16(_val4), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w4), vget_high_s16(_val4), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w4), vget_high_s16(_val4), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w4), vget_high_s16(_val4), 3); + + int16x8_t _w5 = vld1q_s16(k0 + 40); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_low_s16(_val5), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_low_s16(_val5), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_low_s16(_val5), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_low_s16(_val5), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w5), vget_low_s16(_val5), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w5), vget_low_s16(_val5), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w5), vget_low_s16(_val5), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w5), vget_low_s16(_val5), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w5), vget_high_s16(_val5), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w5), vget_high_s16(_val5), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w5), vget_high_s16(_val5), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w5), vget_high_s16(_val5), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w5), vget_high_s16(_val5), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w5), vget_high_s16(_val5), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w5), vget_high_s16(_val5), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w5), vget_high_s16(_val5), 3); + + int16x8_t _w6 = vld1q_s16(k0 + 48); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_low_s16(_val6), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_low_s16(_val6), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_low_s16(_val6), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_low_s16(_val6), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w6), vget_low_s16(_val6), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w6), vget_low_s16(_val6), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w6), vget_low_s16(_val6), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w6), vget_low_s16(_val6), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w6), vget_high_s16(_val6), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w6), vget_high_s16(_val6), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w6), vget_high_s16(_val6), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w6), vget_high_s16(_val6), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w6), vget_high_s16(_val6), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w6), vget_high_s16(_val6), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w6), vget_high_s16(_val6), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w6), vget_high_s16(_val6), 3); + + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_low_s16(_val7), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_low_s16(_val7), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_low_s16(_val7), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_low_s16(_val7), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w7), vget_low_s16(_val7), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w7), vget_low_s16(_val7), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w7), vget_low_s16(_val7), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w7), vget_low_s16(_val7), 3); + _sum8 = vmlal_lane_s16(_sum8, vget_low_s16(_w7), vget_high_s16(_val7), 0); + _sum9 = vmlal_lane_s16(_sum9, vget_high_s16(_w7), vget_high_s16(_val7), 0); + _suma = vmlal_lane_s16(_suma, vget_low_s16(_w7), vget_high_s16(_val7), 1); + _sumb = vmlal_lane_s16(_sumb, vget_high_s16(_w7), vget_high_s16(_val7), 1); + _sumc = vmlal_lane_s16(_sumc, vget_low_s16(_w7), vget_high_s16(_val7), 2); + _sumd = vmlal_lane_s16(_sumd, vget_high_s16(_w7), vget_high_s16(_val7), 2); + _sume = vmlal_lane_s16(_sume, vget_low_s16(_w7), vget_high_s16(_val7), 3); + _sumf = vmlal_lane_s16(_sumf, vget_high_s16(_w7), vget_high_s16(_val7), 3); + + r0 += 64; + k0 += 64; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output0_tm + 4, _sum2); + vst1q_s32(output1_tm + 4, _sum3); + vst1q_s32(output0_tm + 8, _sum4); + vst1q_s32(output1_tm + 8, _sum5); + vst1q_s32(output0_tm + 12, _sum6); + vst1q_s32(output1_tm + 12, _sum7); + vst1q_s32(output0_tm + 16, _sum8); + vst1q_s32(output1_tm + 16, _sum9); + vst1q_s32(output0_tm + 20, _suma); + vst1q_s32(output1_tm + 20, _sumb); + vst1q_s32(output0_tm + 24, _sumc); + vst1q_s32(output1_tm + 24, _sumd); + vst1q_s32(output0_tm + 28, _sume); + vst1q_s32(output1_tm + 28, _sumf); + output0_tm += 32; + output1_tm += 32; + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + +#if __aarch64__ + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + int16x8_t _val2 = vld1q_s16(r0 + 16); + int16x8_t _val3 = vld1q_s16(r0 + 24); + + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 0); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val2), 0); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val3), 0); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val3), 0); + + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val2), 1); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val3), 1); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val3), 1); + + int16x8_t _w2 = vld1q_s16(k0 + 16); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val1), 2); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_low_s16(_val2), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_low_s16(_val2), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_low_s16(_val3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_low_s16(_val3), 2); + + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val1), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_low_s16(_val2), 3); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_low_s16(_val2), 3); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_low_s16(_val3), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_low_s16(_val3), 3); + + int16x8_t _w4 = vld1q_s16(k0 + 32); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_high_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_high_s16(_val1), 0); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w4), vget_high_s16(_val2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w4), vget_high_s16(_val2), 0); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w4), vget_high_s16(_val3), 0); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w4), vget_high_s16(_val3), 0); + + int16x8_t _w5 = vld1q_s16(k0 + 40); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_high_s16(_val1), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_high_s16(_val1), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w5), vget_high_s16(_val2), 1); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w5), vget_high_s16(_val2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w5), vget_high_s16(_val3), 1); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w5), vget_high_s16(_val3), 1); + + int16x8_t _w6 = vld1q_s16(k0 + 48); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_high_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_high_s16(_val1), 2); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w6), vget_high_s16(_val2), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w6), vget_high_s16(_val2), 2); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w6), vget_high_s16(_val3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w6), vget_high_s16(_val3), 2); + + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_high_s16(_val1), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_high_s16(_val1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w7), vget_high_s16(_val2), 3); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w7), vget_high_s16(_val2), 3); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w7), vget_high_s16(_val3), 3); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w7), vget_high_s16(_val3), 3); + + r0 += 32; + k0 += 64; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output0_tm + 4, _sum2); + vst1q_s32(output1_tm + 4, _sum3); + vst1q_s32(output0_tm + 8, _sum4); + vst1q_s32(output1_tm + 8, _sum5); + vst1q_s32(output0_tm + 12, _sum6); + vst1q_s32(output1_tm + 12, _sum7); + output0_tm += 16; + output1_tm += 16; +#else + asm volatile( + "veor q8, q8 \n" + "veor q9, q9 \n" + "veor q10, q10 \n" + "veor q11, q11 \n" + "veor q12, q12 \n" + "veor q13, q13 \n" + "veor q14, q14 \n" + "veor q15, q15 \n" + + "0: \n" + + "pld [%3, #256] \n" + "pld [%3, #512] \n" + "vldm %3!, {d0-d7} \n" + + "pld [%4, #256] \n" + "vld1.s16 {d8-d11}, [%4 :128]! \n" + + "vmlal.s16 q8, d8, d0[0] \n" + "vmlal.s16 q12, d9, d0[0] \n" + "vmlal.s16 q9, d8, d2[0] \n" + "vmlal.s16 q13, d9, d2[0] \n" + "vmlal.s16 q10, d8, d4[0] \n" + "vmlal.s16 q14, d9, d4[0] \n" + "vmlal.s16 q11, d8, d6[0] \n" + "vmlal.s16 q15, d9, d6[0] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d8-d9}, [%4 :128]! \n" + + "vmlal.s16 q8, d10, d0[1] \n" + "vmlal.s16 q12, d11, d0[1] \n" + "vmlal.s16 q9, d10, d2[1] \n" + "vmlal.s16 q13, d11, d2[1] \n" + "vmlal.s16 q10, d10, d4[1] \n" + "vmlal.s16 q14, d11, d4[1] \n" + "vmlal.s16 q11, d10, d6[1] \n" + "vmlal.s16 q15, d11, d6[1] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d10-d11}, [%4 :128]! \n" + + "vmlal.s16 q8, d8, d0[2] \n" + "vmlal.s16 q12, d9, d0[2] \n" + "vmlal.s16 q9, d8, d2[2] \n" + "vmlal.s16 q13, d9, d2[2] \n" + "vmlal.s16 q10, d8, d4[2] \n" + "vmlal.s16 q14, d9, d4[2] \n" + "vmlal.s16 q11, d8, d6[2] \n" + "vmlal.s16 q15, d9, d6[2] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d8-d9}, [%4 :128]! \n" + + "vmlal.s16 q8, d10, d0[3] \n" + "vmlal.s16 q12, d11, d0[3] \n" + "vmlal.s16 q9, d10, d2[3] \n" + "vmlal.s16 q13, d11, d2[3] \n" + "vmlal.s16 q10, d10, d4[3] \n" + "vmlal.s16 q14, d11, d4[3] \n" + "vmlal.s16 q11, d10, d6[3] \n" + "vmlal.s16 q15, d11, d6[3] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d10-d11}, [%4 :128]! \n" + + "vmlal.s16 q8, d8, d1[0] \n" + "vmlal.s16 q12, d9, d1[0] \n" + "vmlal.s16 q9, d8, d3[0] \n" + "vmlal.s16 q13, d9, d3[0] \n" + "vmlal.s16 q10, d8, d5[0] \n" + "vmlal.s16 q14, d9, d5[0] \n" + "vmlal.s16 q11, d8, d7[0] \n" + "vmlal.s16 q15, d9, d7[0] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d8-d9}, [%4 :128]! \n" + + "vmlal.s16 q8, d10, d1[1] \n" + "vmlal.s16 q12, d11, d1[1] \n" + "vmlal.s16 q9, d10, d3[1] \n" + "vmlal.s16 q13, d11, d3[1] \n" + "vmlal.s16 q10, d10, d5[1] \n" + "vmlal.s16 q14, d11, d5[1] \n" + "vmlal.s16 q11, d10, d7[1] \n" + "vmlal.s16 q15, d11, d7[1] \n" + + "pld [%4, #128] \n" + "vld1.s16 {d10-d11}, [%4 :128]! \n" + + "vmlal.s16 q8, d8, d1[2] \n" + "vmlal.s16 q12, d9, d1[2] \n" + "vmlal.s16 q9, d8, d3[2] \n" + "vmlal.s16 q13, d9, d3[2] \n" + "vmlal.s16 q10, d8, d5[2] \n" + "vmlal.s16 q14, d9, d5[2] \n" + "vmlal.s16 q11, d8, d7[2] \n" + "vmlal.s16 q15, d9, d7[2] \n" + + "subs %0, %0, #1 \n" + + "vmlal.s16 q8, d10, d1[3] \n" + "vmlal.s16 q12, d11, d1[3] \n" + "vmlal.s16 q9, d10, d3[3] \n" + "vmlal.s16 q13, d11, d3[3] \n" + "vmlal.s16 q10, d10, d5[3] \n" + "vmlal.s16 q14, d11, d5[3] \n" + "vmlal.s16 q11, d10, d7[3] \n" + "vmlal.s16 q15, d11, d7[3] \n" + + "bne 0b \n" + + "vstm %1!, {d16-d23} \n" + "vstm %2!, {d24-d31} \n" + + : "=r"(nn), + "=r"(output0_tm), + "=r"(output1_tm), + "=r"(r0), + "=r"(k0) + : "0"(nn), + "1"(output0_tm), + "2"(output1_tm), + "3"(r0), + "4"(k0) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); +#endif + } + for (; i + 1 < tiles; i += 2) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); +#else + const short* r0 = bb2.row(i / 4 + (i % 4) / 2); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + int16x8_t _w4 = vld1q_s16(k0 + 32); + int16x8_t _w5 = vld1q_s16(k0 + 40); + int16x8_t _w6 = vld1q_s16(k0 + 48); + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val1), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val1), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val1), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w4), vget_high_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w4), vget_high_s16(_val1), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w5), vget_high_s16(_val1), 1); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w5), vget_high_s16(_val1), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w6), vget_high_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w6), vget_high_s16(_val1), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w7), vget_high_s16(_val1), 3); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w7), vget_high_s16(_val1), 3); + + r0 += 16; + k0 += 64; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + vst1q_s32(output0_tm + 4, _sum2); + vst1q_s32(output1_tm + 4, _sum3); + output0_tm += 8; + output1_tm += 8; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); +#else + const short* r0 = bb2.row(i / 4 + (i % 4) / 2 + i % 2); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + int16x8_t _w4 = vld1q_s16(k0 + 32); + int16x8_t _w5 = vld1q_s16(k0 + 40); + int16x8_t _w6 = vld1q_s16(k0 + 48); + int16x8_t _w7 = vld1q_s16(k0 + 56); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3); + + r0 += 8; + k0 += 64; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output1_tm, _sum1); + output0_tm += 4; + output1_tm += 4; + } + } + } + + remain_outch_start += nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + int* output0_tm = top_blob_tm.channel(p); + + const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; +#if __aarch64__ + for (; i + 11 < tiles; i += 12) + { + const short* r0 = bb2.row(i / 12); + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + asm volatile( + "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 + + "eor v8.16b, v8.16b, v8.16b \n" + "eor v9.16b, v9.16b, v9.16b \n" + + "ld1 {v4.8h, v5.8h}, [%3], #32 \n" // w01 + + "eor v10.16b, v10.16b, v10.16b \n" + "eor v11.16b, v11.16b, v11.16b \n" + + "prfm pldl1keep, [%2, #256] \n" + + "eor v12.16b, v12.16b, v12.16b \n" + "eor v13.16b, v13.16b, v13.16b \n" + + "prfm pldl1keep, [%3, #256] \n" + + "eor v14.16b, v14.16b, v14.16b \n" + "eor v15.16b, v15.16b, v15.16b \n" + "eor v16.16b, v16.16b, v16.16b \n" + "eor v17.16b, v17.16b, v17.16b \n" + "eor v18.16b, v18.16b, v18.16b \n" + "eor v19.16b, v19.16b, v19.16b \n" + + "0: \n" + + "smlal v8.4s, v4.4h, v0.h[0] \n" + "smlal v9.4s, v4.4h, v0.h[1] \n" + "smlal v10.4s, v4.4h, v0.h[2] \n" + "smlal v11.4s, v4.4h, v0.h[3] \n" + "smlal v12.4s, v4.4h, v0.h[4] \n" + "smlal v13.4s, v4.4h, v0.h[5] \n" + "smlal v14.4s, v4.4h, v0.h[6] \n" + "smlal v15.4s, v4.4h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r23 + + "smlal v16.4s, v4.4h, v1.h[0] \n" + "smlal v17.4s, v4.4h, v1.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal v18.4s, v4.4h, v1.h[2] \n" + "smlal v19.4s, v4.4h, v1.h[3] \n" + + "smlal2 v8.4s, v4.8h, v1.h[4] \n" + "smlal2 v9.4s, v4.8h, v1.h[5] \n" + "smlal2 v10.4s, v4.8h, v1.h[6] \n" + "smlal2 v11.4s, v4.8h, v1.h[7] \n" + "smlal2 v12.4s, v4.8h, v2.h[0] \n" + "smlal2 v13.4s, v4.8h, v2.h[1] \n" + "smlal2 v14.4s, v4.8h, v2.h[2] \n" + "smlal2 v15.4s, v4.8h, v2.h[3] \n" + "smlal2 v16.4s, v4.8h, v2.h[4] \n" + "smlal2 v17.4s, v4.8h, v2.h[5] \n" + "smlal2 v18.4s, v4.8h, v2.h[6] \n" + "smlal2 v19.4s, v4.8h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r45 + + "smlal v8.4s, v5.4h, v3.h[0] \n" + "smlal v9.4s, v5.4h, v3.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal v10.4s, v5.4h, v3.h[2] \n" + "smlal v11.4s, v5.4h, v3.h[3] \n" + "smlal v12.4s, v5.4h, v3.h[4] \n" + "smlal v13.4s, v5.4h, v3.h[5] \n" + "smlal v14.4s, v5.4h, v3.h[6] \n" + "smlal v15.4s, v5.4h, v3.h[7] \n" + "smlal v16.4s, v5.4h, v0.h[0] \n" + "smlal v17.4s, v5.4h, v0.h[1] \n" + "smlal v18.4s, v5.4h, v0.h[2] \n" + "smlal v19.4s, v5.4h, v0.h[3] \n" + + "ld1 {v6.8h, v7.8h}, [%3], #32 \n" // w23 + + "smlal2 v8.4s, v5.8h, v0.h[4] \n" + "smlal2 v9.4s, v5.8h, v0.h[5] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal2 v10.4s, v5.8h, v0.h[6] \n" + "smlal2 v11.4s, v5.8h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r67 + + "smlal2 v12.4s, v5.8h, v1.h[0] \n" + "smlal2 v13.4s, v5.8h, v1.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal2 v14.4s, v5.8h, v1.h[2] \n" + "smlal2 v15.4s, v5.8h, v1.h[3] \n" + "smlal2 v16.4s, v5.8h, v1.h[4] \n" + "smlal2 v17.4s, v5.8h, v1.h[5] \n" + "smlal2 v18.4s, v5.8h, v1.h[6] \n" + "smlal2 v19.4s, v5.8h, v1.h[7] \n" + + "smlal v8.4s, v6.4h, v2.h[0] \n" + "smlal v9.4s, v6.4h, v2.h[1] \n" + "smlal v10.4s, v6.4h, v2.h[2] \n" + "smlal v11.4s, v6.4h, v2.h[3] \n" + "smlal v12.4s, v6.4h, v2.h[4] \n" + "smlal v13.4s, v6.4h, v2.h[5] \n" + "smlal v14.4s, v6.4h, v2.h[6] \n" + "smlal v15.4s, v6.4h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r89 + + "smlal v16.4s, v6.4h, v3.h[0] \n" + "smlal v17.4s, v6.4h, v3.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal v18.4s, v6.4h, v3.h[2] \n" + "smlal v19.4s, v6.4h, v3.h[3] \n" + + "smlal2 v8.4s, v6.8h, v3.h[4] \n" + "smlal2 v9.4s, v6.8h, v3.h[5] \n" + "smlal2 v10.4s, v6.8h, v3.h[6] \n" + "smlal2 v11.4s, v6.8h, v3.h[7] \n" + "smlal2 v12.4s, v6.8h, v0.h[0] \n" + "smlal2 v13.4s, v6.8h, v0.h[1] \n" + "smlal2 v14.4s, v6.8h, v0.h[2] \n" + "smlal2 v15.4s, v6.8h, v0.h[3] \n" + "smlal2 v16.4s, v6.8h, v0.h[4] \n" + "smlal2 v17.4s, v6.8h, v0.h[5] \n" + "smlal2 v18.4s, v6.8h, v0.h[6] \n" + "smlal2 v19.4s, v6.8h, v0.h[7] \n" + + "ld1 {v2.8h, v3.8h}, [%2], #32 \n" // r1011 + + "smlal v8.4s, v7.4h, v1.h[0] \n" + "smlal v9.4s, v7.4h, v1.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal v10.4s, v7.4h, v1.h[2] \n" + "smlal v11.4s, v7.4h, v1.h[3] \n" + "smlal v12.4s, v7.4h, v1.h[4] \n" + "smlal v13.4s, v7.4h, v1.h[5] \n" + "smlal v14.4s, v7.4h, v1.h[6] \n" + "smlal v15.4s, v7.4h, v1.h[7] \n" + "smlal v16.4s, v7.4h, v2.h[0] \n" + "smlal v17.4s, v7.4h, v2.h[1] \n" + "smlal v18.4s, v7.4h, v2.h[2] \n" + "smlal v19.4s, v7.4h, v2.h[3] \n" + + "ld1 {v4.8h, v5.8h}, [%3], #32 \n" // w01 + + "smlal2 v8.4s, v7.8h, v2.h[4] \n" + "smlal2 v9.4s, v7.8h, v2.h[5] \n" + + "prfm pldl1keep, [%3, #256] \n" + + "smlal2 v10.4s, v7.8h, v2.h[6] \n" + "smlal2 v11.4s, v7.8h, v2.h[7] \n" + + "ld1 {v0.8h, v1.8h}, [%2], #32 \n" // r01 + + "smlal2 v12.4s, v7.8h, v3.h[0] \n" + "smlal2 v13.4s, v7.8h, v3.h[1] \n" + + "prfm pldl1keep, [%2, #256] \n" + + "smlal2 v14.4s, v7.8h, v3.h[2] \n" + "smlal2 v15.4s, v7.8h, v3.h[3] \n" + "smlal2 v16.4s, v7.8h, v3.h[4] \n" + "smlal2 v17.4s, v7.8h, v3.h[5] \n" + + "subs %w0, %w0, #1 \n" + + "smlal2 v18.4s, v7.8h, v3.h[6] \n" + "smlal2 v19.4s, v7.8h, v3.h[7] \n" + + "bne 0b \n" + + "sub %2, %2, #32 \n" + "sub %3, %3, #32 \n" + + "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%1], #64 \n" + "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%1], #64 \n" + "st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%1], #64 \n" + + : "=r"(nn), // %0 + "=r"(output0_tm), // %1 + "=r"(r0), // %2 + "=r"(k0) // %3 + : "0"(nn), + "1"(output0_tm), + "2"(r0), + "3"(k0) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } + for (; i + 7 < tiles; i += 8) + { + const short* r0 = bb2.row(i / 12 + (i % 12) / 8); + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + int16x8_t _val2 = vld1q_s16(r0 + 16); + int16x8_t _val3 = vld1q_s16(r0 + 24); + int16x8_t _val4 = vld1q_s16(r0 + 32); + int16x8_t _val5 = vld1q_s16(r0 + 40); + int16x8_t _val6 = vld1q_s16(r0 + 48); + int16x8_t _val7 = vld1q_s16(r0 + 56); + + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w0), vget_low_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val0), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w0), vget_low_s16(_val0), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_high_s16(_val0), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w0), vget_high_s16(_val0), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_high_s16(_val0), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w0), vget_high_s16(_val0), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w0), vget_low_s16(_val1), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val1), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w0), vget_low_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w0), vget_high_s16(_val1), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_high_s16(_val1), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w0), vget_high_s16(_val1), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_high_s16(_val1), 3); + + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val2), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w1), vget_low_s16(_val2), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val2), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w1), vget_low_s16(_val2), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_high_s16(_val2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w1), vget_high_s16(_val2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_high_s16(_val2), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w1), vget_high_s16(_val2), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w1), vget_low_s16(_val3), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val3), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w1), vget_low_s16(_val3), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val3), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w1), vget_high_s16(_val3), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_high_s16(_val3), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w1), vget_high_s16(_val3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_high_s16(_val3), 3); + + int16x8_t _w2 = vld1q_s16(k0 + 16); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val4), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w2), vget_low_s16(_val4), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_low_s16(_val4), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w2), vget_low_s16(_val4), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_high_s16(_val4), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w2), vget_high_s16(_val4), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_high_s16(_val4), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w2), vget_high_s16(_val4), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w2), vget_low_s16(_val5), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val5), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w2), vget_low_s16(_val5), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_low_s16(_val5), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w2), vget_high_s16(_val5), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_high_s16(_val5), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w2), vget_high_s16(_val5), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_high_s16(_val5), 3); + + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val6), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_w3), vget_low_s16(_val6), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_low_s16(_val6), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_w3), vget_low_s16(_val6), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_high_s16(_val6), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_w3), vget_high_s16(_val6), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_high_s16(_val6), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_w3), vget_high_s16(_val6), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_w3), vget_low_s16(_val7), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val7), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_w3), vget_low_s16(_val7), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_low_s16(_val7), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_w3), vget_high_s16(_val7), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_high_s16(_val7), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_w3), vget_high_s16(_val7), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_high_s16(_val7), 3); + + r0 += 64; + k0 += 32; + } + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output0_tm + 4, _sum1); + vst1q_s32(output0_tm + 8, _sum2); + vst1q_s32(output0_tm + 12, _sum3); + vst1q_s32(output0_tm + 16, _sum4); + vst1q_s32(output0_tm + 20, _sum5); + vst1q_s32(output0_tm + 24, _sum6); + vst1q_s32(output0_tm + 28, _sum7); + output0_tm += 32; + } +#endif // __aarch64__ + for (; i + 3 < tiles; i += 4) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); +#else + const short* r0 = bb2.row(i / 4); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + +#if __aarch64__ + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + int32x4_t _sum4 = vdupq_n_s32(0); + int32x4_t _sum5 = vdupq_n_s32(0); + int32x4_t _sum6 = vdupq_n_s32(0); + int32x4_t _sum7 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + int16x8_t _val2 = vld1q_s16(r0 + 16); + int16x8_t _val3 = vld1q_s16(r0 + 24); + + int16x8_t _w0 = vld1q_s16(k0); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w0), vget_low_s16(_val2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w0), vget_low_s16(_val2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w0), vget_low_s16(_val3), 0); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w0), vget_low_s16(_val3), 1); + + int16x8_t _w1 = vld1q_s16(k0 + 8); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w1), vget_low_s16(_val2), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w1), vget_low_s16(_val2), 3); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w1), vget_low_s16(_val3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w1), vget_low_s16(_val3), 3); + + int16x8_t _w2 = vld1q_s16(k0 + 16); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_high_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_high_s16(_val1), 1); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w2), vget_high_s16(_val2), 0); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w2), vget_high_s16(_val2), 1); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w2), vget_high_s16(_val3), 0); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w2), vget_high_s16(_val3), 1); + + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_high_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_high_s16(_val1), 3); + _sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_w3), vget_high_s16(_val2), 2); + _sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_w3), vget_high_s16(_val2), 3); + _sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_w3), vget_high_s16(_val3), 2); + _sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_w3), vget_high_s16(_val3), 3); + + r0 += 32; + k0 += 32; + } + + _sum0 = vaddq_s32(_sum0, _sum1); + _sum2 = vaddq_s32(_sum2, _sum3); + _sum4 = vaddq_s32(_sum4, _sum5); + _sum6 = vaddq_s32(_sum6, _sum7); + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output0_tm + 4, _sum2); + vst1q_s32(output0_tm + 8, _sum4); + vst1q_s32(output0_tm + 12, _sum6); + output0_tm += 16; +#else + asm volatile( + "veor q8, q8 \n" + "veor q9, q9 \n" + "veor q10, q10 \n" + "veor q11, q11 \n" + "veor q12, q12 \n" + "veor q13, q13 \n" + "veor q14, q14 \n" + "veor q15, q15 \n" + + "0: \n" + + "pld [%2, #256] \n" + "pld [%2, #512] \n" + "vldm %2!, {d0-d7} \n" + + "pld [%3, #256] \n" + "vld1.s16 {d8-d11}, [%3 :128]! \n" + + "vmlal.s16 q8, d8, d0[0] \n" + "vmlal.s16 q12, d9, d0[1] \n" + "vmlal.s16 q9, d8, d2[0] \n" + "vmlal.s16 q13, d9, d2[1] \n" + "vmlal.s16 q10, d8, d4[0] \n" + "vmlal.s16 q14, d9, d4[1] \n" + "vmlal.s16 q11, d8, d6[0] \n" + "vmlal.s16 q15, d9, d6[1] \n" + + "pld [%3, #128] \n" + "vld1.s16 {d8-d9}, [%3 :128]! \n" + + "vmlal.s16 q8, d10, d0[2] \n" + "vmlal.s16 q12, d11, d0[3] \n" + "vmlal.s16 q9, d10, d2[2] \n" + "vmlal.s16 q13, d11, d2[3] \n" + "vmlal.s16 q10, d10, d4[2] \n" + "vmlal.s16 q14, d11, d4[3] \n" + "vmlal.s16 q11, d10, d6[2] \n" + "vmlal.s16 q15, d11, d6[3] \n" + + "pld [%3, #128] \n" + "vld1.s16 {d10-d11}, [%3 :128]! \n" + + "vmlal.s16 q8, d8, d1[0] \n" + "vmlal.s16 q12, d9, d1[1] \n" + "vmlal.s16 q9, d8, d3[0] \n" + "vmlal.s16 q13, d9, d3[1] \n" + "vmlal.s16 q10, d8, d5[0] \n" + "vmlal.s16 q14, d9, d5[1] \n" + "vmlal.s16 q11, d8, d7[0] \n" + "vmlal.s16 q15, d9, d7[1] \n" + + "subs %0, %0, #1 \n" + + "vmlal.s16 q8, d10, d1[2] \n" + "vmlal.s16 q12, d11, d1[3] \n" + "vmlal.s16 q9, d10, d3[2] \n" + "vmlal.s16 q13, d11, d3[3] \n" + "vmlal.s16 q10, d10, d5[2] \n" + "vmlal.s16 q14, d11, d5[3] \n" + "vmlal.s16 q11, d10, d7[2] \n" + "vmlal.s16 q15, d11, d7[3] \n" + + "bne 0b \n" + + "vadd.s32 q8, q8, q12 \n" + "vadd.s32 q9, q9, q13 \n" + "vadd.s32 q10, q10, q14 \n" + "vadd.s32 q11, q11, q15 \n" + + "vstm %1!, {d16-d23} \n" + + : "=r"(nn), + "=r"(output0_tm), + "=r"(r0), + "=r"(k0) + : "0"(nn), + "1"(output0_tm), + "2"(r0), + "3"(k0) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"); +#endif + } + for (; i + 1 < tiles; i += 2) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); +#else + const short* r0 = bb2.row(i / 4 + (i % 4) / 2); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + int32x4_t _sum2 = vdupq_n_s32(0); + int32x4_t _sum3 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + int16x8_t _val1 = vld1q_s16(r0 + 8); + + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w0), vget_low_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w0), vget_low_s16(_val1), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w1), vget_low_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w1), vget_low_s16(_val1), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w2), vget_high_s16(_val1), 0); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w2), vget_high_s16(_val1), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); + _sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_w3), vget_high_s16(_val1), 2); + _sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_w3), vget_high_s16(_val1), 3); + + r0 += 16; + k0 += 32; + } + + _sum0 = vaddq_s32(_sum0, _sum1); + _sum2 = vaddq_s32(_sum2, _sum3); + + vst1q_s32(output0_tm, _sum0); + vst1q_s32(output0_tm + 4, _sum2); + output0_tm += 8; + } + for (; i < tiles; i++) + { +#if __aarch64__ + const short* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); +#else + const short* r0 = bb2.row(i / 4 + (i % 4) / 2 + i % 2); +#endif + const short* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + for (int j = 0; j < nn; j++) + { + int16x8_t _val0 = vld1q_s16(r0); + + int16x8_t _w0 = vld1q_s16(k0); + int16x8_t _w1 = vld1q_s16(k0 + 8); + int16x8_t _w2 = vld1q_s16(k0 + 16); + int16x8_t _w3 = vld1q_s16(k0 + 24); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 3); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_high_s16(_val0), 0); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_high_s16(_val0), 1); + + _sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_high_s16(_val0), 2); + _sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_high_s16(_val0), 3); + + r0 += 8; + k0 += 32; + } + + _sum0 = vaddq_s32(_sum0, _sum1); + + vst1q_s32(output0_tm, _sum0); + output0_tm += 4; + } + } + } +} diff --git a/src/layer/arm/convolution_winograd_transform_int8.h b/src/layer/arm/convolution_winograd_transform_int8.h new file mode 100644 index 000000000..4e27e8c62 --- /dev/null +++ b/src/layer/arm/convolution_winograd_transform_int8.h @@ -0,0 +1,230 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s1_winograd43_transform_input_int8_neon(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt) +{ + const int w = bottom_blob.w; + const int h = bottom_blob.h; + const int inch = bottom_blob.c; + + const int w_tiles = (w - 2) / 4; + const int h_tiles = (h - 2) / 4; + const int tiles = w_tiles * h_tiles; + + // const float itm[6][6] = { + // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, + // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, + // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, + // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} + // }; + + // 0 = 4 * r00 - 5 * r02 + r04 + // 1 = -4 * (r01 + r02) + r04 + r03 + // 2 = 4 * (r01 - r02) + r04 - r03 + // 3 = -2 * (r01 - r03) + r04 - r02 + // 4 = 2 * (r01 - r03) + r04 - r02 + // 5 = 4 * r01 - 5 * r03 + r05 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < inch; q++) + { + const Mat img0 = bottom_blob.channel(q); + Mat img0_tm = bottom_blob_tm.channel(q); + + short tmp[6][6]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const signed char* r0 = img0.row(i * 4) + (j * 4); + + for (int m = 0; m < 6; m++) + { + signed char r00 = r0[0]; + signed char r01 = r0[1]; + signed char r02 = r0[2]; + signed char r03 = r0[3]; + signed char r04 = r0[4]; + signed char r05 = r0[5]; + + short tmp0m = 4 * r00 - 5 * r02 + r04; + short tmp1m = -4 * (r01 + r02) + r04 + r03; + short tmp2m = 4 * (r01 - r02) + r04 - r03; + short tmp3m = -2 * (r01 - r03) + r04 - r02; + short tmp4m = 2 * (r01 - r03) + r04 - r02; + short tmp5m = 4 * r01 - 5 * r03 + r05; + + tmp[0][m] = tmp0m; + tmp[1][m] = tmp1m; + tmp[2][m] = tmp2m; + tmp[3][m] = tmp3m; + tmp[4][m] = tmp4m; + tmp[5][m] = tmp5m; + + r0 += w; + } + + short* r0_tm_0 = (short*)img0_tm + (i * w_tiles + j); + short* r0_tm_1 = r0_tm_0 + tiles; + short* r0_tm_2 = r0_tm_0 + tiles * 2; + short* r0_tm_3 = r0_tm_0 + tiles * 3; + short* r0_tm_4 = r0_tm_0 + tiles * 4; + short* r0_tm_5 = r0_tm_0 + tiles * 5; + + for (int m = 0; m < 6; m++) + { + short tmp00 = tmp[m][0]; + short tmp01 = tmp[m][1]; + short tmp02 = tmp[m][2]; + short tmp03 = tmp[m][3]; + short tmp04 = tmp[m][4]; + short tmp05 = tmp[m][5]; + + short r0tm0 = 4 * tmp00 - 5 * tmp02 + tmp04; + short r0tm1 = -4 * (tmp01 + tmp02) + tmp04 + tmp03; + short r0tm2 = 4 * (tmp01 - tmp02) + tmp04 - tmp03; + short r0tm3 = -2 * (tmp01 - tmp03) + tmp04 - tmp02; + short r0tm4 = 2 * (tmp01 - tmp03) + tmp04 - tmp02; + short r0tm5 = 4 * tmp01 - 5 * tmp03 + tmp05; + + r0_tm_0[0] = r0tm0; + r0_tm_1[0] = r0tm1; + r0_tm_2[0] = r0tm2; + r0_tm_3[0] = r0tm3; + r0_tm_4[0] = r0tm4; + r0_tm_5[0] = r0tm5; + + r0_tm_0 += tiles * 6; + r0_tm_1 += tiles * 6; + r0_tm_2 += tiles * 6; + r0_tm_3 += tiles * 6; + r0_tm_4 += tiles * 6; + r0_tm_5 += tiles * 6; + } + } + } + } +} + +static void conv3x3s1_winograd43_transform_output_int8_neon(const Mat& top_blob_tm, Mat& top_blob, const Option& opt) +{ + const int outw = top_blob.w; + const int outh = top_blob.h; + const int outch = top_blob.c; + + const int w_tiles = outw / 4; + const int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; + + // const float otm[4][6] = { + // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, + // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} + // }; + + // 0 = r00 + (r01 + r02) + (r03 + r04) + // 1 = (r01 - r02) + (r03 - r04) * 2 + // 2 = (r01 + r02) + (r03 + r04) * 4 + // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + const Mat out0_tm = top_blob_tm.channel(p); + Mat out0 = top_blob.channel(p); + + int tmp[4][6]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const int* output0_tm_0 = (const int*)out0_tm + (i * w_tiles + j) * 1; + const int* output0_tm_1 = output0_tm_0 + tiles * 1; + const int* output0_tm_2 = output0_tm_0 + tiles * 2; + const int* output0_tm_3 = output0_tm_0 + tiles * 3; + const int* output0_tm_4 = output0_tm_0 + tiles * 4; + const int* output0_tm_5 = output0_tm_0 + tiles * 5; + + int* output0 = out0.row(i * 4) + j * 4; + + // TODO neon optimize + for (int m = 0; m < 5; m++) + { + int tmp02a = output0_tm_1[0] + output0_tm_2[0]; + int tmp13a = output0_tm_1[0] - output0_tm_2[0]; + + int tmp02b = output0_tm_3[0] + output0_tm_4[0]; + int tmp13b = output0_tm_3[0] - output0_tm_4[0]; + + tmp[0][m] = output0_tm_0[0] + tmp02a + tmp02b; + tmp[1][m] = tmp13a + tmp13b * 2; + tmp[2][m] = tmp02a + tmp02b * 4; + tmp[3][m] = output0_tm_5[0] * 4 + tmp13a + tmp13b * 8; + + output0_tm_0 += tiles * 6; + output0_tm_1 += tiles * 6; + output0_tm_2 += tiles * 6; + output0_tm_3 += tiles * 6; + output0_tm_4 += tiles * 6; + output0_tm_5 += tiles * 6; + } + for (int m = 5; m < 6; m++) + { + int tmp02a = output0_tm_1[0] + output0_tm_2[0]; + int tmp13a = output0_tm_1[0] - output0_tm_2[0]; + + int tmp02b = output0_tm_3[0] + output0_tm_4[0]; + int tmp13b = output0_tm_3[0] - output0_tm_4[0]; + + tmp[0][m] = (output0_tm_0[0] + tmp02a + tmp02b) * 4; + tmp[1][m] = (tmp13a + tmp13b * 2) * 4; + tmp[2][m] = (tmp02a + tmp02b * 4) * 4; + tmp[3][m] = (output0_tm_5[0] * 4 + tmp13a + tmp13b * 8) * 4; + + output0_tm_0 += tiles * 6; + output0_tm_1 += tiles * 6; + output0_tm_2 += tiles * 6; + output0_tm_3 += tiles * 6; + output0_tm_4 += tiles * 6; + output0_tm_5 += tiles * 6; + } + + for (int m = 0; m < 4; m++) + { + const int* tmp0 = tmp[m]; + + int tmp02a = tmp0[1] + tmp0[2]; + int tmp13a = tmp0[1] - tmp0[2]; + + int tmp02b = tmp0[3] + tmp0[4]; + int tmp13b = tmp0[3] - tmp0[4]; + + output0[0] = (tmp0[0] + tmp02a + tmp02b) / 576; + output0[1] = (tmp13a + tmp13b * 2) / 576; + output0[2] = (tmp02a + tmp02b * 4) / 576; + output0[3] = (tmp0[5] + tmp13a + tmp13b * 8) / 576; + + output0 += outw; + } + } + } + } +} diff --git a/src/layer/arm/convolution_winograd_transform_pack4_int8.h b/src/layer/arm/convolution_winograd_transform_pack4_int8.h new file mode 100644 index 000000000..fff5f7d66 --- /dev/null +++ b/src/layer/arm/convolution_winograd_transform_pack4_int8.h @@ -0,0 +1,178 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s1_winograd43_transform_output_pack4_int8_neon(const Mat& top_blob_tm, Mat& top_blob, const Option& opt) +{ + const int outw = top_blob.w; + const int outh = top_blob.h; + const int outch = top_blob.c; + + const int w_tiles = outw / 4; + const int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; + + // const float otm[4][6] = { + // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, + // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} + // }; + + // 0 = r00 + (r01 + r02) + (r03 + r04) + // 1 = (r01 - r02) + (r03 - r04) * 2 + // 2 = (r01 + r02) + (r03 + r04) * 4 + // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + const Mat out0_tm = top_blob_tm.channel(p); + Mat out0 = top_blob.channel(p); + + int tmp[4][6][4]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const int* output0_tm_0 = (const int*)out0_tm + (i * w_tiles + j) * 4; + const int* output0_tm_1 = output0_tm_0 + tiles * 4; + const int* output0_tm_2 = output0_tm_0 + tiles * 8; + const int* output0_tm_3 = output0_tm_0 + tiles * 12; + const int* output0_tm_4 = output0_tm_0 + tiles * 16; + const int* output0_tm_5 = output0_tm_0 + tiles * 20; + + int* output0 = out0.row(i * 4) + (j * 4) * 4; + + for (int m = 0; m < 5; m++) + { + int32x4_t _out0tm0 = vld1q_s32(output0_tm_0); + int32x4_t _out0tm1 = vld1q_s32(output0_tm_1); + int32x4_t _out0tm2 = vld1q_s32(output0_tm_2); + int32x4_t _out0tm3 = vld1q_s32(output0_tm_3); + int32x4_t _out0tm4 = vld1q_s32(output0_tm_4); + int32x4_t _out0tm5 = vld1q_s32(output0_tm_5); + + int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2); + int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2); + + int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4); + int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4); + + int32x4_t _v2 = vdupq_n_s32(2); + int32x4_t _v4 = vdupq_n_s32(4); + int32x4_t _v8 = vdupq_n_s32(8); + + int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b); + int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2); + int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4); + int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8); + + vst1q_s32(tmp[0][m], _tmp0m); + vst1q_s32(tmp[1][m], _tmp1m); + vst1q_s32(tmp[2][m], _tmp2m); + vst1q_s32(tmp[3][m], _tmp3m); + + output0_tm_0 += tiles * 24; + output0_tm_1 += tiles * 24; + output0_tm_2 += tiles * 24; + output0_tm_3 += tiles * 24; + output0_tm_4 += tiles * 24; + output0_tm_5 += tiles * 24; + } + for (int m = 5; m < 6; m++) + { + int32x4_t _out0tm0 = vld1q_s32(output0_tm_0); + int32x4_t _out0tm1 = vld1q_s32(output0_tm_1); + int32x4_t _out0tm2 = vld1q_s32(output0_tm_2); + int32x4_t _out0tm3 = vld1q_s32(output0_tm_3); + int32x4_t _out0tm4 = vld1q_s32(output0_tm_4); + int32x4_t _out0tm5 = vld1q_s32(output0_tm_5); + + int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2); + int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2); + + int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4); + int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4); + + int32x4_t _v2 = vdupq_n_s32(2); + int32x4_t _v4 = vdupq_n_s32(4); + int32x4_t _v8 = vdupq_n_s32(8); + + int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b); + int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2); + int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4); + int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8); + + _tmp0m = vmulq_s32(_tmp0m, _v4); + _tmp1m = vmulq_s32(_tmp1m, _v4); + _tmp2m = vmulq_s32(_tmp2m, _v4); + _tmp3m = vmulq_s32(_tmp3m, _v4); + + vst1q_s32(tmp[0][m], _tmp0m); + vst1q_s32(tmp[1][m], _tmp1m); + vst1q_s32(tmp[2][m], _tmp2m); + vst1q_s32(tmp[3][m], _tmp3m); + + output0_tm_0 += tiles * 24; + output0_tm_1 += tiles * 24; + output0_tm_2 += tiles * 24; + output0_tm_3 += tiles * 24; + output0_tm_4 += tiles * 24; + output0_tm_5 += tiles * 24; + } + + for (int m = 0; m < 4; m++) + { + int32x4_t _tmp00 = vld1q_s32(tmp[m][0]); + int32x4_t _tmp01 = vld1q_s32(tmp[m][1]); + int32x4_t _tmp02 = vld1q_s32(tmp[m][2]); + int32x4_t _tmp03 = vld1q_s32(tmp[m][3]); + int32x4_t _tmp04 = vld1q_s32(tmp[m][4]); + int32x4_t _tmp05 = vld1q_s32(tmp[m][5]); + + int32x4_t _tmp02a = vaddq_s32(_tmp01, _tmp02); + int32x4_t _tmp13a = vsubq_s32(_tmp01, _tmp02); + + int32x4_t _tmp02b = vaddq_s32(_tmp03, _tmp04); + int32x4_t _tmp13b = vsubq_s32(_tmp03, _tmp04); + + int32x4_t _v2 = vdupq_n_s32(2); + int32x4_t _v4 = vdupq_n_s32(4); + int32x4_t _v8 = vdupq_n_s32(8); + + int32x4_t _out00 = vaddq_s32(vaddq_s32(_tmp00, _tmp02a), _tmp02b); + int32x4_t _out01 = vmlaq_s32(_tmp13a, _tmp13b, _v2); + int32x4_t _out02 = vmlaq_s32(_tmp02a, _tmp02b, _v4); + int32x4_t _out03 = vmlaq_s32(vaddq_s32(_tmp05, _tmp13a), _tmp13b, _v8); + + // TODO use integer trick for division by 576 + float32x4_t _v576 = vdupq_n_f32(1.0 / 576); + _out00 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out00), _v576)); + _out01 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out01), _v576)); + _out02 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out02), _v576)); + _out03 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out03), _v576)); + + vst1q_s32(output0, _out00); + vst1q_s32(output0 + 4, _out01); + vst1q_s32(output0 + 8, _out02); + vst1q_s32(output0 + 12, _out03); + + output0 += outw * 4; + } + } + } + } +} diff --git a/src/layer/arm/convolution_winograd_transform_pack8_int8.h b/src/layer/arm/convolution_winograd_transform_pack8_int8.h new file mode 100644 index 000000000..f0d8981ef --- /dev/null +++ b/src/layer/arm/convolution_winograd_transform_pack8_int8.h @@ -0,0 +1,131 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s1_winograd43_transform_input_pack8_int8_neon(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt) +{ + const int w = bottom_blob.w; + const int h = bottom_blob.h; + const int inch = bottom_blob.c; + + const int w_tiles = (w - 2) / 4; + const int h_tiles = (h - 2) / 4; + const int tiles = w_tiles * h_tiles; + + // const float itm[6][6] = { + // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, + // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, + // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, + // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} + // }; + + // 0 = 4 * r00 - 5 * r02 + r04 + // 1 = -4 * (r01 + r02) + r04 + r03 + // 2 = 4 * (r01 - r02) + r04 - r03 + // 3 = -2 * (r01 - r03) + r04 - r02 + // 4 = 2 * (r01 - r03) + r04 - r02 + // 5 = 4 * r01 - 5 * r03 + r05 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < inch; q++) + { + const Mat img0 = bottom_blob.channel(q); + Mat img0_tm = bottom_blob_tm.channel(q); + + short tmp[6][6][8]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const signed char* r0 = img0.row(i * 4) + (j * 4) * 8; + + for (int m = 0; m < 6; m++) + { + int8x8_t _r00 = vld1_s8(r0); + int8x8_t _r01 = vld1_s8(r0 + 8); + int8x8_t _r02 = vld1_s8(r0 + 16); + int8x8_t _r03 = vld1_s8(r0 + 24); + int8x8_t _r04 = vld1_s8(r0 + 32); + int8x8_t _r05 = vld1_s8(r0 + 40); + + int8x8_t _v4s8 = vdup_n_s8(4); + int8x8_t _v5s8 = vdup_n_s8(5); + int16x8_t _v2 = vdupq_n_s16(2); + int16x8_t _v4 = vdupq_n_s16(4); + + int16x8_t _tmp0m = vsubq_s16(vaddw_s8(vmull_s8(_r00, _v4s8), _r04), vmull_s8(_r02, _v5s8)); + int16x8_t _tmp1m = vmlsq_s16(vaddl_s8(_r04, _r03), vaddl_s8(_r01, _r02), _v4); + int16x8_t _tmp2m = vmlaq_s16(vsubl_s8(_r04, _r03), vsubl_s8(_r01, _r02), _v4); + int16x8_t _tmp3m = vmlsq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); + int16x8_t _tmp4m = vmlaq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2); + int16x8_t _tmp5m = vsubq_s16(vaddw_s8(vmull_s8(_r01, _v4s8), _r05), vmull_s8(_r03, _v5s8)); + + vst1q_s16(tmp[0][m], _tmp0m); + vst1q_s16(tmp[1][m], _tmp1m); + vst1q_s16(tmp[2][m], _tmp2m); + vst1q_s16(tmp[3][m], _tmp3m); + vst1q_s16(tmp[4][m], _tmp4m); + vst1q_s16(tmp[5][m], _tmp5m); + + r0 += w * 8; + } + + short* r0_tm_0 = (short*)img0_tm + (i * w_tiles + j) * 8; + short* r0_tm_1 = r0_tm_0 + tiles * 8; + short* r0_tm_2 = r0_tm_0 + tiles * 16; + short* r0_tm_3 = r0_tm_0 + tiles * 24; + short* r0_tm_4 = r0_tm_0 + tiles * 32; + short* r0_tm_5 = r0_tm_0 + tiles * 40; + + for (int m = 0; m < 6; m++) + { + int16x8_t _tmp00 = vld1q_s16(tmp[m][0]); + int16x8_t _tmp01 = vld1q_s16(tmp[m][1]); + int16x8_t _tmp02 = vld1q_s16(tmp[m][2]); + int16x8_t _tmp03 = vld1q_s16(tmp[m][3]); + int16x8_t _tmp04 = vld1q_s16(tmp[m][4]); + int16x8_t _tmp05 = vld1q_s16(tmp[m][5]); + + int16x8_t _v2 = vdupq_n_s16(2); + int16x8_t _v4 = vdupq_n_s16(4); + int16x8_t _v5 = vdupq_n_s16(5); + + int16x8_t _r0tm0 = vmlsq_s16(vmlaq_s16(_tmp04, _tmp00, _v4), _tmp02, _v5); + int16x8_t _r0tm1 = vmlsq_s16(vaddq_s16(_tmp04, _tmp03), vaddq_s16(_tmp01, _tmp02), _v4); + int16x8_t _r0tm2 = vmlaq_s16(vsubq_s16(_tmp04, _tmp03), vsubq_s16(_tmp01, _tmp02), _v4); + int16x8_t _r0tm3 = vmlsq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); + int16x8_t _r0tm4 = vmlaq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2); + int16x8_t _r0tm5 = vmlsq_s16(vmlaq_s16(_tmp05, _tmp01, _v4), _tmp03, _v5); + + vst1q_s16(r0_tm_0, _r0tm0); + vst1q_s16(r0_tm_1, _r0tm1); + vst1q_s16(r0_tm_2, _r0tm2); + vst1q_s16(r0_tm_3, _r0tm3); + vst1q_s16(r0_tm_4, _r0tm4); + vst1q_s16(r0_tm_5, _r0tm5); + + r0_tm_0 += tiles * 48; + r0_tm_1 += tiles * 48; + r0_tm_2 += tiles * 48; + r0_tm_3 += tiles * 48; + r0_tm_4 += tiles * 48; + r0_tm_5 += tiles * 48; + } + } + } + } +} diff --git a/src/layer/mips/convolution_winograd_transform_int8.h b/src/layer/mips/convolution_winograd_transform_int8.h index ee849eace..7bcb8b94e 100644 --- a/src/layer/mips/convolution_winograd_transform_int8.h +++ b/src/layer/mips/convolution_winograd_transform_int8.h @@ -165,11 +165,6 @@ static void conv3x3s1_winograd43_transform_output_int8_msa(const Mat& top_blob_t int* output0 = out0.row(i * 4) + j * 4; - // 0 = r00 + (r01 + r02) + (r03 + r04) - // 1 = (r01 - r02) + (r03 - r04) * 2 - // 2 = (r01 + r02) + (r03 + r04) * 4 - // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 - for (int m = 0; m < 5; m++) { int tmp02a = output0_tm_1[0] + output0_tm_2[0];