arm neon optimization for general convolution im2col sgemm (#2668)

* arm neon optimization for conv3x3s1 winograd42

* better condition

* Update test_convolution.cpp

* Update test_convolution.cpp

* more proper conditions

* arm neon optimization for general im2col sgemm pack4

* add sgemm

* wip

* wip

* fix armv7 build

* more conditions blah blah

* code format

* fix convolution

* move packed convolution to seperated header source

* unify weight data bf16

* proper conditions

* conv3x3s2 sgemm pack4 test

src/layer/arm/convolution_1x1_pack8_fp16s.h

@@ -12,774 +12,17 @@
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
 
-static void conv1x1s1_sgemm_transform_kernel_pack8_fp16sa_neon(const Mat& kernel, Mat& kernel_tm_pack8, int inch, int outch)
-{
-    // interleave
-    // src = inch-outch
-    // dst = 8b-8a-inch/8a-outch/8b
-    kernel_tm_pack8.create(1, inch / 8, outch / 8, (size_t)2u * 64, 64);
-
-    int q = 0;
-    for (; q + 7 < outch; q += 8)
-    {
-        const float* k0 = (const float*)kernel + (q + 0) * inch;
-        const float* k1 = (const float*)kernel + (q + 1) * inch;
-        const float* k2 = (const float*)kernel + (q + 2) * inch;
-        const float* k3 = (const float*)kernel + (q + 3) * inch;
-        const float* k4 = (const float*)kernel + (q + 4) * inch;
-        const float* k5 = (const float*)kernel + (q + 5) * inch;
-        const float* k6 = (const float*)kernel + (q + 6) * inch;
-        const float* k7 = (const float*)kernel + (q + 7) * inch;
-
-        __fp16* g0 = kernel_tm_pack8.channel(q / 8);
-
-        for (int p = 0; p + 7 < inch; p += 8)
-        {
-            for (int i = 0; i < 8; i++)
-            {
-                g0[0] = (__fp16)k0[i];
-                g0[1] = (__fp16)k1[i];
-                g0[2] = (__fp16)k2[i];
-                g0[3] = (__fp16)k3[i];
-                g0[4] = (__fp16)k4[i];
-                g0[5] = (__fp16)k5[i];
-                g0[6] = (__fp16)k6[i];
-                g0[7] = (__fp16)k7[i];
-
-                g0 += 8;
-            }
-
-            k0 += 8;
-            k1 += 8;
-            k2 += 8;
-            k3 += 8;
-            k4 += 8;
-            k5 += 8;
-            k6 += 8;
-            k7 += 8;
-        }
-    }
-}
-
 static void conv1x1s1_sgemm_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
 {
     int w = bottom_blob.w;
     int h = bottom_blob.h;
-    int inch = bottom_blob.c;
-    int outch = top_blob.c;
-
-    size_t elemsize = bottom_blob.elemsize;
-    int elempack = bottom_blob.elempack;
-
     const int size = w * h;
 
-    const __fp16* bias = _bias;
-
-    // interleave
-    Mat tmp;
-    if (size >= 12)
-        tmp.create(12, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, elemsize, elempack, opt.workspace_allocator);
-    else if (size >= 8)
-        tmp.create(8, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator);
-    else if (size >= 4)
-        tmp.create(4, inch, size / 4 + (size % 4) / 2 + size % 2, elemsize, elempack, opt.workspace_allocator);
-    else if (size >= 2)
-        tmp.create(2, inch, size / 2 + size % 2, elemsize, elempack, opt.workspace_allocator);
-    else // if (size >= 1)
-        tmp.create(1, inch, size, elemsize, elempack, opt.workspace_allocator);
-    {
-        int nn_size;
-        int remain_size_start;
-
-        nn_size = size / 12;
-        remain_size_start = nn_size * 12;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int ii = 0; ii < nn_size; ii++)
-        {
-            int i = ii * 12;
-
-            const __fp16* img0 = bottom_blob.channel(0);
-            img0 += i * 8;
-
-            __fp16* tmpptr = tmp.channel(i / 12);
-
-            for (int q = 0; q < inch; q++)
-            {
-                // transpose 12x8
-                asm volatile(
-                    "prfm   pldl1keep, [%0, #512]   \n"
-                    "ld4    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
-                    "ld4    {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n"
-                    "ld4    {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n"
-
-                    "sub    %0, %0, #128            \n"
-
-                    "uzp1   v20.8h, v0.8h, v4.8h    \n" // 0
-                    "uzp1   v21.8h, v16.8h, v1.8h   \n" // 1
-                    "uzp1   v22.8h, v5.8h, v17.8h   \n" // 2
-                    "uzp1   v23.8h, v2.8h, v6.8h    \n" // 3
-                    "uzp1   v24.8h, v18.8h, v3.8h   \n" // 4
-                    "uzp1   v25.8h, v7.8h, v19.8h   \n" // 5
-                    "uzp2   v26.8h, v0.8h, v4.8h    \n" // 6
-                    "uzp2   v27.8h, v16.8h, v1.8h   \n" // 7
-                    "uzp2   v28.8h, v5.8h, v17.8h   \n" // 8
-                    "uzp2   v29.8h, v2.8h, v6.8h    \n" // 9
-                    "uzp2   v30.8h, v18.8h, v3.8h   \n" // 10
-                    "uzp2   v31.8h, v7.8h, v19.8h   \n" // 11
-
-                    "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
-                    "st1    {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n"
-                    "st1    {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n"
-                    : "=r"(img0),  // %0
-                    "=r"(tmpptr) // %1
-                    : "0"(img0),
-                    "1"(tmpptr)
-                    : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
-                img0 += bottom_blob.cstep * 8;
-            }
-        }
-
-        nn_size = (size - remain_size_start) >> 3;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int ii = 0; ii < nn_size; ii++)
-        {
-            int i = remain_size_start + ii * 8;
-
-            const __fp16* img0 = bottom_blob.channel(0);
-            img0 += i * 8;
-
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8);
-
-            for (int q = 0; q < inch; q++)
-            {
-                // transpose 8x8
-                asm volatile(
-                    "prfm   pldl1keep, [%0, #512]   \n"
-                    "ld4    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
-                    "ld4    {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n"
-                    "sub    %0, %0, #64             \n"
-
-                    "uzp1   v16.8h, v0.8h, v4.8h    \n"
-                    "uzp2   v20.8h, v0.8h, v4.8h    \n"
-                    "uzp1   v17.8h, v1.8h, v5.8h    \n"
-                    "uzp2   v21.8h, v1.8h, v5.8h    \n"
-                    "uzp1   v18.8h, v2.8h, v6.8h    \n"
-                    "uzp2   v22.8h, v2.8h, v6.8h    \n"
-                    "uzp1   v19.8h, v3.8h, v7.8h    \n"
-                    "uzp2   v23.8h, v3.8h, v7.8h    \n"
-
-                    "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
-                    "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
-                    : "=r"(img0),  // %0
-                    "=r"(tmpptr) // %1
-                    : "0"(img0),
-                    "1"(tmpptr)
-                    : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
-
-                img0 += bottom_blob.cstep * 8;
-            }
-        }
-
-        remain_size_start += nn_size << 3;
-        nn_size = (size - remain_size_start) >> 2;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int ii = 0; ii < nn_size; ii++)
-        {
-            int i = remain_size_start + ii * 4;
-
-            const __fp16* img0 = bottom_blob.channel(0);
-            img0 += i * 8;
-
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);
-
-            for (int q = 0; q < inch; q++)
-            {
-                asm volatile(
-                    "prfm   pldl1keep, [%0, #512]   \n"
-                    "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n"
-                    "st1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n"
-                    : "=r"(img0),  // %0
-                    "=r"(tmpptr) // %1
-                    : "0"(img0),
-                    "1"(tmpptr)
-                    : "memory", "v0", "v1", "v2", "v3");
-
-                img0 += bottom_blob.cstep * 8;
-            }
-        }
-
-        remain_size_start += nn_size << 2;
-        nn_size = (size - remain_size_start) >> 1;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int ii = 0; ii < nn_size; ii++)
-        {
-            int i = remain_size_start + ii * 2;
-
-            const __fp16* img0 = bottom_blob.channel(0);
-            img0 += i * 8;
-
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);
-
-            for (int q = 0; q < inch; q++)
-            {
-                asm volatile(
-                    "prfm   pldl1keep, [%0, #256]   \n"
-                    "ld1    {v0.8h, v1.8h}, [%0]    \n"
-                    "st1    {v0.8h, v1.8h}, [%1], #32 \n"
-                    : "=r"(img0),  // %0
-                    "=r"(tmpptr) // %1
-                    : "0"(img0),
-                    "1"(tmpptr)
-                    : "memory", "v0", "v1");
-
-                img0 += bottom_blob.cstep * 8;
-            }
-        }
-
-        remain_size_start += nn_size << 1;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int i = remain_size_start; i < size; i++)
-        {
-            const __fp16* img0 = bottom_blob.channel(0);
-            img0 += i * 8;
-
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);
-
-            for (int q = 0; q < inch; q++)
-            {
-                asm volatile(
-                    "prfm   pldl1keep, [%0, #128]   \n"
-                    "ld1    {v0.8h}, [%0]           \n"
-                    "st1    {v0.8h}, [%1], #16      \n"
-                    : "=r"(img0),  // %0
-                    "=r"(tmpptr) // %1
-                    : "0"(img0),
-                    "1"(tmpptr)
-                    : "memory", "v0");
-
-                img0 += bottom_blob.cstep * 8;
-            }
-        }
-    }
-
-    #pragma omp parallel for num_threads(opt.num_threads)
-    for (int p = 0; p < outch; p++)
-    {
-        __fp16* outptr0 = top_blob.channel(p);
-
-        const __fp16 zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
-        const __fp16* biasptr = bias ? bias + p * 8 : zeros;
-
-        int i = 0;
-        for (; i + 11 < size; i += 12)
-        {
-            __fp16* tmpptr = tmp.channel(i / 12);
-            const __fp16* kptr0 = kernel.channel(p);
-
-            int nn = inch; // inch always > 0
-
-            asm volatile(
-                "ld1    {v20.8h}, [%8]              \n"
-                "mov    v21.16b, v20.16b            \n"
-                "mov    v22.16b, v20.16b            \n"
-                "mov    v23.16b, v20.16b            \n"
-                "mov    v24.16b, v20.16b            \n"
-                "mov    v25.16b, v20.16b            \n"
-                "mov    v26.16b, v20.16b            \n"
-                "mov    v27.16b, v20.16b            \n"
-                "mov    v28.16b, v20.16b            \n"
-                "mov    v29.16b, v20.16b            \n"
-                "mov    v30.16b, v20.16b            \n"
-                "mov    v31.16b, v20.16b            \n"
-
-                "0:                                 \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w0123
-
-                "fmla   v20.8h, v12.8h, v0.h[0]     \n"
-                "fmla   v21.8h, v12.8h, v0.h[1]     \n"
-                "fmla   v22.8h, v12.8h, v0.h[2]     \n"
-                "fmla   v23.8h, v12.8h, v0.h[3]     \n"
-                "fmla   v24.8h, v12.8h, v0.h[4]     \n"
-                "fmla   v25.8h, v12.8h, v0.h[5]     \n"
-                "fmla   v26.8h, v12.8h, v0.h[6]     \n"
-                "fmla   v27.8h, v12.8h, v0.h[7]     \n"
-                "fmla   v28.8h, v12.8h, v1.h[0]     \n"
-                "fmla   v29.8h, v12.8h, v1.h[1]     \n"
-                "fmla   v30.8h, v12.8h, v1.h[2]     \n"
-                "fmla   v31.8h, v12.8h, v1.h[3]     \n"
-
-                "fmla   v20.8h, v13.8h, v1.h[4]     \n"
-                "fmla   v21.8h, v13.8h, v1.h[5]     \n"
-                "fmla   v22.8h, v13.8h, v1.h[6]     \n"
-                "fmla   v23.8h, v13.8h, v1.h[7]     \n"
-                "fmla   v24.8h, v13.8h, v2.h[0]     \n"
-                "fmla   v25.8h, v13.8h, v2.h[1]     \n"
-                "fmla   v26.8h, v13.8h, v2.h[2]     \n"
-                "fmla   v27.8h, v13.8h, v2.h[3]     \n"
-                "fmla   v28.8h, v13.8h, v2.h[4]     \n"
-                "fmla   v29.8h, v13.8h, v2.h[5]     \n"
-                "fmla   v30.8h, v13.8h, v2.h[6]     \n"
-                "fmla   v31.8h, v13.8h, v2.h[7]     \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567
-
-                "fmla   v20.8h, v14.8h, v3.h[0]     \n"
-                "fmla   v21.8h, v14.8h, v3.h[1]     \n"
-                "fmla   v22.8h, v14.8h, v3.h[2]     \n"
-                "fmla   v23.8h, v14.8h, v3.h[3]     \n"
-                "fmla   v24.8h, v14.8h, v3.h[4]     \n"
-                "fmla   v25.8h, v14.8h, v3.h[5]     \n"
-                "fmla   v26.8h, v14.8h, v3.h[6]     \n"
-                "fmla   v27.8h, v14.8h, v3.h[7]     \n"
-                "fmla   v28.8h, v14.8h, v4.h[0]     \n"
-                "fmla   v29.8h, v14.8h, v4.h[1]     \n"
-                "fmla   v30.8h, v14.8h, v4.h[2]     \n"
-                "fmla   v31.8h, v14.8h, v4.h[3]     \n"
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%3], #64 \n" // w4567
-
-                "fmla   v20.8h, v15.8h, v4.h[4]     \n"
-                "fmla   v21.8h, v15.8h, v4.h[5]     \n"
-                "fmla   v22.8h, v15.8h, v4.h[6]     \n"
-                "fmla   v23.8h, v15.8h, v4.h[7]     \n"
-                "fmla   v24.8h, v15.8h, v5.h[0]     \n"
-                "fmla   v25.8h, v15.8h, v5.h[1]     \n"
-                "fmla   v26.8h, v15.8h, v5.h[2]     \n"
-                "fmla   v27.8h, v15.8h, v5.h[3]     \n"
-                "fmla   v28.8h, v15.8h, v5.h[4]     \n"
-                "fmla   v29.8h, v15.8h, v5.h[5]     \n"
-                "fmla   v30.8h, v15.8h, v5.h[6]     \n"
-                "fmla   v31.8h, v15.8h, v5.h[7]     \n"
-
-                "fmla   v20.8h, v16.8h, v6.h[0]     \n"
-                "fmla   v21.8h, v16.8h, v6.h[1]     \n"
-                "fmla   v22.8h, v16.8h, v6.h[2]     \n"
-                "fmla   v23.8h, v16.8h, v6.h[3]     \n"
-                "fmla   v24.8h, v16.8h, v6.h[4]     \n"
-                "fmla   v25.8h, v16.8h, v6.h[5]     \n"
-                "fmla   v26.8h, v16.8h, v6.h[6]     \n"
-                "fmla   v27.8h, v16.8h, v6.h[7]     \n"
-                "fmla   v28.8h, v16.8h, v7.h[0]     \n"
-                "fmla   v29.8h, v16.8h, v7.h[1]     \n"
-                "fmla   v30.8h, v16.8h, v7.h[2]     \n"
-                "fmla   v31.8h, v16.8h, v7.h[3]     \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%2], #64 \n" // r891011
-
-                "fmla   v20.8h, v17.8h, v7.h[4]     \n"
-                "fmla   v21.8h, v17.8h, v7.h[5]     \n"
-                "fmla   v22.8h, v17.8h, v7.h[6]     \n"
-                "fmla   v23.8h, v17.8h, v7.h[7]     \n"
-                "fmla   v24.8h, v17.8h, v8.h[0]     \n"
-                "fmla   v25.8h, v17.8h, v8.h[1]     \n"
-                "fmla   v26.8h, v17.8h, v8.h[2]     \n"
-                "fmla   v27.8h, v17.8h, v8.h[3]     \n"
-                "fmla   v28.8h, v17.8h, v8.h[4]     \n"
-                "fmla   v29.8h, v17.8h, v8.h[5]     \n"
-                "fmla   v30.8h, v17.8h, v8.h[6]     \n"
-                "fmla   v31.8h, v17.8h, v8.h[7]     \n"
-
-                "fmla   v20.8h, v18.8h, v9.h[0]     \n"
-                "fmla   v21.8h, v18.8h, v9.h[1]     \n"
-                "fmla   v22.8h, v18.8h, v9.h[2]     \n"
-                "fmla   v23.8h, v18.8h, v9.h[3]     \n"
-                "fmla   v24.8h, v18.8h, v9.h[4]     \n"
-                "fmla   v25.8h, v18.8h, v9.h[5]     \n"
-                "fmla   v26.8h, v18.8h, v9.h[6]     \n"
-                "fmla   v27.8h, v18.8h, v9.h[7]     \n"
-                "fmla   v28.8h, v18.8h, v10.h[0]    \n"
-                "fmla   v29.8h, v18.8h, v10.h[1]    \n"
-                "fmla   v30.8h, v18.8h, v10.h[2]    \n"
-                "fmla   v31.8h, v18.8h, v10.h[3]    \n"
-
-                "subs   %w0, %w0, #1                \n"
-
-                "fmla   v20.8h, v19.8h, v10.h[4]    \n"
-                "fmla   v21.8h, v19.8h, v10.h[5]    \n"
-                "fmla   v22.8h, v19.8h, v10.h[6]    \n"
-                "fmla   v23.8h, v19.8h, v10.h[7]    \n"
-                "fmla   v24.8h, v19.8h, v11.h[0]    \n"
-                "fmla   v25.8h, v19.8h, v11.h[1]    \n"
-                "fmla   v26.8h, v19.8h, v11.h[2]    \n"
-                "fmla   v27.8h, v19.8h, v11.h[3]    \n"
-                "fmla   v28.8h, v19.8h, v11.h[4]    \n"
-                "fmla   v29.8h, v19.8h, v11.h[5]    \n"
-                "fmla   v30.8h, v19.8h, v11.h[6]    \n"
-                "fmla   v31.8h, v19.8h, v11.h[7]    \n"
-
-                "bne    0b                          \n"
-
-                "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
-                "st1    {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n"
-                "st1    {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n"
-
-                : "=r"(nn),      // %0
-                "=r"(outptr0), // %1
-                "=r"(tmpptr),  // %2
-                "=r"(kptr0)    // %3
-                : "0"(nn),
-                "1"(outptr0),
-                "2"(tmpptr),
-                "3"(kptr0),
-                "r"(biasptr) // %8
-                : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
-        }
-        for (; i + 7 < size; i += 8)
-        {
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8);
-            const __fp16* kptr0 = kernel.channel(p);
-
-            int nn = inch; // inch always > 0
-
-            asm volatile(
-                "ld1    {v16.8h}, [%8]              \n"
-                "mov    v17.16b, v16.16b            \n"
-                "mov    v18.16b, v16.16b            \n"
-                "mov    v19.16b, v16.16b            \n"
-                "mov    v20.16b, v16.16b            \n"
-                "mov    v21.16b, v16.16b            \n"
-                "mov    v22.16b, v16.16b            \n"
-                "mov    v23.16b, v16.16b            \n"
-
-                "0:                                 \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
-
-                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
-                "fmla   v17.8h, v8.8h, v0.h[1]      \n"
-                "fmla   v18.8h, v8.8h, v0.h[2]      \n"
-                "fmla   v19.8h, v8.8h, v0.h[3]      \n"
-                "fmla   v20.8h, v8.8h, v0.h[4]      \n"
-                "fmla   v21.8h, v8.8h, v0.h[5]      \n"
-                "fmla   v22.8h, v8.8h, v0.h[6]      \n"
-                "fmla   v23.8h, v8.8h, v0.h[7]      \n"
-
-                "fmla   v16.8h, v9.8h, v1.h[0]      \n"
-                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
-                "fmla   v18.8h, v9.8h, v1.h[2]      \n"
-                "fmla   v19.8h, v9.8h, v1.h[3]      \n"
-                "fmla   v20.8h, v9.8h, v1.h[4]      \n"
-                "fmla   v21.8h, v9.8h, v1.h[5]      \n"
-                "fmla   v22.8h, v9.8h, v1.h[6]      \n"
-                "fmla   v23.8h, v9.8h, v1.h[7]      \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567
-
-                "fmla   v16.8h, v10.8h, v2.h[0]     \n"
-                "fmla   v17.8h, v10.8h, v2.h[1]     \n"
-                "fmla   v18.8h, v10.8h, v2.h[2]     \n"
-                "fmla   v19.8h, v10.8h, v2.h[3]     \n"
-                "fmla   v20.8h, v10.8h, v2.h[4]     \n"
-                "fmla   v21.8h, v10.8h, v2.h[5]     \n"
-                "fmla   v22.8h, v10.8h, v2.h[6]     \n"
-                "fmla   v23.8h, v10.8h, v2.h[7]     \n"
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
-
-                "fmla   v16.8h, v11.8h, v3.h[0]     \n"
-                "fmla   v17.8h, v11.8h, v3.h[1]     \n"
-                "fmla   v18.8h, v11.8h, v3.h[2]     \n"
-                "fmla   v19.8h, v11.8h, v3.h[3]     \n"
-                "fmla   v20.8h, v11.8h, v3.h[4]     \n"
-                "fmla   v21.8h, v11.8h, v3.h[5]     \n"
-                "fmla   v22.8h, v11.8h, v3.h[6]     \n"
-                "fmla   v23.8h, v11.8h, v3.h[7]     \n"
-
-                "fmla   v16.8h, v12.8h, v4.h[0]     \n"
-                "fmla   v17.8h, v12.8h, v4.h[1]     \n"
-                "fmla   v18.8h, v12.8h, v4.h[2]     \n"
-                "fmla   v19.8h, v12.8h, v4.h[3]     \n"
-                "fmla   v20.8h, v12.8h, v4.h[4]     \n"
-                "fmla   v21.8h, v12.8h, v4.h[5]     \n"
-                "fmla   v22.8h, v12.8h, v4.h[6]     \n"
-                "fmla   v23.8h, v12.8h, v4.h[7]     \n"
-
-                "fmla   v16.8h, v13.8h, v5.h[0]     \n"
-                "fmla   v17.8h, v13.8h, v5.h[1]     \n"
-                "fmla   v18.8h, v13.8h, v5.h[2]     \n"
-                "fmla   v19.8h, v13.8h, v5.h[3]     \n"
-                "fmla   v20.8h, v13.8h, v5.h[4]     \n"
-                "fmla   v21.8h, v13.8h, v5.h[5]     \n"
-                "fmla   v22.8h, v13.8h, v5.h[6]     \n"
-                "fmla   v23.8h, v13.8h, v5.h[7]     \n"
-
-                "fmla   v16.8h, v14.8h, v6.h[0]     \n"
-                "fmla   v17.8h, v14.8h, v6.h[1]     \n"
-                "fmla   v18.8h, v14.8h, v6.h[2]     \n"
-                "fmla   v19.8h, v14.8h, v6.h[3]     \n"
-                "fmla   v20.8h, v14.8h, v6.h[4]     \n"
-                "fmla   v21.8h, v14.8h, v6.h[5]     \n"
-                "fmla   v22.8h, v14.8h, v6.h[6]     \n"
-                "fmla   v23.8h, v14.8h, v6.h[7]     \n"
-
-                "subs   %w0, %w0, #1                \n"
-
-                "fmla   v16.8h, v15.8h, v7.h[0]     \n"
-                "fmla   v17.8h, v15.8h, v7.h[1]     \n"
-                "fmla   v18.8h, v15.8h, v7.h[2]     \n"
-                "fmla   v19.8h, v15.8h, v7.h[3]     \n"
-                "fmla   v20.8h, v15.8h, v7.h[4]     \n"
-                "fmla   v21.8h, v15.8h, v7.h[5]     \n"
-                "fmla   v22.8h, v15.8h, v7.h[6]     \n"
-                "fmla   v23.8h, v15.8h, v7.h[7]     \n"
-
-                "bne    0b                          \n"
-
-                "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
-                "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
-
-                : "=r"(nn),      // %0
-                "=r"(outptr0), // %1
-                "=r"(tmpptr),  // %2
-                "=r"(kptr0)    // %3
-                : "0"(nn),
-                "1"(outptr0),
-                "2"(tmpptr),
-                "3"(kptr0),
-                "r"(biasptr) // %8
-                : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
-        }
-        for (; i + 3 < size; i += 4)
-        {
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);
-            const __fp16* kptr0 = kernel.channel(p);
-
-            int nn = inch; // inch always > 0
-
-            asm volatile(
-                "ld1    {v16.8h}, [%8]              \n"
-                "mov    v17.16b, v16.16b            \n"
-                "mov    v18.16b, v16.16b            \n"
-                "mov    v19.16b, v16.16b            \n"
-
-                "0:                                 \n"
-
-                "prfm   pldl1keep, [%2, #512]       \n"
-                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
-
-                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
-                "fmla   v17.8h, v8.8h, v1.h[0]      \n"
-                "fmla   v18.8h, v8.8h, v2.h[0]      \n"
-                "fmla   v19.8h, v8.8h, v3.h[0]      \n"
-
-                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
-                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
-                "fmla   v18.8h, v9.8h, v2.h[1]      \n"
-                "fmla   v19.8h, v9.8h, v3.h[1]      \n"
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
-
-                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
-                "fmla   v17.8h, v10.8h, v1.h[2]     \n"
-                "fmla   v18.8h, v10.8h, v2.h[2]     \n"
-                "fmla   v19.8h, v10.8h, v3.h[2]     \n"
-
-                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
-                "fmla   v17.8h, v11.8h, v1.h[3]     \n"
-                "fmla   v18.8h, v11.8h, v2.h[3]     \n"
-                "fmla   v19.8h, v11.8h, v3.h[3]     \n"
-
-                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
-                "fmla   v17.8h, v12.8h, v1.h[4]     \n"
-                "fmla   v18.8h, v12.8h, v2.h[4]     \n"
-                "fmla   v19.8h, v12.8h, v3.h[4]     \n"
-
-                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
-                "fmla   v17.8h, v13.8h, v1.h[5]     \n"
-                "fmla   v18.8h, v13.8h, v2.h[5]     \n"
-                "fmla   v19.8h, v13.8h, v3.h[5]     \n"
-
-                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
-                "fmla   v17.8h, v14.8h, v1.h[6]     \n"
-                "fmla   v18.8h, v14.8h, v2.h[6]     \n"
-                "fmla   v19.8h, v14.8h, v3.h[6]     \n"
-
-                "subs   %w0, %w0, #1                \n"
-
-                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
-                "fmla   v17.8h, v15.8h, v1.h[7]     \n"
-                "fmla   v18.8h, v15.8h, v2.h[7]     \n"
-                "fmla   v19.8h, v15.8h, v3.h[7]     \n"
-
-                "bne    0b                          \n"
-
-                "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
-
-                : "=r"(nn),      // %0
-                "=r"(outptr0), // %1
-                "=r"(tmpptr),  // %2
-                "=r"(kptr0)    // %3
-                : "0"(nn),
-                "1"(outptr0),
-                "2"(tmpptr),
-                "3"(kptr0),
-                "r"(biasptr) // %8
-                : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19");
-        }
-        for (; i + 1 < size; i += 2)
-        {
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);
-            const __fp16* kptr0 = kernel.channel(p);
-
-            int nn = inch; // inch always > 0
-
-            asm volatile(
-                "ld1    {v16.8h}, [%8]              \n"
-                "mov    v17.16b, v16.16b            \n"
-
-                "0:                                 \n"
-
-                "prfm   pldl1keep, [%2, #256]       \n"
-                "ld1    {v0.8h, v1.8h}, [%2], #32   \n" // r01
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
-
-                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
-                "fmla   v17.8h, v8.8h, v1.h[0]      \n"
-
-                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
-                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
-
-                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
-                "fmla   v17.8h, v10.8h, v1.h[2]     \n"
-
-                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
-                "fmla   v17.8h, v11.8h, v1.h[3]     \n"
-
-                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
-                "fmla   v17.8h, v12.8h, v1.h[4]     \n"
-
-                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
-                "fmla   v17.8h, v13.8h, v1.h[5]     \n"
-
-                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
-                "fmla   v17.8h, v14.8h, v1.h[6]     \n"
-
-                "subs   %w0, %w0, #1                \n"
-
-                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
-                "fmla   v17.8h, v15.8h, v1.h[7]     \n"
-
-                "bne    0b                          \n"
-
-                "st1    {v16.8h, v17.8h}, [%1], #32 \n"
-
-                : "=r"(nn),      // %0
-                "=r"(outptr0), // %1
-                "=r"(tmpptr),  // %2
-                "=r"(kptr0)    // %3
-                : "0"(nn),
-                "1"(outptr0),
-                "2"(tmpptr),
-                "3"(kptr0),
-                "r"(biasptr) // %8
-                : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17");
-        }
-        for (; i < size; i++)
-        {
-            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);
-            const __fp16* kptr0 = kernel.channel(p);
-
-            int nn = inch; // inch always > 0
-
-            asm volatile(
-                "ld1    {v16.8h}, [%8]              \n"
-
-                "0:                                 \n"
-
-                "prfm   pldl1keep, [%2, #128]       \n"
-                "ld1    {v0.8h}, [%2], #16          \n" // r0
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
-
-                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
-                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
-
-                "prfm   pldl1keep, [%3, #512]       \n"
-                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
-
-                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
-                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
-
-                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
-                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
-
-                "subs   %w0, %w0, #1                \n"
-
-                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
-                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
-
-                "bne    0b                          \n"
-
-                "st1    {v16.8h}, [%1], #16         \n"
-
-                : "=r"(nn),      // %0
-                "=r"(outptr0), // %1
-                "=r"(tmpptr),  // %2
-                "=r"(kptr0)    // %3
-                : "0"(nn),
-                "1"(outptr0),
-                "2"(tmpptr),
-                "3"(kptr0),
-                "r"(biasptr) // %8
-                : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16");
-        }
-    }
+    Mat bottom_im2col = bottom_blob;
+    bottom_im2col.w = size;
+    bottom_im2col.h = 1;
 
-    //     // NOTE sgemm
-    //     for (; p<outch; p++)
-    //     {
-    //         Mat out0 = top_blob.channel(p);
-    //
-    //         const __fp16 bias0 = bias ? bias[p] : 0.f;
-    //
-    //         __fp16* outptr0 = out0;
-    //
-    //         for (int i=0; i<size; i++)
-    //         {
-    //             __fp16 sum = bias0;
-    //
-    //             const __fp16* kptr = _kernel.channel(p);
-    //
-    //             for (int q=0; q<inch; q++)
-    //             {
-    //                 const __fp16* img0 = bottom_blob.channel(q);
-    //
-    //                 sum += img0[i] * kptr[0];
-    //                 kptr ++;
-    //             }
-    //
-    //             outptr0[i] = sum;
-    //         }
-    //     }
+    im2col_sgemm_pack8_fp16sa_neon(bottom_im2col, top_blob, kernel, _bias, opt);
 }
 
 static void conv1x1s2_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)

src/layer/arm/convolution_3x3_pack4.h

@@ -4985,3 +4985,144 @@ static void conv3x3s2_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Ma
         }
     }
 }
+
+static void conv3x3s2_im2col_sgemm_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    const int size = outw * outh;
+
+    // im2col
+    Mat bottom_im2col(size, 9, inch, 16u, 4, opt.workspace_allocator);
+    {
+        const int gap = (w * 2 - outw * 2) * 4;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < inch; p++)
+        {
+            const Mat img = bottom_blob.channel(p);
+            Mat out = bottom_im2col.channel(p);
+
+            float* ptr0 = out.row(0);
+            float* ptr1 = out.row(1);
+            float* ptr2 = out.row(2);
+            float* ptr3 = out.row(3);
+            float* ptr4 = out.row(4);
+            float* ptr5 = out.row(5);
+            float* ptr6 = out.row(6);
+            float* ptr7 = out.row(7);
+            float* ptr8 = out.row(8);
+
+            const float* r0 = img.row(0);
+            const float* r1 = img.row(1);
+            const float* r2 = img.row(2);
+
+            for (int i = 0; i < outh; i++)
+            {
+                int j = 0;
+                for (; j + 1 < outw; j += 2)
+                {
+                    float32x4_t _r00 = vld1q_f32(r0);
+                    float32x4_t _r01 = vld1q_f32(r0 + 4);
+                    float32x4_t _r02 = vld1q_f32(r0 + 8);
+                    float32x4_t _r03 = vld1q_f32(r0 + 12);
+                    float32x4_t _r04 = vld1q_f32(r0 + 16);
+
+                    float32x4_t _r10 = vld1q_f32(r1);
+                    float32x4_t _r11 = vld1q_f32(r1 + 4);
+                    float32x4_t _r12 = vld1q_f32(r1 + 8);
+                    float32x4_t _r13 = vld1q_f32(r1 + 12);
+                    float32x4_t _r14 = vld1q_f32(r1 + 16);
+
+                    float32x4_t _r20 = vld1q_f32(r2);
+                    float32x4_t _r21 = vld1q_f32(r2 + 4);
+                    float32x4_t _r22 = vld1q_f32(r2 + 8);
+                    float32x4_t _r23 = vld1q_f32(r2 + 12);
+                    float32x4_t _r24 = vld1q_f32(r2 + 16);
+
+                    vst1q_f32(ptr0, _r00);
+                    vst1q_f32(ptr0 + 4, _r02);
+                    vst1q_f32(ptr1, _r01);
+                    vst1q_f32(ptr1 + 4, _r03);
+                    vst1q_f32(ptr2, _r02);
+                    vst1q_f32(ptr2 + 4, _r04);
+
+                    vst1q_f32(ptr3, _r10);
+                    vst1q_f32(ptr3 + 4, _r12);
+                    vst1q_f32(ptr4, _r11);
+                    vst1q_f32(ptr4 + 4, _r13);
+                    vst1q_f32(ptr5, _r12);
+                    vst1q_f32(ptr5 + 4, _r14);
+
+                    vst1q_f32(ptr6, _r20);
+                    vst1q_f32(ptr6 + 4, _r22);
+                    vst1q_f32(ptr7, _r21);
+                    vst1q_f32(ptr7 + 4, _r23);
+                    vst1q_f32(ptr8, _r22);
+                    vst1q_f32(ptr8 + 4, _r24);
+
+                    r0 += 16;
+                    r1 += 16;
+                    r2 += 16;
+
+                    ptr0 += 8;
+                    ptr1 += 8;
+                    ptr2 += 8;
+                    ptr3 += 8;
+                    ptr4 += 8;
+                    ptr5 += 8;
+                    ptr6 += 8;
+                    ptr7 += 8;
+                    ptr8 += 8;
+                }
+                for (; j < outw; j++)
+                {
+                    float32x4_t _r00 = vld1q_f32(r0);
+                    float32x4_t _r01 = vld1q_f32(r0 + 4);
+                    float32x4_t _r02 = vld1q_f32(r0 + 8);
+
+                    float32x4_t _r10 = vld1q_f32(r1);
+                    float32x4_t _r11 = vld1q_f32(r1 + 4);
+                    float32x4_t _r12 = vld1q_f32(r1 + 8);
+
+                    float32x4_t _r20 = vld1q_f32(r2);
+                    float32x4_t _r21 = vld1q_f32(r2 + 4);
+                    float32x4_t _r22 = vld1q_f32(r2 + 8);
+
+                    vst1q_f32(ptr0, _r00);
+                    vst1q_f32(ptr1, _r01);
+                    vst1q_f32(ptr2, _r02);
+                    vst1q_f32(ptr3, _r10);
+                    vst1q_f32(ptr4, _r11);
+                    vst1q_f32(ptr5, _r12);
+                    vst1q_f32(ptr6, _r20);
+                    vst1q_f32(ptr7, _r21);
+                    vst1q_f32(ptr8, _r22);
+
+                    r0 += 8;
+                    r1 += 8;
+                    r2 += 8;
+
+                    ptr0 += 4;
+                    ptr1 += 4;
+                    ptr2 += 4;
+                    ptr3 += 4;
+                    ptr4 += 4;
+                    ptr5 += 4;
+                    ptr6 += 4;
+                    ptr7 += 4;
+                    ptr8 += 4;
+                }
+
+                r0 += gap;
+                r1 += gap;
+                r2 += gap;
+            }
+        }
+    }
+
+    im2col_sgemm_pack4_neon(bottom_im2col, top_blob, kernel, _bias, opt);
+}

src/layer/arm/convolution_arm.h

@@ -46,7 +46,6 @@ public:
     bool use_winograd3x3;
     bool use_sgemm1x1;
     Mat weight_3x3_winograd64_data;
-    Mat weight_1x1_sgemm_data;
     Mat weight_3x3s2_data;
     Mat weight_sgemm_data;
 
@@ -59,15 +58,13 @@ public:
     Mat weight_data_pack4to1;
 
     Mat weight_3x3_winograd42_data_pack4;
+    Mat weight_sgemm_data_pack4;
 
     // fp16
     Mat weight_data_fp16;
     Mat bias_data_fp16;
 
     // bf16
-    Mat weight_data_pack4_bf16;
-    Mat weight_data_pack1to4_bf16;
-    Mat weight_data_pack4to1_bf16;
     Mat weight_data_bf16;
 
     // int8

src/layer/arm/convolution_bf16s.h

@@ -0,0 +1,115 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_bf16s(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        unsigned short* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * channels * p;
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const unsigned short* sptr = m.row<unsigned short>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float val = bfloat16_to_float32(sptr[space_ofs[k]]);
+                        float wt = bfloat16_to_float32(kptr[k]);
+                        sum += val * wt;
+                    }
+
+                    kptr += maxk;
+                }
+
+                if (activation_type == 1)
+                {
+                    sum = std::max(sum, 0.f);
+                }
+                else if (activation_type == 2)
+                {
+                    float slope = activation_params[0];
+                    sum = sum > 0.f ? sum : sum * slope;
+                }
+                else if (activation_type == 3)
+                {
+                    float min = activation_params[0];
+                    float max = activation_params[1];
+                    if (sum < min)
+                        sum = min;
+                    if (sum > max)
+                        sum = max;
+                }
+                else if (activation_type == 4)
+                {
+                    sum = static_cast<float>(1.f / (1.f + exp(-sum)));
+                }
+                else if (activation_type == 5)
+                {
+                    sum = static_cast<float>(sum * tanh(log(exp(sum) + 1.f)));
+                }
+
+                outptr[j] = float32_to_bfloat16(sum);
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_fp16s.h

@@ -0,0 +1,90 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_fp16s(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<__fp16>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float val = (float)sptr[space_ofs[k]];
+                        float w = (float)kptr[k];
+                        sum += val * w;
+                    }
+
+                    kptr += maxk;
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = (__fp16)sum;
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_pack1to4.h

@@ -0,0 +1,131 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack1to4_neon(const Mat& weight_data, Mat& weight_data_pack1to4, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4b-kw-kh-inch-outch/4b
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_pack1to4.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4);
+
+    for (int q = 0; q + 3 < num_output; q += 4)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        const Mat k1 = weight_data_r2.channel(q + 1);
+        const Mat k2 = weight_data_r2.channel(q + 2);
+        const Mat k3 = weight_data_r2.channel(q + 3);
+
+        Mat g0 = weight_data_pack1to4.channel(q / 4);
+
+        for (int p = 0; p < num_input; p++)
+        {
+            const float* k00 = k0.row(p);
+            const float* k10 = k1.row(p);
+            const float* k20 = k2.row(p);
+            const float* k30 = k3.row(p);
+
+            float* g00 = g0.row(p);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = k00[k];
+                g00[1] = k10[k];
+                g00[2] = k20[k];
+                g00[3] = k30[k];
+
+                g00 += 4;
+            }
+        }
+    }
+}
+
+static void convolution_pack1to4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack1to4, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        float* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4;
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const float* sptr = m.row(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++) // 29.23
+                    {
+                        float32x4_t _val = vdupq_n_f32(sptr[space_ofs[k]]);
+                        float32x4_t _w = vld1q_f32(kptr);
+                        _sum = vmlaq_f32(_sum, _val, _w);
+
+                        kptr += 4;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1q_f32(outptr + j * 4, _sum);
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack1to4_bf16s.h

@@ -0,0 +1,131 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack1to4_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4b-kw-kh-inch-outch/4b
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_bf16.create(maxk, num_input, num_output / 4, (size_t)2 * 4, 4);
+
+    for (int q = 0; q + 3 < num_output; q += 4)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        const Mat k1 = weight_data_r2.channel(q + 1);
+        const Mat k2 = weight_data_r2.channel(q + 2);
+        const Mat k3 = weight_data_r2.channel(q + 3);
+
+        Mat g0 = weight_data_bf16.channel(q / 4);
+
+        for (int p = 0; p < num_input; p++)
+        {
+            const float* k00 = k0.row(p);
+            const float* k10 = k1.row(p);
+            const float* k20 = k2.row(p);
+            const float* k30 = k3.row(p);
+
+            unsigned short* g00 = g0.row<unsigned short>(p);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = float32_to_bfloat16(k00[k]);
+                g00[1] = float32_to_bfloat16(k10[k]);
+                g00[2] = float32_to_bfloat16(k20[k]);
+                g00[3] = float32_to_bfloat16(k30[k]);
+
+                g00 += 4;
+            }
+        }
+    }
+}
+
+static void convolution_pack1to4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        unsigned short* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const unsigned short* kptr = weight_data_bf16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const unsigned short* sptr = m.row<const unsigned short>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vdupq_n_f32(bfloat16_to_float32(sptr[space_ofs[k]]));
+                        float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr));
+                        _sum = vmlaq_f32(_sum, _val, _w);
+
+                        kptr += 4;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum));
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack1to4_fp16s.h

@@ -0,0 +1,167 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack1to4_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vcvt_f32_f16(vdup_n_f16(sptr[space_ofs[k]]));
+                        float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
+                        _sum = vfmaq_f32(_sum, _val, _w);
+
+                        kptr += 4;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum));
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}
+
+static void convolution_pack1to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x4_t _sum = vdup_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1_f16(bias_data_ptr + p * 4);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x4_t _val = vdup_n_f16(sptr[space_ofs[k]]);
+                        float16x4_t _w = vld1_f16(kptr);
+                        _sum = vfma_f16(_sum, _val, _w);
+
+                        kptr += 4;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_f16(outptr + j * 4, _sum);
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack1to8_fp16s.h

@@ -0,0 +1,90 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x8_t _sum = vdupq_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f16(bias_data_ptr + p * 8);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x8_t _val = vdupq_n_f16(sptr[space_ofs[k]]);
+                        float16x8_t _w = vld1q_f16(kptr);
+                        _sum = vfmaq_f16(_sum, _val, _w);
+
+                        kptr += 8;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1q_f16(outptr + j * 8, _sum);
+            }
+
+            outptr += outw * 8;
+        }
+    }
+}

src/layer/arm/convolution_pack4.h

@@ -0,0 +1,175 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack4_neon(const Mat& weight_data, Mat& weight_data_pack4, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4b-4a-kw-kh-inch/4a-outch/4b
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_pack4.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16);
+
+    for (int q = 0; q + 3 < num_output; q += 4)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        const Mat k1 = weight_data_r2.channel(q + 1);
+        const Mat k2 = weight_data_r2.channel(q + 2);
+        const Mat k3 = weight_data_r2.channel(q + 3);
+
+        Mat g0 = weight_data_pack4.channel(q / 4);
+
+        for (int p = 0; p + 3 < num_input; p += 4)
+        {
+            const float* k00 = k0.row(p);
+            const float* k01 = k0.row(p + 1);
+            const float* k02 = k0.row(p + 2);
+            const float* k03 = k0.row(p + 3);
+
+            const float* k10 = k1.row(p);
+            const float* k11 = k1.row(p + 1);
+            const float* k12 = k1.row(p + 2);
+            const float* k13 = k1.row(p + 3);
+
+            const float* k20 = k2.row(p);
+            const float* k21 = k2.row(p + 1);
+            const float* k22 = k2.row(p + 2);
+            const float* k23 = k2.row(p + 3);
+
+            const float* k30 = k3.row(p);
+            const float* k31 = k3.row(p + 1);
+            const float* k32 = k3.row(p + 2);
+            const float* k33 = k3.row(p + 3);
+
+            float* g00 = g0.row(p / 4);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = k00[k];
+                g00[1] = k10[k];
+                g00[2] = k20[k];
+                g00[3] = k30[k];
+
+                g00[4] = k01[k];
+                g00[5] = k11[k];
+                g00[6] = k21[k];
+                g00[7] = k31[k];
+
+                g00[8] = k02[k];
+                g00[9] = k12[k];
+                g00[10] = k22[k];
+                g00[11] = k32[k];
+
+                g00[12] = k03[k];
+                g00[13] = k13[k];
+                g00[14] = k23[k];
+                g00[15] = k33[k];
+
+                g00 += 16;
+            }
+        }
+    }
+}
+
+static void convolution_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack4, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        float* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16;
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const float* sptr = m.row(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++) // 29.23
+                    {
+                        float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4);
+
+                        float32x4_t _w0 = vld1q_f32(kptr);
+                        float32x4_t _w1 = vld1q_f32(kptr + 4);
+                        float32x4_t _w2 = vld1q_f32(kptr + 8);
+                        float32x4_t _w3 = vld1q_f32(kptr + 12);
+
+#if __aarch64__
+                        _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);
+                        _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);
+                        _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);
+                        _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);
+#else
+                        _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);
+                        _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);
+                        _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);
+                        _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);
+#endif
+
+                        kptr += 16;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1q_f32(outptr + j * 4, _sum);
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack4_bf16s.h

@@ -0,0 +1,176 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack4_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4b-4a-kw-kh-inch/4a-outch/4b
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_bf16.create(maxk, num_input / 4, num_output / 4, (size_t)2 * 16, 16);
+
+    for (int q = 0; q + 3 < num_output; q += 4)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        const Mat k1 = weight_data_r2.channel(q + 1);
+        const Mat k2 = weight_data_r2.channel(q + 2);
+        const Mat k3 = weight_data_r2.channel(q + 3);
+
+        Mat g0 = weight_data_bf16.channel(q / 4);
+
+        for (int p = 0; p + 3 < num_input; p += 4)
+        {
+            const float* k00 = k0.row(p);
+            const float* k01 = k0.row(p + 1);
+            const float* k02 = k0.row(p + 2);
+            const float* k03 = k0.row(p + 3);
+
+            const float* k10 = k1.row(p);
+            const float* k11 = k1.row(p + 1);
+            const float* k12 = k1.row(p + 2);
+            const float* k13 = k1.row(p + 3);
+
+            const float* k20 = k2.row(p);
+            const float* k21 = k2.row(p + 1);
+            const float* k22 = k2.row(p + 2);
+            const float* k23 = k2.row(p + 3);
+
+            const float* k30 = k3.row(p);
+            const float* k31 = k3.row(p + 1);
+            const float* k32 = k3.row(p + 2);
+            const float* k33 = k3.row(p + 3);
+
+            unsigned short* g00 = g0.row<unsigned short>(p / 4);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = float32_to_bfloat16(k00[k]);
+                g00[1] = float32_to_bfloat16(k10[k]);
+                g00[2] = float32_to_bfloat16(k20[k]);
+                g00[3] = float32_to_bfloat16(k30[k]);
+
+                g00[4] = float32_to_bfloat16(k01[k]);
+                g00[5] = float32_to_bfloat16(k11[k]);
+                g00[6] = float32_to_bfloat16(k21[k]);
+                g00[7] = float32_to_bfloat16(k31[k]);
+
+                g00[8] = float32_to_bfloat16(k02[k]);
+                g00[9] = float32_to_bfloat16(k12[k]);
+                g00[10] = float32_to_bfloat16(k22[k]);
+                g00[11] = float32_to_bfloat16(k32[k]);
+
+                g00[12] = float32_to_bfloat16(k03[k]);
+                g00[13] = float32_to_bfloat16(k13[k]);
+                g00[14] = float32_to_bfloat16(k23[k]);
+                g00[15] = float32_to_bfloat16(k33[k]);
+
+                g00 += 16;
+            }
+        }
+    }
+}
+
+static void convolution_pack4_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        unsigned short* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const unsigned short* kptr = weight_data_bf16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const unsigned short* sptr = m.row<const unsigned short>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4));
+
+                        float32x4_t _w0 = vcvt_f32_bf16(vld1_u16(kptr));
+                        float32x4_t _w1 = vcvt_f32_bf16(vld1_u16(kptr + 4));
+                        float32x4_t _w2 = vcvt_f32_bf16(vld1_u16(kptr + 8));
+                        float32x4_t _w3 = vcvt_f32_bf16(vld1_u16(kptr + 12));
+
+#if __aarch64__
+                        _sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);
+                        _sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);
+                        _sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);
+                        _sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);
+#else
+                        _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);
+                        _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);
+                        _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);
+                        _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);
+#endif
+
+                        kptr += 16;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_u16(outptr + j * 4, vcvt_bf16_f32(_sum));
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack4_fp16s.h

@@ -0,0 +1,183 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack4_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float32x4_t _sum = vdupq_n_f32(0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f32(bias_data_ptr + p * 4);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4));
+
+                        float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr));
+                        float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4));
+                        float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8));
+                        float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12));
+
+                        _sum = vfmaq_laneq_f32(_sum, _w0, _val, 0);
+                        _sum = vfmaq_laneq_f32(_sum, _w1, _val, 1);
+                        _sum = vfmaq_laneq_f32(_sum, _w2, _val, 2);
+                        _sum = vfmaq_laneq_f32(_sum, _w3, _val, 3);
+
+                        kptr += 16;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_f16(outptr + j * 4, vcvt_f16_f32(_sum));
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}
+
+static void convolution_pack4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x4_t _sum = vdup_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1_f16(bias_data_ptr + p * 4);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4);
+
+                        float16x4_t _w0 = vld1_f16(kptr);
+                        float16x4_t _w1 = vld1_f16(kptr + 4);
+                        float16x4_t _w2 = vld1_f16(kptr + 8);
+                        float16x4_t _w3 = vld1_f16(kptr + 12);
+
+                        _sum = vfma_lane_f16(_sum, _w0, _val, 0);
+                        _sum = vfma_lane_f16(_sum, _w1, _val, 1);
+                        _sum = vfma_lane_f16(_sum, _w2, _val, 2);
+                        _sum = vfma_lane_f16(_sum, _w3, _val, 3);
+
+                        kptr += 16;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_f16(outptr + j * 4, _sum);
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_pack4to1.h

@@ -0,0 +1,134 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack4to1_neon(const Mat& weight_data, Mat& weight_data_pack4to1, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4a-kw-kh-inch/4a-outch
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_pack4to1.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4);
+
+    for (int q = 0; q < num_output; q++)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        Mat g0 = weight_data_pack4to1.channel(q);
+
+        for (int p = 0; p + 3 < num_input; p += 4)
+        {
+            const float* k00 = k0.row(p);
+            const float* k01 = k0.row(p + 1);
+            const float* k02 = k0.row(p + 2);
+            const float* k03 = k0.row(p + 3);
+
+            float* g00 = g0.row(p / 4);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = k00[k];
+                g00[1] = k01[k];
+                g00[2] = k02[k];
+                g00[3] = k03[k];
+
+                g00 += 4;
+            }
+        }
+    }
+}
+
+static void convolution_pack4to1_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_pack4to1, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        float* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4;
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const float* sptr = m.row(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++) // 29.23
+                    {
+                        float32x4_t _val = vld1q_f32(sptr + space_ofs[k] * 4);
+                        float32x4_t _w = vld1q_f32(kptr);
+                        float32x4_t _s4 = vmulq_f32(_val, _w);
+#if __aarch64__
+                        sum += vaddvq_f32(_s4); // dot
+#else
+                        float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));
+                        _ss = vpadd_f32(_ss, _ss);
+                        sum += vget_lane_f32(_ss, 0);
+#endif
+
+                        kptr += 4;
+                    }
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = sum;
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_pack4to1_bf16s.h

@@ -0,0 +1,134 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_transform_kernel_pack4to1_bf16s_neon(const Mat& weight_data, Mat& weight_data_bf16, int num_input, int num_output, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // src = kw-kh-inch-outch
+    // dst = 4a-kw-kh-inch/4a-outch
+    Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+    weight_data_bf16.create(maxk, num_input / 4, num_output, (size_t)2 * 4, 4);
+
+    for (int q = 0; q < num_output; q++)
+    {
+        const Mat k0 = weight_data_r2.channel(q);
+        Mat g0 = weight_data_bf16.channel(q);
+
+        for (int p = 0; p + 3 < num_input; p += 4)
+        {
+            const float* k00 = k0.row(p);
+            const float* k01 = k0.row(p + 1);
+            const float* k02 = k0.row(p + 2);
+            const float* k03 = k0.row(p + 3);
+
+            unsigned short* g00 = g0.row<unsigned short>(p / 4);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                g00[0] = float32_to_bfloat16(k00[k]);
+                g00[1] = float32_to_bfloat16(k01[k]);
+                g00[2] = float32_to_bfloat16(k02[k]);
+                g00[3] = float32_to_bfloat16(k03[k]);
+
+                g00 += 4;
+            }
+        }
+    }
+}
+
+static void convolution_pack4to1_bf16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_bf16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        unsigned short* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const unsigned short* kptr = weight_data_bf16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const unsigned short* sptr = m.row<const unsigned short>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vcvt_f32_bf16(vld1_u16(sptr + space_ofs[k] * 4));
+                        float32x4_t _w = vcvt_f32_bf16(vld1_u16(kptr));
+                        float32x4_t _s4 = vmulq_f32(_val, _w);
+#if __aarch64__
+                        sum += vaddvq_f32(_s4); // dot
+#else
+                        float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));
+                        _ss = vpadd_f32(_ss, _ss);
+                        sum += vget_lane_f32(_ss, 0);
+#endif
+
+                        kptr += 4;
+                    }
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = float32_to_bfloat16(sum);
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_pack4to1_fp16s.h

@@ -0,0 +1,171 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack4to1_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr + space_ofs[k] * 4));
+                        float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
+                        float32x4_t _s4 = vmulq_f32(_val, _w);
+
+                        sum += vaddvq_f32(_s4); // dot
+
+                        kptr += 4;
+                    }
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = (__fp16)sum;
+            }
+
+            outptr += outw;
+        }
+    }
+}
+
+static void convolution_pack4to1_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4);
+                        float16x4_t _w = vld1_f16(kptr);
+                        float16x4_t _s4 = vmul_f16(_val, _w);
+
+                        sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot
+
+                        kptr += 4;
+                    }
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = sum;
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_pack4to8_fp16s.h

@@ -0,0 +1,98 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack4to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x8_t _sum = vdupq_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f16(bias_data_ptr + p * 8);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4);
+
+                        float16x8_t _w0 = vld1q_f16(kptr);
+                        float16x8_t _w1 = vld1q_f16(kptr + 8);
+                        float16x8_t _w2 = vld1q_f16(kptr + 16);
+                        float16x8_t _w3 = vld1q_f16(kptr + 24);
+
+                        _sum = vfmaq_lane_f16(_sum, _w0, _val, 0);
+                        _sum = vfmaq_lane_f16(_sum, _w1, _val, 1);
+                        _sum = vfmaq_lane_f16(_sum, _w2, _val, 2);
+                        _sum = vfmaq_lane_f16(_sum, _w3, _val, 3);
+
+                        kptr += 32;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1q_f16(outptr + j * 8, _sum);
+            }
+
+            outptr += outw * 8;
+        }
+    }
+}

src/layer/arm/convolution_pack8_fp16s.h

@@ -0,0 +1,106 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x8_t _sum = vdupq_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1q_f16(bias_data_ptr + p * 8);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);
+
+                        float16x8_t _w0 = vld1q_f16(kptr);
+                        float16x8_t _w1 = vld1q_f16(kptr + 8);
+                        float16x8_t _w2 = vld1q_f16(kptr + 16);
+                        float16x8_t _w3 = vld1q_f16(kptr + 24);
+                        float16x8_t _w4 = vld1q_f16(kptr + 32);
+                        float16x8_t _w5 = vld1q_f16(kptr + 40);
+                        float16x8_t _w6 = vld1q_f16(kptr + 48);
+                        float16x8_t _w7 = vld1q_f16(kptr + 56);
+
+                        _sum = vfmaq_laneq_f16(_sum, _w0, _val, 0);
+                        _sum = vfmaq_laneq_f16(_sum, _w1, _val, 1);
+                        _sum = vfmaq_laneq_f16(_sum, _w2, _val, 2);
+                        _sum = vfmaq_laneq_f16(_sum, _w3, _val, 3);
+                        _sum = vfmaq_laneq_f16(_sum, _w4, _val, 4);
+                        _sum = vfmaq_laneq_f16(_sum, _w5, _val, 5);
+                        _sum = vfmaq_laneq_f16(_sum, _w6, _val, 6);
+                        _sum = vfmaq_laneq_f16(_sum, _w7, _val, 7);
+
+                        kptr += 64;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1q_f16(outptr + j * 8, _sum);
+            }
+
+            outptr += outw * 8;
+        }
+    }
+}

src/layer/arm/convolution_pack8to1_fp16s.h

@@ -0,0 +1,93 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack8to1_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const float* bias_data_ptr = bias_data;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float sum = 0.f;
+
+                if (bias_data_ptr)
+                {
+                    sum = bias_data_ptr[p];
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);
+                        float16x8_t _w = vld1q_f16(kptr);
+                        float16x8_t _s8 = vmulq_f16(_val, _w);
+
+                        float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8));
+                        sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot
+
+                        kptr += 8;
+                    }
+                }
+
+                sum = activation_ss(sum, activation_type, activation_params);
+
+                outptr[j] = sum;
+            }
+
+            outptr += outw;
+        }
+    }
+}

src/layer/arm/convolution_pack8to4_fp16s.h

@@ -0,0 +1,106 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void convolution_pack8to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data_fp16, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, int activation_type, const Mat& activation_params, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    const __fp16* bias_data_ptr = bias_data_fp16;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                float16x4_t _sum = vdup_n_f16((__fp16)0.f);
+
+                if (bias_data_ptr)
+                {
+                    _sum = vld1_f16(bias_data_ptr + p * 4);
+                }
+
+                const __fp16* kptr = weight_data_fp16.channel(p);
+
+                // channels
+                for (int q = 0; q < channels; q++)
+                {
+                    const Mat m = bottom_blob.channel(q);
+                    const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);
+
+                        float16x4_t _w0 = vld1_f16(kptr);
+                        float16x4_t _w1 = vld1_f16(kptr + 4);
+                        float16x4_t _w2 = vld1_f16(kptr + 8);
+                        float16x4_t _w3 = vld1_f16(kptr + 12);
+                        float16x4_t _w4 = vld1_f16(kptr + 16);
+                        float16x4_t _w5 = vld1_f16(kptr + 20);
+                        float16x4_t _w6 = vld1_f16(kptr + 24);
+                        float16x4_t _w7 = vld1_f16(kptr + 28);
+
+                        _sum = vfma_laneq_f16(_sum, _w0, _val, 0);
+                        _sum = vfma_laneq_f16(_sum, _w1, _val, 1);
+                        _sum = vfma_laneq_f16(_sum, _w2, _val, 2);
+                        _sum = vfma_laneq_f16(_sum, _w3, _val, 3);
+                        _sum = vfma_laneq_f16(_sum, _w4, _val, 4);
+                        _sum = vfma_laneq_f16(_sum, _w5, _val, 5);
+                        _sum = vfma_laneq_f16(_sum, _w6, _val, 6);
+                        _sum = vfma_laneq_f16(_sum, _w7, _val, 7);
+
+                        kptr += 32;
+                    }
+                }
+
+                _sum = activation_ps(_sum, activation_type, activation_params);
+
+                vst1_f16(outptr + j * 4, _sum);
+            }
+
+            outptr += outw * 4;
+        }
+    }
+}

src/layer/arm/convolution_sgemm_pack8_fp16s.h

@@ -0,0 +1,819 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void im2col_sgemm_pack8_fp16sa_neon(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    // Mat bottom_im2col(size, maxk, inch, 16u, 8, opt.workspace_allocator);
+
+    const int size = bottom_im2col.w;
+    const int maxk = bottom_im2col.h;
+    const int inch = bottom_im2col.c;
+
+    const int outch = top_blob.c;
+
+    const __fp16* bias = _bias;
+
+    // permute
+    Mat tmp;
+    if (size >= 12)
+        tmp.create(12 * maxk, inch, size / 12 + (size % 12) / 8 + (size % 12 % 8) / 4 + (size % 12 % 4) / 2 + size % 12 % 2, 16u, 8, opt.workspace_allocator);
+    else if (size >= 8)
+        tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 16u, 8, opt.workspace_allocator);
+    else if (size >= 4)
+        tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 16u, 8, opt.workspace_allocator);
+    else if (size >= 2)
+        tmp.create(2 * maxk, inch, size / 2 + size % 2, 16u, 8, opt.workspace_allocator);
+    else
+        tmp.create(maxk, inch, size, 16u, 8, opt.workspace_allocator);
+    {
+        int nn_size = size / 12;
+        int remain_size_start = 0;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 12;
+
+            __fp16* tmpptr = tmp.channel(i / 12);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    // transpose 12x8
+                    asm volatile(
+                        "prfm   pldl1keep, [%0, #512]   \n"
+                        "ld4    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
+                        "ld4    {v4.8h, v5.8h, v6.8h, v7.8h}, [%0], #64 \n"
+                        "ld4    {v16.8h, v17.8h, v18.8h, v19.8h}, [%0] \n"
+
+                        "sub    %0, %0, #128            \n"
+
+                        "uzp1   v20.8h, v0.8h, v4.8h    \n" // 0
+                        "uzp1   v21.8h, v16.8h, v1.8h   \n" // 1
+                        "uzp1   v22.8h, v5.8h, v17.8h   \n" // 2
+                        "uzp1   v23.8h, v2.8h, v6.8h    \n" // 3
+                        "uzp1   v24.8h, v18.8h, v3.8h   \n" // 4
+                        "uzp1   v25.8h, v7.8h, v19.8h   \n" // 5
+                        "uzp2   v26.8h, v0.8h, v4.8h    \n" // 6
+                        "uzp2   v27.8h, v16.8h, v1.8h   \n" // 7
+                        "uzp2   v28.8h, v5.8h, v17.8h   \n" // 8
+                        "uzp2   v29.8h, v2.8h, v6.8h    \n" // 9
+                        "uzp2   v30.8h, v18.8h, v3.8h   \n" // 10
+                        "uzp2   v31.8h, v7.8h, v19.8h   \n" // 11
+
+                        "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
+                        "st1    {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n"
+                        "st1    {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n"
+                        : "=r"(img0),  // %0
+                        "=r"(tmpptr) // %1
+                        : "0"(img0),
+                        "1"(tmpptr)
+                        : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
+                    img0 += size * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size * 12;
+        nn_size = (size - remain_size_start) >> 3;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 8;
+
+            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    // transpose 8x8
+                    asm volatile(
+                        "prfm   pldl1keep, [%0, #512]   \n"
+                        "ld4    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
+                        "ld4    {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n"
+                        "sub    %0, %0, #64             \n"
+
+                        "uzp1   v16.8h, v0.8h, v4.8h    \n"
+                        "uzp2   v20.8h, v0.8h, v4.8h    \n"
+                        "uzp1   v17.8h, v1.8h, v5.8h    \n"
+                        "uzp2   v21.8h, v1.8h, v5.8h    \n"
+                        "uzp1   v18.8h, v2.8h, v6.8h    \n"
+                        "uzp2   v22.8h, v2.8h, v6.8h    \n"
+                        "uzp1   v19.8h, v3.8h, v7.8h    \n"
+                        "uzp2   v23.8h, v3.8h, v7.8h    \n"
+
+                        "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
+                        "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
+                        : "=r"(img0),  // %0
+                        "=r"(tmpptr) // %1
+                        : "0"(img0),
+                        "1"(tmpptr)
+                        : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+                    img0 += size * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 3;
+        nn_size = (size - remain_size_start) >> 2;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 4;
+
+            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    asm volatile(
+                        "prfm   pldl1keep, [%0, #512]   \n"
+                        "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n"
+                        "st1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n"
+                        : "=r"(img0),  // %0
+                        "=r"(tmpptr) // %1
+                        : "0"(img0),
+                        "1"(tmpptr)
+                        : "memory", "v0", "v1", "v2", "v3");
+                    img0 += size * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 2;
+        nn_size = (size - remain_size_start) >> 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 2;
+
+            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    asm volatile(
+                        "prfm   pldl1keep, [%0, #256]   \n"
+                        "ld1    {v0.8h, v1.8h}, [%0]    \n"
+                        "st1    {v0.8h, v1.8h}, [%1], #32 \n"
+                        : "=r"(img0),  // %0
+                        "=r"(tmpptr) // %1
+                        : "0"(img0),
+                        "1"(tmpptr)
+                        : "memory", "v0", "v1");
+                    img0 += size * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i = remain_size_start; i < size; i++)
+        {
+            __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * 8;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    asm volatile(
+                        "prfm   pldl1keep, [%0, #128]   \n"
+                        "ld1    {v0.8h}, [%0]           \n"
+                        "st1    {v0.8h}, [%1], #16      \n"
+                        : "=r"(img0),  // %0
+                        "=r"(tmpptr) // %1
+                        : "0"(img0),
+                        "1"(tmpptr)
+                        : "memory", "v0");
+                    img0 += size * 8;
+                }
+            }
+        }
+    }
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr0 = top_blob.channel(p);
+
+        const __fp16 zeros[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
+        const __fp16* biasptr = bias ? bias + p * 8 : zeros;
+
+        int i = 0;
+        for (; i + 11 < size; i += 12)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 12);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk; // inch always > 0
+
+            asm volatile(
+                "ld1    {v20.8h}, [%8]              \n"
+                "mov    v21.16b, v20.16b            \n"
+                "mov    v22.16b, v20.16b            \n"
+                "mov    v23.16b, v20.16b            \n"
+                "mov    v24.16b, v20.16b            \n"
+                "mov    v25.16b, v20.16b            \n"
+                "mov    v26.16b, v20.16b            \n"
+                "mov    v27.16b, v20.16b            \n"
+                "mov    v28.16b, v20.16b            \n"
+                "mov    v29.16b, v20.16b            \n"
+                "mov    v30.16b, v20.16b            \n"
+                "mov    v31.16b, v20.16b            \n"
+
+                "0:                                 \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w0123
+
+                "fmla   v20.8h, v12.8h, v0.h[0]     \n"
+                "fmla   v21.8h, v12.8h, v0.h[1]     \n"
+                "fmla   v22.8h, v12.8h, v0.h[2]     \n"
+                "fmla   v23.8h, v12.8h, v0.h[3]     \n"
+                "fmla   v24.8h, v12.8h, v0.h[4]     \n"
+                "fmla   v25.8h, v12.8h, v0.h[5]     \n"
+                "fmla   v26.8h, v12.8h, v0.h[6]     \n"
+                "fmla   v27.8h, v12.8h, v0.h[7]     \n"
+                "fmla   v28.8h, v12.8h, v1.h[0]     \n"
+                "fmla   v29.8h, v12.8h, v1.h[1]     \n"
+                "fmla   v30.8h, v12.8h, v1.h[2]     \n"
+                "fmla   v31.8h, v12.8h, v1.h[3]     \n"
+
+                "fmla   v20.8h, v13.8h, v1.h[4]     \n"
+                "fmla   v21.8h, v13.8h, v1.h[5]     \n"
+                "fmla   v22.8h, v13.8h, v1.h[6]     \n"
+                "fmla   v23.8h, v13.8h, v1.h[7]     \n"
+                "fmla   v24.8h, v13.8h, v2.h[0]     \n"
+                "fmla   v25.8h, v13.8h, v2.h[1]     \n"
+                "fmla   v26.8h, v13.8h, v2.h[2]     \n"
+                "fmla   v27.8h, v13.8h, v2.h[3]     \n"
+                "fmla   v28.8h, v13.8h, v2.h[4]     \n"
+                "fmla   v29.8h, v13.8h, v2.h[5]     \n"
+                "fmla   v30.8h, v13.8h, v2.h[6]     \n"
+                "fmla   v31.8h, v13.8h, v2.h[7]     \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567
+
+                "fmla   v20.8h, v14.8h, v3.h[0]     \n"
+                "fmla   v21.8h, v14.8h, v3.h[1]     \n"
+                "fmla   v22.8h, v14.8h, v3.h[2]     \n"
+                "fmla   v23.8h, v14.8h, v3.h[3]     \n"
+                "fmla   v24.8h, v14.8h, v3.h[4]     \n"
+                "fmla   v25.8h, v14.8h, v3.h[5]     \n"
+                "fmla   v26.8h, v14.8h, v3.h[6]     \n"
+                "fmla   v27.8h, v14.8h, v3.h[7]     \n"
+                "fmla   v28.8h, v14.8h, v4.h[0]     \n"
+                "fmla   v29.8h, v14.8h, v4.h[1]     \n"
+                "fmla   v30.8h, v14.8h, v4.h[2]     \n"
+                "fmla   v31.8h, v14.8h, v4.h[3]     \n"
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%3], #64 \n" // w4567
+
+                "fmla   v20.8h, v15.8h, v4.h[4]     \n"
+                "fmla   v21.8h, v15.8h, v4.h[5]     \n"
+                "fmla   v22.8h, v15.8h, v4.h[6]     \n"
+                "fmla   v23.8h, v15.8h, v4.h[7]     \n"
+                "fmla   v24.8h, v15.8h, v5.h[0]     \n"
+                "fmla   v25.8h, v15.8h, v5.h[1]     \n"
+                "fmla   v26.8h, v15.8h, v5.h[2]     \n"
+                "fmla   v27.8h, v15.8h, v5.h[3]     \n"
+                "fmla   v28.8h, v15.8h, v5.h[4]     \n"
+                "fmla   v29.8h, v15.8h, v5.h[5]     \n"
+                "fmla   v30.8h, v15.8h, v5.h[6]     \n"
+                "fmla   v31.8h, v15.8h, v5.h[7]     \n"
+
+                "fmla   v20.8h, v16.8h, v6.h[0]     \n"
+                "fmla   v21.8h, v16.8h, v6.h[1]     \n"
+                "fmla   v22.8h, v16.8h, v6.h[2]     \n"
+                "fmla   v23.8h, v16.8h, v6.h[3]     \n"
+                "fmla   v24.8h, v16.8h, v6.h[4]     \n"
+                "fmla   v25.8h, v16.8h, v6.h[5]     \n"
+                "fmla   v26.8h, v16.8h, v6.h[6]     \n"
+                "fmla   v27.8h, v16.8h, v6.h[7]     \n"
+                "fmla   v28.8h, v16.8h, v7.h[0]     \n"
+                "fmla   v29.8h, v16.8h, v7.h[1]     \n"
+                "fmla   v30.8h, v16.8h, v7.h[2]     \n"
+                "fmla   v31.8h, v16.8h, v7.h[3]     \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%2], #64 \n" // r891011
+
+                "fmla   v20.8h, v17.8h, v7.h[4]     \n"
+                "fmla   v21.8h, v17.8h, v7.h[5]     \n"
+                "fmla   v22.8h, v17.8h, v7.h[6]     \n"
+                "fmla   v23.8h, v17.8h, v7.h[7]     \n"
+                "fmla   v24.8h, v17.8h, v8.h[0]     \n"
+                "fmla   v25.8h, v17.8h, v8.h[1]     \n"
+                "fmla   v26.8h, v17.8h, v8.h[2]     \n"
+                "fmla   v27.8h, v17.8h, v8.h[3]     \n"
+                "fmla   v28.8h, v17.8h, v8.h[4]     \n"
+                "fmla   v29.8h, v17.8h, v8.h[5]     \n"
+                "fmla   v30.8h, v17.8h, v8.h[6]     \n"
+                "fmla   v31.8h, v17.8h, v8.h[7]     \n"
+
+                "fmla   v20.8h, v18.8h, v9.h[0]     \n"
+                "fmla   v21.8h, v18.8h, v9.h[1]     \n"
+                "fmla   v22.8h, v18.8h, v9.h[2]     \n"
+                "fmla   v23.8h, v18.8h, v9.h[3]     \n"
+                "fmla   v24.8h, v18.8h, v9.h[4]     \n"
+                "fmla   v25.8h, v18.8h, v9.h[5]     \n"
+                "fmla   v26.8h, v18.8h, v9.h[6]     \n"
+                "fmla   v27.8h, v18.8h, v9.h[7]     \n"
+                "fmla   v28.8h, v18.8h, v10.h[0]    \n"
+                "fmla   v29.8h, v18.8h, v10.h[1]    \n"
+                "fmla   v30.8h, v18.8h, v10.h[2]    \n"
+                "fmla   v31.8h, v18.8h, v10.h[3]    \n"
+
+                "subs   %w0, %w0, #1                \n"
+
+                "fmla   v20.8h, v19.8h, v10.h[4]    \n"
+                "fmla   v21.8h, v19.8h, v10.h[5]    \n"
+                "fmla   v22.8h, v19.8h, v10.h[6]    \n"
+                "fmla   v23.8h, v19.8h, v10.h[7]    \n"
+                "fmla   v24.8h, v19.8h, v11.h[0]    \n"
+                "fmla   v25.8h, v19.8h, v11.h[1]    \n"
+                "fmla   v26.8h, v19.8h, v11.h[2]    \n"
+                "fmla   v27.8h, v19.8h, v11.h[3]    \n"
+                "fmla   v28.8h, v19.8h, v11.h[4]    \n"
+                "fmla   v29.8h, v19.8h, v11.h[5]    \n"
+                "fmla   v30.8h, v19.8h, v11.h[6]    \n"
+                "fmla   v31.8h, v19.8h, v11.h[7]    \n"
+
+                "bne    0b                          \n"
+
+                "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
+                "st1    {v24.8h, v25.8h, v26.8h, v27.8h}, [%1], #64 \n"
+                "st1    {v28.8h, v29.8h, v30.8h, v31.8h}, [%1], #64 \n"
+
+                : "=r"(nn),      // %0
+                "=r"(outptr0), // %1
+                "=r"(tmpptr),  // %2
+                "=r"(kptr0)    // %3
+                : "0"(nn),
+                "1"(outptr0),
+                "2"(tmpptr),
+                "3"(kptr0),
+                "r"(biasptr) // %8
+                : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
+        }
+        for (; i + 7 < size; i += 8)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk; // inch always > 0
+
+            asm volatile(
+                "ld1    {v16.8h}, [%8]              \n"
+                "mov    v17.16b, v16.16b            \n"
+                "mov    v18.16b, v16.16b            \n"
+                "mov    v19.16b, v16.16b            \n"
+                "mov    v20.16b, v16.16b            \n"
+                "mov    v21.16b, v16.16b            \n"
+                "mov    v22.16b, v16.16b            \n"
+                "mov    v23.16b, v16.16b            \n"
+
+                "0:                                 \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
+
+                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
+                "fmla   v17.8h, v8.8h, v0.h[1]      \n"
+                "fmla   v18.8h, v8.8h, v0.h[2]      \n"
+                "fmla   v19.8h, v8.8h, v0.h[3]      \n"
+                "fmla   v20.8h, v8.8h, v0.h[4]      \n"
+                "fmla   v21.8h, v8.8h, v0.h[5]      \n"
+                "fmla   v22.8h, v8.8h, v0.h[6]      \n"
+                "fmla   v23.8h, v8.8h, v0.h[7]      \n"
+
+                "fmla   v16.8h, v9.8h, v1.h[0]      \n"
+                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
+                "fmla   v18.8h, v9.8h, v1.h[2]      \n"
+                "fmla   v19.8h, v9.8h, v1.h[3]      \n"
+                "fmla   v20.8h, v9.8h, v1.h[4]      \n"
+                "fmla   v21.8h, v9.8h, v1.h[5]      \n"
+                "fmla   v22.8h, v9.8h, v1.h[6]      \n"
+                "fmla   v23.8h, v9.8h, v1.h[7]      \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v4.8h, v5.8h, v6.8h, v7.8h}, [%2], #64 \n" // r4567
+
+                "fmla   v16.8h, v10.8h, v2.h[0]     \n"
+                "fmla   v17.8h, v10.8h, v2.h[1]     \n"
+                "fmla   v18.8h, v10.8h, v2.h[2]     \n"
+                "fmla   v19.8h, v10.8h, v2.h[3]     \n"
+                "fmla   v20.8h, v10.8h, v2.h[4]     \n"
+                "fmla   v21.8h, v10.8h, v2.h[5]     \n"
+                "fmla   v22.8h, v10.8h, v2.h[6]     \n"
+                "fmla   v23.8h, v10.8h, v2.h[7]     \n"
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
+
+                "fmla   v16.8h, v11.8h, v3.h[0]     \n"
+                "fmla   v17.8h, v11.8h, v3.h[1]     \n"
+                "fmla   v18.8h, v11.8h, v3.h[2]     \n"
+                "fmla   v19.8h, v11.8h, v3.h[3]     \n"
+                "fmla   v20.8h, v11.8h, v3.h[4]     \n"
+                "fmla   v21.8h, v11.8h, v3.h[5]     \n"
+                "fmla   v22.8h, v11.8h, v3.h[6]     \n"
+                "fmla   v23.8h, v11.8h, v3.h[7]     \n"
+
+                "fmla   v16.8h, v12.8h, v4.h[0]     \n"
+                "fmla   v17.8h, v12.8h, v4.h[1]     \n"
+                "fmla   v18.8h, v12.8h, v4.h[2]     \n"
+                "fmla   v19.8h, v12.8h, v4.h[3]     \n"
+                "fmla   v20.8h, v12.8h, v4.h[4]     \n"
+                "fmla   v21.8h, v12.8h, v4.h[5]     \n"
+                "fmla   v22.8h, v12.8h, v4.h[6]     \n"
+                "fmla   v23.8h, v12.8h, v4.h[7]     \n"
+
+                "fmla   v16.8h, v13.8h, v5.h[0]     \n"
+                "fmla   v17.8h, v13.8h, v5.h[1]     \n"
+                "fmla   v18.8h, v13.8h, v5.h[2]     \n"
+                "fmla   v19.8h, v13.8h, v5.h[3]     \n"
+                "fmla   v20.8h, v13.8h, v5.h[4]     \n"
+                "fmla   v21.8h, v13.8h, v5.h[5]     \n"
+                "fmla   v22.8h, v13.8h, v5.h[6]     \n"
+                "fmla   v23.8h, v13.8h, v5.h[7]     \n"
+
+                "fmla   v16.8h, v14.8h, v6.h[0]     \n"
+                "fmla   v17.8h, v14.8h, v6.h[1]     \n"
+                "fmla   v18.8h, v14.8h, v6.h[2]     \n"
+                "fmla   v19.8h, v14.8h, v6.h[3]     \n"
+                "fmla   v20.8h, v14.8h, v6.h[4]     \n"
+                "fmla   v21.8h, v14.8h, v6.h[5]     \n"
+                "fmla   v22.8h, v14.8h, v6.h[6]     \n"
+                "fmla   v23.8h, v14.8h, v6.h[7]     \n"
+
+                "subs   %w0, %w0, #1                \n"
+
+                "fmla   v16.8h, v15.8h, v7.h[0]     \n"
+                "fmla   v17.8h, v15.8h, v7.h[1]     \n"
+                "fmla   v18.8h, v15.8h, v7.h[2]     \n"
+                "fmla   v19.8h, v15.8h, v7.h[3]     \n"
+                "fmla   v20.8h, v15.8h, v7.h[4]     \n"
+                "fmla   v21.8h, v15.8h, v7.h[5]     \n"
+                "fmla   v22.8h, v15.8h, v7.h[6]     \n"
+                "fmla   v23.8h, v15.8h, v7.h[7]     \n"
+
+                "bne    0b                          \n"
+
+                "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
+                "st1    {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
+
+                : "=r"(nn),      // %0
+                "=r"(outptr0), // %1
+                "=r"(tmpptr),  // %2
+                "=r"(kptr0)    // %3
+                : "0"(nn),
+                "1"(outptr0),
+                "2"(tmpptr),
+                "3"(kptr0),
+                "r"(biasptr) // %8
+                : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+        }
+        for (; i + 3 < size; i += 4)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk; // inch always > 0
+
+            asm volatile(
+                "ld1    {v16.8h}, [%8]              \n"
+                "mov    v17.16b, v16.16b            \n"
+                "mov    v18.16b, v16.16b            \n"
+                "mov    v19.16b, v16.16b            \n"
+
+                "0:                                 \n"
+
+                "prfm   pldl1keep, [%2, #512]       \n"
+                "ld1    {v0.8h, v1.8h, v2.8h, v3.8h}, [%2], #64 \n" // r0123
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
+
+                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
+                "fmla   v17.8h, v8.8h, v1.h[0]      \n"
+                "fmla   v18.8h, v8.8h, v2.h[0]      \n"
+                "fmla   v19.8h, v8.8h, v3.h[0]      \n"
+
+                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
+                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
+                "fmla   v18.8h, v9.8h, v2.h[1]      \n"
+                "fmla   v19.8h, v9.8h, v3.h[1]      \n"
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
+
+                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
+                "fmla   v17.8h, v10.8h, v1.h[2]     \n"
+                "fmla   v18.8h, v10.8h, v2.h[2]     \n"
+                "fmla   v19.8h, v10.8h, v3.h[2]     \n"
+
+                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
+                "fmla   v17.8h, v11.8h, v1.h[3]     \n"
+                "fmla   v18.8h, v11.8h, v2.h[3]     \n"
+                "fmla   v19.8h, v11.8h, v3.h[3]     \n"
+
+                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
+                "fmla   v17.8h, v12.8h, v1.h[4]     \n"
+                "fmla   v18.8h, v12.8h, v2.h[4]     \n"
+                "fmla   v19.8h, v12.8h, v3.h[4]     \n"
+
+                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
+                "fmla   v17.8h, v13.8h, v1.h[5]     \n"
+                "fmla   v18.8h, v13.8h, v2.h[5]     \n"
+                "fmla   v19.8h, v13.8h, v3.h[5]     \n"
+
+                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
+                "fmla   v17.8h, v14.8h, v1.h[6]     \n"
+                "fmla   v18.8h, v14.8h, v2.h[6]     \n"
+                "fmla   v19.8h, v14.8h, v3.h[6]     \n"
+
+                "subs   %w0, %w0, #1                \n"
+
+                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
+                "fmla   v17.8h, v15.8h, v1.h[7]     \n"
+                "fmla   v18.8h, v15.8h, v2.h[7]     \n"
+                "fmla   v19.8h, v15.8h, v3.h[7]     \n"
+
+                "bne    0b                          \n"
+
+                "st1    {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
+
+                : "=r"(nn),      // %0
+                "=r"(outptr0), // %1
+                "=r"(tmpptr),  // %2
+                "=r"(kptr0)    // %3
+                : "0"(nn),
+                "1"(outptr0),
+                "2"(tmpptr),
+                "3"(kptr0),
+                "r"(biasptr) // %8
+                : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19");
+        }
+        for (; i + 1 < size; i += 2)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk; // inch always > 0
+
+            asm volatile(
+                "ld1    {v16.8h}, [%8]              \n"
+                "mov    v17.16b, v16.16b            \n"
+
+                "0:                                 \n"
+
+                "prfm   pldl1keep, [%2, #256]       \n"
+                "ld1    {v0.8h, v1.8h}, [%2], #32   \n" // r01
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
+
+                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
+                "fmla   v17.8h, v8.8h, v1.h[0]      \n"
+
+                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
+                "fmla   v17.8h, v9.8h, v1.h[1]      \n"
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
+
+                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
+                "fmla   v17.8h, v10.8h, v1.h[2]     \n"
+
+                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
+                "fmla   v17.8h, v11.8h, v1.h[3]     \n"
+
+                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
+                "fmla   v17.8h, v12.8h, v1.h[4]     \n"
+
+                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
+                "fmla   v17.8h, v13.8h, v1.h[5]     \n"
+
+                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
+                "fmla   v17.8h, v14.8h, v1.h[6]     \n"
+
+                "subs   %w0, %w0, #1                \n"
+
+                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
+                "fmla   v17.8h, v15.8h, v1.h[7]     \n"
+
+                "bne    0b                          \n"
+
+                "st1    {v16.8h, v17.8h}, [%1], #32 \n"
+
+                : "=r"(nn),      // %0
+                "=r"(outptr0), // %1
+                "=r"(tmpptr),  // %2
+                "=r"(kptr0)    // %3
+                : "0"(nn),
+                "1"(outptr0),
+                "2"(tmpptr),
+                "3"(kptr0),
+                "r"(biasptr) // %8
+                : "cc", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17");
+        }
+        for (; i < size; i++)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk; // inch always > 0
+
+            asm volatile(
+                "ld1    {v16.8h}, [%8]              \n"
+
+                "0:                                 \n"
+
+                "prfm   pldl1keep, [%2, #128]       \n"
+                "ld1    {v0.8h}, [%2], #16          \n" // r0
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v8.8h, v9.8h, v10.8h, v11.8h}, [%3], #64 \n" // w0123
+
+                "fmla   v16.8h, v8.8h, v0.h[0]      \n"
+                "fmla   v16.8h, v9.8h, v0.h[1]      \n"
+
+                "prfm   pldl1keep, [%3, #512]       \n"
+                "ld1    {v12.8h, v13.8h, v14.8h, v15.8h}, [%3], #64 \n" // w4567
+
+                "fmla   v16.8h, v10.8h, v0.h[2]     \n"
+                "fmla   v16.8h, v11.8h, v0.h[3]     \n"
+
+                "fmla   v16.8h, v12.8h, v0.h[4]     \n"
+                "fmla   v16.8h, v13.8h, v0.h[5]     \n"
+
+                "subs   %w0, %w0, #1                \n"
+
+                "fmla   v16.8h, v14.8h, v0.h[6]     \n"
+                "fmla   v16.8h, v15.8h, v0.h[7]     \n"
+
+                "bne    0b                          \n"
+
+                "st1    {v16.8h}, [%1], #16         \n"
+
+                : "=r"(nn),      // %0
+                "=r"(outptr0), // %1
+                "=r"(tmpptr),  // %2
+                "=r"(kptr0)    // %3
+                : "0"(nn),
+                "1"(outptr0),
+                "2"(tmpptr),
+                "3"(kptr0),
+                "r"(biasptr) // %8
+                : "cc", "memory", "v0", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16");
+        }
+    }
+}
+
+static void convolution_im2col_sgemm_transform_kernel_pack8_fp16sa_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h)
+{
+    const int maxk = kernel_w * kernel_h;
+
+    // interleave
+    // src = maxk-inch-outch
+    // dst = 8b-8a-maxk-inch/8a-outch/8b
+    Mat kernel = _kernel.reshape(maxk, inch, outch);
+    kernel_tm.create(64 * maxk, inch / 8, outch / 8, 2u);
+
+    for (int q = 0; q + 7 < outch; q += 8)
+    {
+        Mat g0 = kernel_tm.channel(q / 8);
+
+        for (int p = 0; p + 7 < inch; p += 8)
+        {
+            __fp16* g00 = g0.row<__fp16>(p / 8);
+
+            for (int k = 0; k < maxk; k++)
+            {
+                for (int i = 0; i < 8; i++)
+                {
+                    for (int j = 0; j < 8; j++)
+                    {
+                        const float* k00 = kernel.channel(q + j).row(p + i);
+
+                        g00[0] = (__fp16)k00[k];
+
+                        g00++;
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void convolution_im2col_sgemm_pack8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    const int size = outw * outh;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // im2col
+    Mat bottom_im2col(size, maxk, inch, 16u, 8, opt.workspace_allocator);
+    {
+        const int gap = (w * stride_h - outw * stride_w) * 8;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < inch; p++)
+        {
+            const Mat img = bottom_blob.channel(p);
+            __fp16* ptr = bottom_im2col.channel(p);
+
+            for (int u = 0; u < kernel_h; u++)
+            {
+                for (int v = 0; v < kernel_w; v++)
+                {
+                    const __fp16* sptr = img.row<const __fp16>(dilation_h * u) + dilation_w * v * 8;
+
+                    for (int i = 0; i < outh; i++)
+                    {
+                        int j = 0;
+                        for (; j + 3 < outw; j += 4)
+                        {
+                            float16x8_t _val0 = vld1q_f16(sptr);
+                            float16x8_t _val1 = vld1q_f16(sptr + stride_w * 8);
+                            float16x8_t _val2 = vld1q_f16(sptr + stride_w * 16);
+                            float16x8_t _val3 = vld1q_f16(sptr + stride_w * 24);
+                            vst1q_f16(ptr, _val0);
+                            vst1q_f16(ptr + 8, _val1);
+                            vst1q_f16(ptr + 16, _val2);
+                            vst1q_f16(ptr + 24, _val3);
+
+                            sptr += stride_w * 32;
+                            ptr += 32;
+                        }
+                        for (; j + 1 < outw; j += 2)
+                        {
+                            float16x8_t _val0 = vld1q_f16(sptr);
+                            float16x8_t _val1 = vld1q_f16(sptr + stride_w * 8);
+                            vst1q_f16(ptr, _val0);
+                            vst1q_f16(ptr + 8, _val1);
+
+                            sptr += stride_w * 16;
+                            ptr += 16;
+                        }
+                        for (; j < outw; j++)
+                        {
+                            float16x8_t _val = vld1q_f16(sptr);
+                            vst1q_f16(ptr, _val);
+
+                            sptr += stride_w * 8;
+                            ptr += 4;
+                        }
+
+                        sptr += gap;
+                    }
+                }
+            }
+        }
+    }
+
+    im2col_sgemm_pack8_fp16sa_neon(bottom_im2col, top_blob, kernel, _bias, opt);
+}

tests/test_convolution.cpp

@@ -117,7 +117,8 @@ static int test_convolution_0()
            || test_convolution(13, 16, 16, 24, 3, 1, 1, 1, 1)
            || test_convolution(8, 8, 16, 24, 3, 1, 1, 1, 0)
            || test_convolution(4, 8, 16, 24, 3, 1, 1, 1, 1)
-           || test_convolution(4, 20, 16, 24, 3, 1, 1, 1, 0);
+           || test_convolution(4, 20, 16, 24, 3, 1, 1, 1, 0)
+           || test_convolution(6, 7, 64, 64, 3, 1, 2, 0, 1);
 }
 
 static int test_convolution_vec(int w, int outch, int kernel, int dilation, int stride, int pad, int bias)