From 790829bc62f487cccc36d7040eeb0ed55fc831d7 Mon Sep 17 00:00:00 2001
From: nihui <shuizhuyuanluo@126.com>
Date: Sun, 12 Nov 2017 13:56:46 +0800
Subject: [PATCH] partition dot tiles and reuse kernel register, over 20%
 improvement for tiny image

---
 src/layer/arm/convolution_3x3.h   | 950 ++++++++++++++++++++++++++++++
 src/layer/arm/convolution_arm.cpp |  22 +-
 2 files changed, 971 insertions(+), 1 deletion(-)

diff --git a/src/layer/arm/convolution_3x3.h b/src/layer/arm/convolution_3x3.h
index 52fba9bca..8bfaddda1 100644
--- a/src/layer/arm/convolution_3x3.h
+++ b/src/layer/arm/convolution_3x3.h
@@ -1303,6 +1303,956 @@ static void conv3x3s1_winograd64_neon(const Mat& bottom_blob, Mat& top_blob, con
     copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w);
 }
 
+static void conv3x3s1_winograd64_neon2(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& _bias)
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    // pad to 6n+2
+    Mat bottom_blob_bordered = bottom_blob;
+
+    outw = (outw + 5) / 6 * 6;
+    outh = (outh + 5) / 6 * 6;
+
+    w = outw + 2;
+    h = outh + 2;
+    copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f);
+
+    const float* bias = _bias;
+
+    // BEGIN transform input
+    Mat bottom_blob_tm;
+    {
+        int w_tm = outw / 6 * 8;
+        int h_tm = outh / 6 * 8;
+        bottom_blob_tm.create(2*8, 4 * w_tm/8 * h_tm/8, inch);
+        const int tiles = w_tm/8 * h_tm/8;
+
+//         const float itm[8][8] = {
+//             {1.0f,  0.0f, -5.25f,  0.00f,  5.25f,  0.00f, -1.0f, 0.0f},
+//
+//             {0.0f,  1.0f,  1.00f, -4.25f, -4.25f,  1.00f,  1.0f, 0.0f},
+//             {0.0f, -1.0f,  1.00f,  4.25f, -4.25f, -1.00f,  1.0f, 0.0f},
+//
+//             {0.0f,  0.5f,  0.25f, -2.50f, -1.25f,  2.00f,  1.0f, 0.0f},
+//             {0.0f, -0.5f,  0.25f,  2.50f, -1.25f, -2.00f,  1.0f, 0.0f},
+//
+//             {0.0f,  2.0f,  4.00f, -2.50f, -5.00f,  0.50f,  1.0f, 0.0f},
+//             {0.0f, -2.0f,  4.00f,  2.50f, -5.00f, -0.50f,  1.0f, 0.0f},
+//
+//             {0.0f, -1.0f,  0.00f,  5.25f,  0.00f, -5.25f,  0.0f, 1.0f}
+//         };
+
+        // 0 = r00 - r06 + (r04 - r02) * 5.25
+        // 7 = r07 - r01 + (r03 - r05) * 5.25
+
+        // 1 = (r02 + r06 - r04 * 4.25) + (r01 - r03 * 4.25 + r05)
+        // 2 = (r02 + r06 - r04 * 4.25) - (r01 - r03 * 4.25 + r05)
+
+        // 3 = (r06 + r02 * 0.25 - r04 * 1.25) + (r01 * 0.5 - r03 * 2.5 + r05 * 2)
+        // 4 = (r06 + r02 * 0.25 - r04 * 1.25) - (r01 * 0.5 - r03 * 2.5 + r05 * 2)
+
+        // reuse r04 * 1.25
+        // reuse r03 * 2.5
+        // 5 = (r06 + (r02 - r04 * 1.25) * 4) + (r01 * 2 - r03 * 2.5 + r05 * 0.5)
+        // 6 = (r06 + (r02 - r04 * 1.25) * 4) - (r01 * 2 - r03 * 2.5 + r05 * 0.5)
+
+        #pragma omp parallel for
+        for (int q = 0; q<inch; q++)
+        {
+            const Mat img0 = bottom_blob_bordered.channel(q);
+            Mat img0_tm = bottom_blob_tm.channel(q);
+
+            float tmp[8][8];
+
+            // tile
+            for (int i=0; i<h_tm/8; i++)
+            {
+                for (int j=0; j<w_tm/8; j++)
+                {
+                    const float* r0 = img0.row(i * 6) + j * 6;
+                    float* r0_tm01 = img0_tm.row(i * w_tm/8 + j);
+                    float* r0_tm23 = img0_tm.row(tiles + i * w_tm/8 + j);
+                    float* r0_tm45 = img0_tm.row(tiles * 2 + i * w_tm/8 + j);
+                    float* r0_tm67 = img0_tm.row(tiles * 3 + i * w_tm/8 + j);
+
+                    for (int m=0; m<8; m++)
+                    {
+                        tmp[0][m] = r0[0] - r0[6] + (r0[4] - r0[2]) * 5.25;
+                        tmp[7][m] = r0[7] - r0[1] + (r0[3] - r0[5]) * 5.25;
+
+                        float tmp12a = (r0[2] + r0[6] - r0[4] * 4.25);
+                        float tmp12b = (r0[1] + r0[5] - r0[3] * 4.25);
+
+                        tmp[1][m] = tmp12a + tmp12b;
+                        tmp[2][m] = tmp12a - tmp12b;
+
+                        float tmp34a = (r0[6] + r0[2] * 0.25 - r0[4] * 1.25);
+                        float tmp34b = (r0[1] * 0.5 - r0[3] * 2.5 + r0[5] * 2);
+
+                        tmp[3][m] = tmp34a + tmp34b;
+                        tmp[4][m] = tmp34a - tmp34b;
+
+                        float tmp56a = (r0[6] + (r0[2] - r0[4] * 1.25) * 4);
+                        float tmp56b = (r0[1] * 2 - r0[3] * 2.5 + r0[5] * 0.5);
+
+                        tmp[5][m] = tmp56a + tmp56b;
+                        tmp[6][m] = tmp56a - tmp56b;
+
+                        r0 += w;
+                    }
+
+                    float* r0_tms[4] = { r0_tm01, r0_tm23, r0_tm45, r0_tm67 };
+
+                    for (int m=0; m<8; m++)
+                    {
+                        const float* tmp0 = tmp[m];
+
+                        float* r0_tm = r0_tms[m/2] + (m%2) * 8;
+
+                        r0_tm[0] = tmp0[0] - tmp0[6] + (tmp0[4] - tmp0[2]) * 5.25;
+                        r0_tm[7] = tmp0[7] - tmp0[1] + (tmp0[3] - tmp0[5]) * 5.25;
+
+                        float tmp12a = (tmp0[2] + tmp0[6] - tmp0[4] * 4.25);
+                        float tmp12b = (tmp0[1] - tmp0[3] * 4.25 + tmp0[5]);
+
+                        r0_tm[1] = tmp12a + tmp12b;
+                        r0_tm[2] = tmp12a - tmp12b;
+
+                        float tmp34a = (tmp0[6] + tmp0[2] * 0.25 - tmp0[4] * 1.25);
+                        float tmp34b = (tmp0[1] * 0.5 - tmp0[3] * 2.5 + tmp0[5] * 2);
+
+                        r0_tm[3] = tmp34a + tmp34b;
+                        r0_tm[4] = tmp34a - tmp34b;
+
+                        float tmp56a = (tmp0[6] + (tmp0[2] - tmp0[4] * 1.25) * 4);
+                        float tmp56b = (tmp0[1] * 2 - tmp0[3] * 2.5 + tmp0[5] * 0.5);
+
+                        r0_tm[5] = tmp56a + tmp56b;
+                        r0_tm[6] = tmp56a - tmp56b;
+                    }
+                }
+            }
+        }
+
+    }
+    // END transform input
+
+    // BEGIN dot
+    Mat top_blob_tm;
+    {
+        int w_tm = outw / 6 * 8;
+        int h_tm = outh / 6 * 8;
+        top_blob_tm.create(2*8, 4 * w_tm/8 * h_tm/8, outch);
+
+        const int tiles = h_tm/8 * w_tm/8;
+
+        #pragma omp parallel for
+        for (int p = 0; p<outch; p++)
+        {
+            Mat out0_tm = top_blob_tm.channel(p);
+            const Mat kernel0_tm = kernel_tm.channel(p);
+
+            out0_tm.fill(0.f);
+
+            int q = 0;
+            for (; q+1<inch; q+=2)
+            {
+                const float* r0 = bottom_blob_tm.channel(q);
+                const float* r1 = bottom_blob_tm.channel(q+1);
+
+                const float* k0 = kernel0_tm.row(q);
+                const float* k1 = kernel0_tm.row(q+1);
+
+                float* output0_tm = out0_tm;
+
+                for (int r=0; r<4; r++)
+                {
+#if __ARM_NEON
+#if __aarch64__
+                float32x4_t _k0 = vld1q_f32(k0);
+                float32x4_t _k0n = vld1q_f32(k0+4);
+                float32x4_t _k0nn = vld1q_f32(k0+8);
+                float32x4_t _k0nnn = vld1q_f32(k0+12);
+                float32x4_t _k1 = vld1q_f32(k1);
+                float32x4_t _k1n = vld1q_f32(k1+4);
+                float32x4_t _k1nn = vld1q_f32(k1+8);
+                float32x4_t _k1nnn = vld1q_f32(k1+12);
+#else
+                float32x4_t _k0;
+                float32x4_t _k0n;
+                float32x4_t _k0nn;
+                float32x4_t _k0nnn;
+                float32x4_t _k1;
+                float32x4_t _k1n;
+                float32x4_t _k1nn;
+                float32x4_t _k1nnn;
+
+                asm volatile(
+                    "pld        [%0, #512]              \n"
+                    "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"
+                    "pld        [%1, #512]              \n"
+                    "vld1.f32   {%e4-%f4}, [%1 :128]!   \n"
+
+                    "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"
+                    "vld1.f32   {%e5-%f5}, [%1 :128]!   \n"
+
+                    "vld1.f32   {%e6-%f6}, [%0 :128]!   \n"
+                    "vld1.f32   {%e8-%f8}, [%1 :128]!   \n"
+
+                    "vld1.f32   {%e7-%f7}, [%0 :128]!   \n"
+                    "vld1.f32   {%e9-%f9}, [%1 :128]!   \n"
+                    : "=r"(k0),     // %0
+                      "=r"(k1),     // %1
+                      "=w"(_k0),    // %2
+                      "=w"(_k0n),   // %3
+                      "=w"(_k1),    // %4
+                      "=w"(_k1n),   // %5
+                      "=w"(_k0nn),  // %6
+                      "=w"(_k0nnn), // %7
+                      "=w"(_k1nn),  // %8
+                      "=w"(_k1nnn)  // %9
+                    : "0"(k0),
+                      "1"(k1)
+                    : "cc", "memory"
+                );
+#endif // __aarch64__
+#endif // __ARM_NEON
+
+                // tile
+#if __ARM_NEON
+                int nn = tiles >> 2;
+                int remain = tiles & 3;
+#else
+                int remain = tiles;
+#endif // __ARM_NEON
+
+#if __ARM_NEON
+#if __aarch64__
+                for (; nn>0; nn--)
+                {
+                    float32x4_t _output0_tm = vld1q_f32(output0_tm);
+                    float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    float32x4_t _r0 = vld1q_f32(r0);
+                    float32x4_t _r0n = vld1q_f32(r0+4);
+                    float32x4_t _r1 = vld1q_f32(r1);
+                    float32x4_t _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+                }
+#else
+                if (nn > 0)
+                {
+#if 1
+                    asm volatile(
+                        "mov        r4, %1                  \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q8            \n"
+                        "vmla.f32   q9, q13, %q9            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "0:                                 \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm
+
+                        "vmla.f32   q10, q12, %q12          \n"
+                        "vmla.f32   q11, q13, %q13          \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "vmla.f32   q8, q14, %q10           \n"
+                        "vmla.f32   q9, q15, %q11           \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vmla.f32   q10, q14, %q14          \n"
+                        "vmla.f32   q11, q15, %q15          \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q8            \n"
+                        "vmla.f32   q9, q13, %q9            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vst1.f32   {d20-d23}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm
+
+                        "vmla.f32   q10, q12, %q12          \n"
+                        "vmla.f32   q11, q13, %q13          \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "vmla.f32   q8, q14, %q10           \n"
+                        "vmla.f32   q9, q15, %q11           \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vmla.f32   q10, q14, %q14          \n"
+                        "vmla.f32   q11, q15, %q15          \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q8            \n"
+                        "vmla.f32   q9, q13, %q9            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vst1.f32   {d20-d23}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm
+
+                        "vmla.f32   q10, q12, %q12          \n"
+                        "vmla.f32   q11, q13, %q13          \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "vmla.f32   q8, q14, %q10           \n"
+                        "vmla.f32   q9, q15, %q11           \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vmla.f32   q10, q14, %q14          \n"
+                        "vmla.f32   q11, q15, %q15          \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q8            \n"
+                        "vmla.f32   q9, q13, %q9            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vst1.f32   {d20-d23}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%1 :128]!   \n"// q10 q11 = _output0_tm
+
+                        "vmla.f32   q10, q12, %q12          \n"
+                        "vmla.f32   q11, q13, %q13          \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "vmla.f32   q8, q14, %q10           \n"
+                        "vmla.f32   q9, q15, %q11           \n"
+
+                        "pld        [%3, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%3 :128]!   \n"// q14 q15 = _r1
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "vmla.f32   q10, q14, %q14          \n"
+                        "vmla.f32   q11, q15, %q15          \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]!   \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%1 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q8            \n"
+                        "vmla.f32   q9, q13, %q9            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%2 :128]!   \n"// q12 q13 = _r0
+
+                        "subs       %0, #1                  \n"
+
+                        "vst1.f32   {d20-d23}, [r4 :128]!   \n"
+
+                        "bne        0b                      \n"
+
+                        "sub        %1, #32                 \n"
+                        "sub        %2, #64                 \n"
+                        : "=r"(nn),         // %0
+                          "=r"(output0_tm), // %1
+                          "=r"(r0),         // %2
+                          "=r"(r1)          // %3
+                        : "0"(nn),
+                          "1"(output0_tm),
+                          "2"(r0),
+                          "3"(r1),
+                          "w"(_k0),         // %8
+                          "w"(_k0n),        // %9
+                          "w"(_k1),         // %10
+                          "w"(_k1n),        // %11
+                          "w"(_k0nn),       // %12
+                          "w"(_k0nnn),      // %13
+                          "w"(_k1nn),       // %14
+                          "w"(_k1nnn)       // %15
+                        : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
+                    );
+                }
+#endif
+
+#endif // __aarch64__
+#endif // __ARM_NEON
+                for (; remain>0; remain--)
+                {
+#if __ARM_NEON
+#if __aarch64__
+                    float32x4_t _output0_tm = vld1q_f32(output0_tm);
+                    float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    float32x4_t _r0 = vld1q_f32(r0);
+                    float32x4_t _r0n = vld1q_f32(r0+4);
+                    float32x4_t _r1 = vld1q_f32(r1);
+                    float32x4_t _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1n);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+                    _r1 = vld1q_f32(r1);
+                    _r1n = vld1q_f32(r1+4);
+
+                    r0 += 8;
+                    r1 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+                    _output0_tm = vmlaq_f32(_output0_tm, _r1, _k1nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r1n, _k1nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+#else
+                    asm volatile(
+                        "mov        r4, %0                  \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0
+
+                        "pld        [%0, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%0 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q6            \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%2 :128]!   \n"// q14 q15 = _r1
+                        "vmla.f32   q9, q13, %q7            \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0
+
+                        "vmla.f32   q8, q14, %q8            \n"
+
+                        "pld        [%0, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%0 :128]    \n"// q10 q11 = _output0_tm
+                        "vmla.f32   q9, q15, %q9            \n"
+
+                        "vmla.f32   q10, q12, %q10          \n"
+                        "vmla.f32   q11, q13, %q11          \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]    \n"
+
+                        "pld        [%2, #256]              \n"
+                        "vld1.f32   {d28-d31}, [%2 :128]!   \n"// q14 q15 = _r1
+
+                        "vmla.f32   q10, q14, %q12          \n"
+                        "vmla.f32   q11, q15, %q13          \n"
+
+                        "vst1.f32   {d20-d23}, [%0 :128]!   \n"
+                        : "=r"(output0_tm), // %0
+                          "=r"(r0),         // %1
+                          "=r"(r1)          // %2
+                        : "0"(output0_tm),
+                          "1"(r0),
+                          "2"(r1),
+                          "w"(_k0),         // %6
+                          "w"(_k0n),        // %7
+                          "w"(_k1),         // %8
+                          "w"(_k1n),        // %9
+                          "w"(_k0nn),       // %10
+                          "w"(_k0nnn),      // %11
+                          "w"(_k1nn),       // %12
+                          "w"(_k1nnn)       // %13
+                        : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
+                    );
+#endif // __aarch64__
+#else
+                    for (int m=0; m<16; m++)
+                    {
+                        output0_tm[m] += r0[m] * k0[m];
+                        output0_tm[m] += r1[m] * k1[m];
+                    }
+
+                    r0 += 16;
+                    r1 += 16;
+                    output0_tm += 16;
+#endif // __ARM_NEON
+                }
+
+#if __ARM_NEON
+#if __aarch64__
+                k0 += 16;
+                k1 += 16;
+#endif // __aarch64__
+#else
+                k0 += 16;
+                k1 += 16;
+#endif // __ARM_NEON
+                }
+            }
+
+            for (; q<inch; q++)
+            {
+                const float* r0 = bottom_blob_tm.channel(q);
+
+                const float* k0 = kernel0_tm.row(q);
+
+                float* output0_tm = out0_tm;
+
+                for (int r=0; r<4; r++)
+                {
+#if __ARM_NEON
+#if __aarch64__
+                float32x4_t _k0 = vld1q_f32(k0);
+                float32x4_t _k0n = vld1q_f32(k0+4);
+                float32x4_t _k0nn = vld1q_f32(k0+8);
+                float32x4_t _k0nnn = vld1q_f32(k0+12);
+#else
+                float32x4_t _k0;
+                float32x4_t _k0n;
+                float32x4_t _k0nn;
+                float32x4_t _k0nnn;
+
+                asm volatile(
+                    "pld        [%0, #512]              \n"
+                    "vld1.f32   {%e1-%f1}, [%0 :128]!   \n"
+                    "vld1.f32   {%e2-%f2}, [%0 :128]!   \n"
+                    "vld1.f32   {%e3-%f3}, [%0 :128]!   \n"
+                    "vld1.f32   {%e4-%f4}, [%0 :128]!   \n"
+                    : "=r"(k0),     // %0
+                      "=w"(_k0),    // %1
+                      "=w"(_k0n),   // %2
+                      "=w"(_k0nn),  // %3
+                      "=w"(_k0nnn)  // %4
+                    : "0"(k0)
+                    : "cc", "memory"
+                );
+#endif // __aarch64__
+#endif // __ARM_NEON
+
+                // tile
+                for (int i=0; i<tiles; i++)
+                {
+#if __ARM_NEON
+#if __aarch64__
+                    float32x4_t _output0_tm = vld1q_f32(output0_tm);
+                    float32x4_t _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    float32x4_t _r0 = vld1q_f32(r0);
+                    float32x4_t _r0n = vld1q_f32(r0+4);
+
+                    r0 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0n);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+
+                    _output0_tm = vld1q_f32(output0_tm);
+                    _output0_tmn = vld1q_f32(output0_tm+4);
+
+                    _r0 = vld1q_f32(r0);
+                    _r0n = vld1q_f32(r0+4);
+
+                    r0 += 8;
+
+                    _output0_tm = vmlaq_f32(_output0_tm, _r0, _k0nn);
+                    _output0_tmn = vmlaq_f32(_output0_tmn, _r0n, _k0nnn);
+
+                    vst1q_f32(output0_tm, _output0_tm);
+                    vst1q_f32(output0_tm+4, _output0_tmn);
+
+                    output0_tm += 8;
+#else
+                    asm volatile(
+                        "mov        r4, %0                  \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0
+
+                        "pld        [%0, #256]              \n"
+                        "vld1.f32   {d16-d19}, [%0 :128]!   \n"// q8 q9 = _output0_tm
+
+                        "vmla.f32   q8, q12, %q4            \n"
+                        "vmla.f32   q9, q13, %q5            \n"
+
+                        "pld        [%1, #256]              \n"
+                        "vld1.f32   {d24-d27}, [%1 :128]!   \n"// q12 q13 = _r0
+
+                        "pld        [%0, #256]              \n"
+                        "vld1.f32   {d20-d23}, [%0 :128]    \n"// q10 q11 = _output0_tm
+
+                        "vmla.f32   q10, q12, %q6           \n"
+
+                        "vst1.f32   {d16-d19}, [r4 :128]    \n"
+
+                        "vmla.f32   q11, q13, %q7           \n"
+
+                        "vst1.f32   {d20-d23}, [%0 :128]!   \n"
+                        : "=r"(output0_tm), // %0
+                          "=r"(r0)          // %1
+                        : "0"(output0_tm),
+                          "1"(r0),
+                          "w"(_k0),         // %4
+                          "w"(_k0n),        // %5
+                          "w"(_k0nn),       // %6
+                          "w"(_k0nnn)       // %7
+                        : "cc", "memory", "r4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
+                    );
+#endif // __aarch64__
+#else
+                    for (int m=0; m<16; m++)
+                    {
+                        output0_tm[m] += r0[m] * k0[m];
+                    }
+
+                    r0 += 16;
+                    output0_tm += 16;
+#endif // __ARM_NEON
+                }
+
+#if __ARM_NEON
+#if __aarch64__
+                k0 += 16;
+#endif // __aarch64__
+#else
+                k0 += 16;
+#endif // __ARM_NEON
+                }
+            }
+        }
+    }
+    bottom_blob_tm = Mat();
+    // END dot
+
+    // BEGIN transform output
+    Mat top_blob_bordered;
+    top_blob_bordered.create(outw, outh, outch);
+    {
+//         const float otm[6][8] = {
+//             {1.0f,  1.0f,   1.0f,   1.0f,   1.0f,  32.0f, 32.0f, 0.0f},
+//             {0.0f,  1.0f,  -1.0f,   2.0f,  -2.0f,  16.0f,-16.0f, 0.0f},
+//             {0.0f,  1.0f,   1.0f,   4.0f,   4.0f,   8.0f,  8.0f, 0.0f},
+//             {0.0f,  1.0f,  -1.0f,   8.0f,  -8.0f,   4.0f, -4.0f, 0.0f},
+//             {0.0f,  1.0f,   1.0f,  16.0f,  16.0f,   2.0f,  2.0f, 0.0f},
+//             {0.0f,  1.0f,  -1.0f,  32.0f, -32.0f,   1.0f, -1.0f, 1.0f}
+//         };
+
+        // 0 = r0 + (r1 + r2) + (r3 + r4)     + (r5 + r6) * 32
+        // 1 =      (r1 - r2) + (r3 - r4) * 2 + (r5 - r6) * 16
+        // 2 =      (r1 + r2) + (r3 + r4) * 4 + (r5 + r6) * 8
+        // 3 =      (r1 - r2) + (r3 - r4) * 8 + (r5 - r6) * 4
+        // 4 =      (r1 + r2) + (r3 + r4) * 16+ (r5 + r6) * 2
+        // 5 = r7 + (r1 - r2) + (r3 - r4) * 32+ (r5 - r6)
+
+        int w_tm = outw / 6 * 8;
+        int h_tm = outh / 6 * 8;
+        const int tiles = w_tm/8 * h_tm/8;
+
+        #pragma omp parallel for
+        for (int p = 0; p<outch; p++)
+        {
+            const Mat out0_tm = top_blob_tm.channel(p);
+            Mat out0 = top_blob_bordered.channel(p);
+
+            const float bias0 = bias ? bias[p] : 0.f;
+
+            float tmp[6][8];
+
+            // tile
+            for (int i=0; i<outh/6; i++)
+            {
+                for (int j=0; j<outw/6; j++)
+                {
+                    const float* output0_tm01 = out0_tm.row(i * w_tm/8 + j);
+                    const float* output0_tm23 = out0_tm.row(tiles + i * w_tm/8 + j);
+                    const float* output0_tm45 = out0_tm.row(tiles * 2 + i * w_tm/8 + j);
+                    const float* output0_tm67 = out0_tm.row(tiles * 3 + i * w_tm/8 + j);
+                    float* output0 = out0.row(i * 6) + j * 6;
+
+                    const float* output0_tms[4] = { output0_tm01, output0_tm23, output0_tm45, output0_tm67 };
+
+                    for (int m=0; m<8; m++)
+                    {
+                        const float* output0_tm = output0_tms[m/2] + (m%2) * 8;
+
+                        float tmp024a = output0_tm[1] + output0_tm[2];
+                        float tmp135a = output0_tm[1] - output0_tm[2];
+
+                        float tmp024b = output0_tm[3] + output0_tm[4];
+                        float tmp135b = output0_tm[3] - output0_tm[4];
+
+                        float tmp024c = output0_tm[5] + output0_tm[6];
+                        float tmp135c = output0_tm[5] - output0_tm[6];
+
+                        tmp[0][m] = output0_tm[0] + tmp024a + tmp024b + tmp024c * 32;
+                        tmp[2][m] = tmp024a + tmp024b * 4 + tmp024c * 8;
+                        tmp[4][m] = tmp024a + tmp024b * 16 + tmp024c + tmp024c;
+
+                        tmp[1][m] = tmp135a + tmp135b + tmp135b + tmp135c * 16;
+                        tmp[3][m] = tmp135a + tmp135b * 8 + tmp135c * 4;
+                        tmp[5][m] = output0_tm[7] + tmp135a + tmp135b * 32 + tmp135c;
+                    }
+
+                    for (int m=0; m<6; m++)
+                    {
+                        const float* tmp0 = tmp[m];
+
+                        float tmp024a = tmp0[1] + tmp0[2];
+                        float tmp135a = tmp0[1] - tmp0[2];
+
+                        float tmp024b = tmp0[3] + tmp0[4];
+                        float tmp135b = tmp0[3] - tmp0[4];
+
+                        float tmp024c = tmp0[5] + tmp0[6];
+                        float tmp135c = tmp0[5] - tmp0[6];
+
+                        output0[0] = bias0 + tmp0[0] + tmp024a + tmp024b + tmp024c * 32;
+                        output0[2] = bias0 + tmp024a + tmp024b * 4 + tmp024c * 8;
+                        output0[4] = bias0 + tmp024a + tmp024b * 16 + tmp024c + tmp024c;
+
+                        output0[1] = bias0 + tmp135a + tmp135b + tmp135b + tmp135c * 16;
+                        output0[3] = bias0 + tmp135a + tmp135b * 8 + tmp135c * 4;
+                        output0[5] = bias0 + tmp0[7] + tmp135a + tmp135b * 32 + tmp135c;
+
+                        output0 += outw;
+                    }
+                }
+            }
+        }
+    }
+    // END transform output
+
+    // cut result pad
+    copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w);
+}
+
 static void conv3x3s2_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& _kernel, const Mat& _bias)
 {
     int w = bottom_blob.w;
diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp
index 6595fcd53..1a1b5f1dc 100644
--- a/src/layer/arm/convolution_arm.cpp
+++ b/src/layer/arm/convolution_arm.cpp
@@ -14,6 +14,8 @@
 
 #include "convolution_arm.h"
 
+#include "cpu.h"
+
 namespace ncnn {
 
 #include "convolution_1x1.h"
@@ -178,7 +180,25 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob) const
         return -100;
 
     if (use_winograd3x3)
-        conv3x3s1_winograd64_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data);
+    {
+        if (w <= 50 && h <= 50)
+        {
+            // another path for small image
+            conv3x3s1_winograd64_neon2(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data);
+        }
+        else
+        {
+            int num_threads = get_omp_num_threads();
+            if (num_threads == 1 || (w <= 80 && h <= 80))
+            {
+                conv3x3s1_winograd64_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd64_data, bias_data);
+            }
+            else
+            {
+                conv(bottom_blob_bordered, top_blob, weight_data, bias_data);
+            }
+        }
+    }
     else
         conv(bottom_blob_bordered, top_blob, weight_data, bias_data);