From 1ea8bfbd2e7b43c14a8ccacd58a3f172cc218306 Mon Sep 17 00:00:00 2001 From: nihui Date: Sun, 28 Mar 2021 11:30:26 +0800 Subject: [PATCH] x86 avx2 conv3x3s1 pack8 direct optimization, fix #2789 --- src/layer/x86/convolution_3x3_pack8.h | 705 ++++++++++++++++++++++++++ src/layer/x86/convolution_x86.cpp | 12 +- tests/test_convolution.cpp | 1 + 3 files changed, 715 insertions(+), 3 deletions(-) diff --git a/src/layer/x86/convolution_3x3_pack8.h b/src/layer/x86/convolution_3x3_pack8.h index 913c50707..6859de76f 100644 --- a/src/layer/x86/convolution_3x3_pack8.h +++ b/src/layer/x86/convolution_3x3_pack8.h @@ -12,6 +12,711 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. +static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + const float* bias = _bias; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + Mat out = top_blob.channel(p); + + __m256 _bias0 = bias ? _mm256_loadu_ps(bias + p * 8) : _mm256_set1_ps(0.f); + out.fill(_bias0); + + for (int q = 0; q < inch; q++) + { + float* outptr = out; + + const Mat img0 = bottom_blob.channel(q); + + const float* r0 = img0.row(0); + const float* r1 = img0.row(1); + const float* r2 = img0.row(2); + + const float* kptr = kernel.channel(p).row(q); + + int i = 0; + for (; i < outh; i++) + { + int j = 0; + for (; j + 1 < outw; j += 2) + { + __m256 _sum00 = _mm256_loadu_ps(outptr); + __m256 _sum01 = _mm256_setzero_ps(); + __m256 _sum10 = _mm256_loadu_ps(outptr + 8); + __m256 _sum11 = _mm256_setzero_ps(); + + __m256 _r000 = _mm256_broadcast_ss(r0 + 0); + __m256 _r001 = _mm256_broadcast_ss(r0 + 1); + __m256 _r002 = _mm256_broadcast_ss(r0 + 2); + __m256 _r003 = _mm256_broadcast_ss(r0 + 3); + __m256 _r004 = _mm256_broadcast_ss(r0 + 4); + __m256 _r005 = _mm256_broadcast_ss(r0 + 5); + __m256 _r006 = _mm256_broadcast_ss(r0 + 6); + __m256 _r007 = _mm256_broadcast_ss(r0 + 7); + + __m256 _k00 = _mm256_loadu_ps(kptr); + __m256 _k01 = _mm256_loadu_ps(kptr + 8); + __m256 _k02 = _mm256_loadu_ps(kptr + 16); + __m256 _k03 = _mm256_loadu_ps(kptr + 24); + __m256 _k04 = _mm256_loadu_ps(kptr + 32); + __m256 _k05 = _mm256_loadu_ps(kptr + 40); + __m256 _k06 = _mm256_loadu_ps(kptr + 48); + __m256 _k07 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r000, _k00, _sum00); + _sum01 = _mm256_fmadd_ps(_r001, _k01, _sum01); + _sum00 = _mm256_fmadd_ps(_r002, _k02, _sum00); + _sum01 = _mm256_fmadd_ps(_r003, _k03, _sum01); + _sum00 = _mm256_fmadd_ps(_r004, _k04, _sum00); + _sum01 = _mm256_fmadd_ps(_r005, _k05, _sum01); + _sum00 = _mm256_fmadd_ps(_r006, _k06, _sum00); + _sum01 = _mm256_fmadd_ps(_r007, _k07, _sum01); + + __m256 _r010 = _mm256_broadcast_ss(r0 + 8); + __m256 _r011 = _mm256_broadcast_ss(r0 + 9); + __m256 _r012 = _mm256_broadcast_ss(r0 + 10); + __m256 _r013 = _mm256_broadcast_ss(r0 + 11); + __m256 _r014 = _mm256_broadcast_ss(r0 + 12); + __m256 _r015 = _mm256_broadcast_ss(r0 + 13); + __m256 _r016 = _mm256_broadcast_ss(r0 + 14); + __m256 _r017 = _mm256_broadcast_ss(r0 + 15); + + _sum10 = _mm256_fmadd_ps(_r010, _k00, _sum10); + _sum11 = _mm256_fmadd_ps(_r011, _k01, _sum11); + _sum10 = _mm256_fmadd_ps(_r012, _k02, _sum10); + _sum11 = _mm256_fmadd_ps(_r013, _k03, _sum11); + _sum10 = _mm256_fmadd_ps(_r014, _k04, _sum10); + _sum11 = _mm256_fmadd_ps(_r015, _k05, _sum11); + _sum10 = _mm256_fmadd_ps(_r016, _k06, _sum10); + _sum11 = _mm256_fmadd_ps(_r017, _k07, _sum11); + + __m256 _k10 = _mm256_loadu_ps(kptr); + __m256 _k11 = _mm256_loadu_ps(kptr + 8); + __m256 _k12 = _mm256_loadu_ps(kptr + 16); + __m256 _k13 = _mm256_loadu_ps(kptr + 24); + __m256 _k14 = _mm256_loadu_ps(kptr + 32); + __m256 _k15 = _mm256_loadu_ps(kptr + 40); + __m256 _k16 = _mm256_loadu_ps(kptr + 48); + __m256 _k17 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r010, _k10, _sum00); + _sum01 = _mm256_fmadd_ps(_r011, _k11, _sum01); + _sum00 = _mm256_fmadd_ps(_r012, _k12, _sum00); + _sum01 = _mm256_fmadd_ps(_r013, _k13, _sum01); + _sum00 = _mm256_fmadd_ps(_r014, _k14, _sum00); + _sum01 = _mm256_fmadd_ps(_r015, _k15, _sum01); + _sum00 = _mm256_fmadd_ps(_r016, _k16, _sum00); + _sum01 = _mm256_fmadd_ps(_r017, _k17, _sum01); + + __m256 _r020 = _mm256_broadcast_ss(r0 + 16); + __m256 _r021 = _mm256_broadcast_ss(r0 + 17); + __m256 _r022 = _mm256_broadcast_ss(r0 + 18); + __m256 _r023 = _mm256_broadcast_ss(r0 + 19); + __m256 _r024 = _mm256_broadcast_ss(r0 + 20); + __m256 _r025 = _mm256_broadcast_ss(r0 + 21); + __m256 _r026 = _mm256_broadcast_ss(r0 + 22); + __m256 _r027 = _mm256_broadcast_ss(r0 + 23); + + _sum10 = _mm256_fmadd_ps(_r020, _k10, _sum10); + _sum11 = _mm256_fmadd_ps(_r021, _k11, _sum11); + _sum10 = _mm256_fmadd_ps(_r022, _k12, _sum10); + _sum11 = _mm256_fmadd_ps(_r023, _k13, _sum11); + _sum10 = _mm256_fmadd_ps(_r024, _k14, _sum10); + _sum11 = _mm256_fmadd_ps(_r025, _k15, _sum11); + _sum10 = _mm256_fmadd_ps(_r026, _k16, _sum10); + _sum11 = _mm256_fmadd_ps(_r027, _k17, _sum11); + + __m256 _k20 = _mm256_loadu_ps(kptr); + __m256 _k21 = _mm256_loadu_ps(kptr + 8); + __m256 _k22 = _mm256_loadu_ps(kptr + 16); + __m256 _k23 = _mm256_loadu_ps(kptr + 24); + __m256 _k24 = _mm256_loadu_ps(kptr + 32); + __m256 _k25 = _mm256_loadu_ps(kptr + 40); + __m256 _k26 = _mm256_loadu_ps(kptr + 48); + __m256 _k27 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r020, _k20, _sum00); + _sum01 = _mm256_fmadd_ps(_r021, _k21, _sum01); + _sum00 = _mm256_fmadd_ps(_r022, _k22, _sum00); + _sum01 = _mm256_fmadd_ps(_r023, _k23, _sum01); + _sum00 = _mm256_fmadd_ps(_r024, _k24, _sum00); + _sum01 = _mm256_fmadd_ps(_r025, _k25, _sum01); + _sum00 = _mm256_fmadd_ps(_r026, _k26, _sum00); + _sum01 = _mm256_fmadd_ps(_r027, _k27, _sum01); + + __m256 _r030 = _mm256_broadcast_ss(r0 + 24); + __m256 _r031 = _mm256_broadcast_ss(r0 + 25); + __m256 _r032 = _mm256_broadcast_ss(r0 + 26); + __m256 _r033 = _mm256_broadcast_ss(r0 + 27); + __m256 _r034 = _mm256_broadcast_ss(r0 + 28); + __m256 _r035 = _mm256_broadcast_ss(r0 + 29); + __m256 _r036 = _mm256_broadcast_ss(r0 + 30); + __m256 _r037 = _mm256_broadcast_ss(r0 + 31); + + _sum10 = _mm256_fmadd_ps(_r030, _k20, _sum10); + _sum11 = _mm256_fmadd_ps(_r031, _k21, _sum11); + _sum10 = _mm256_fmadd_ps(_r032, _k22, _sum10); + _sum11 = _mm256_fmadd_ps(_r033, _k23, _sum11); + _sum10 = _mm256_fmadd_ps(_r034, _k24, _sum10); + _sum11 = _mm256_fmadd_ps(_r035, _k25, _sum11); + _sum10 = _mm256_fmadd_ps(_r036, _k26, _sum10); + _sum11 = _mm256_fmadd_ps(_r037, _k27, _sum11); + + __m256 _r100 = _mm256_broadcast_ss(r1 + 0); + __m256 _r101 = _mm256_broadcast_ss(r1 + 1); + __m256 _r102 = _mm256_broadcast_ss(r1 + 2); + __m256 _r103 = _mm256_broadcast_ss(r1 + 3); + __m256 _r104 = _mm256_broadcast_ss(r1 + 4); + __m256 _r105 = _mm256_broadcast_ss(r1 + 5); + __m256 _r106 = _mm256_broadcast_ss(r1 + 6); + __m256 _r107 = _mm256_broadcast_ss(r1 + 7); + + __m256 _k30 = _mm256_loadu_ps(kptr); + __m256 _k31 = _mm256_loadu_ps(kptr + 8); + __m256 _k32 = _mm256_loadu_ps(kptr + 16); + __m256 _k33 = _mm256_loadu_ps(kptr + 24); + __m256 _k34 = _mm256_loadu_ps(kptr + 32); + __m256 _k35 = _mm256_loadu_ps(kptr + 40); + __m256 _k36 = _mm256_loadu_ps(kptr + 48); + __m256 _k37 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r100, _k30, _sum00); + _sum01 = _mm256_fmadd_ps(_r101, _k31, _sum01); + _sum00 = _mm256_fmadd_ps(_r102, _k32, _sum00); + _sum01 = _mm256_fmadd_ps(_r103, _k33, _sum01); + _sum00 = _mm256_fmadd_ps(_r104, _k34, _sum00); + _sum01 = _mm256_fmadd_ps(_r105, _k35, _sum01); + _sum00 = _mm256_fmadd_ps(_r106, _k36, _sum00); + _sum01 = _mm256_fmadd_ps(_r107, _k37, _sum01); + + __m256 _r110 = _mm256_broadcast_ss(r1 + 8); + __m256 _r111 = _mm256_broadcast_ss(r1 + 9); + __m256 _r112 = _mm256_broadcast_ss(r1 + 10); + __m256 _r113 = _mm256_broadcast_ss(r1 + 11); + __m256 _r114 = _mm256_broadcast_ss(r1 + 12); + __m256 _r115 = _mm256_broadcast_ss(r1 + 13); + __m256 _r116 = _mm256_broadcast_ss(r1 + 14); + __m256 _r117 = _mm256_broadcast_ss(r1 + 15); + + _sum10 = _mm256_fmadd_ps(_r110, _k30, _sum10); + _sum11 = _mm256_fmadd_ps(_r111, _k31, _sum11); + _sum10 = _mm256_fmadd_ps(_r112, _k32, _sum10); + _sum11 = _mm256_fmadd_ps(_r113, _k33, _sum11); + _sum10 = _mm256_fmadd_ps(_r114, _k34, _sum10); + _sum11 = _mm256_fmadd_ps(_r115, _k35, _sum11); + _sum10 = _mm256_fmadd_ps(_r116, _k36, _sum10); + _sum11 = _mm256_fmadd_ps(_r117, _k37, _sum11); + + __m256 _k40 = _mm256_loadu_ps(kptr); + __m256 _k41 = _mm256_loadu_ps(kptr + 8); + __m256 _k42 = _mm256_loadu_ps(kptr + 16); + __m256 _k43 = _mm256_loadu_ps(kptr + 24); + __m256 _k44 = _mm256_loadu_ps(kptr + 32); + __m256 _k45 = _mm256_loadu_ps(kptr + 40); + __m256 _k46 = _mm256_loadu_ps(kptr + 48); + __m256 _k47 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r110, _k40, _sum00); + _sum01 = _mm256_fmadd_ps(_r111, _k41, _sum01); + _sum00 = _mm256_fmadd_ps(_r112, _k42, _sum00); + _sum01 = _mm256_fmadd_ps(_r113, _k43, _sum01); + _sum00 = _mm256_fmadd_ps(_r114, _k44, _sum00); + _sum01 = _mm256_fmadd_ps(_r115, _k45, _sum01); + _sum00 = _mm256_fmadd_ps(_r116, _k46, _sum00); + _sum01 = _mm256_fmadd_ps(_r117, _k47, _sum01); + + __m256 _r120 = _mm256_broadcast_ss(r1 + 16); + __m256 _r121 = _mm256_broadcast_ss(r1 + 17); + __m256 _r122 = _mm256_broadcast_ss(r1 + 18); + __m256 _r123 = _mm256_broadcast_ss(r1 + 19); + __m256 _r124 = _mm256_broadcast_ss(r1 + 20); + __m256 _r125 = _mm256_broadcast_ss(r1 + 21); + __m256 _r126 = _mm256_broadcast_ss(r1 + 22); + __m256 _r127 = _mm256_broadcast_ss(r1 + 23); + + _sum10 = _mm256_fmadd_ps(_r120, _k40, _sum10); + _sum11 = _mm256_fmadd_ps(_r121, _k41, _sum11); + _sum10 = _mm256_fmadd_ps(_r122, _k42, _sum10); + _sum11 = _mm256_fmadd_ps(_r123, _k43, _sum11); + _sum10 = _mm256_fmadd_ps(_r124, _k44, _sum10); + _sum11 = _mm256_fmadd_ps(_r125, _k45, _sum11); + _sum10 = _mm256_fmadd_ps(_r126, _k46, _sum10); + _sum11 = _mm256_fmadd_ps(_r127, _k47, _sum11); + + __m256 _k50 = _mm256_loadu_ps(kptr); + __m256 _k51 = _mm256_loadu_ps(kptr + 8); + __m256 _k52 = _mm256_loadu_ps(kptr + 16); + __m256 _k53 = _mm256_loadu_ps(kptr + 24); + __m256 _k54 = _mm256_loadu_ps(kptr + 32); + __m256 _k55 = _mm256_loadu_ps(kptr + 40); + __m256 _k56 = _mm256_loadu_ps(kptr + 48); + __m256 _k57 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r120, _k50, _sum00); + _sum01 = _mm256_fmadd_ps(_r121, _k51, _sum01); + _sum00 = _mm256_fmadd_ps(_r122, _k52, _sum00); + _sum01 = _mm256_fmadd_ps(_r123, _k53, _sum01); + _sum00 = _mm256_fmadd_ps(_r124, _k54, _sum00); + _sum01 = _mm256_fmadd_ps(_r125, _k55, _sum01); + _sum00 = _mm256_fmadd_ps(_r126, _k56, _sum00); + _sum01 = _mm256_fmadd_ps(_r127, _k57, _sum01); + + __m256 _r130 = _mm256_broadcast_ss(r1 + 24); + __m256 _r131 = _mm256_broadcast_ss(r1 + 25); + __m256 _r132 = _mm256_broadcast_ss(r1 + 26); + __m256 _r133 = _mm256_broadcast_ss(r1 + 27); + __m256 _r134 = _mm256_broadcast_ss(r1 + 28); + __m256 _r135 = _mm256_broadcast_ss(r1 + 29); + __m256 _r136 = _mm256_broadcast_ss(r1 + 30); + __m256 _r137 = _mm256_broadcast_ss(r1 + 31); + + _sum10 = _mm256_fmadd_ps(_r130, _k50, _sum10); + _sum11 = _mm256_fmadd_ps(_r131, _k51, _sum11); + _sum10 = _mm256_fmadd_ps(_r132, _k52, _sum10); + _sum11 = _mm256_fmadd_ps(_r133, _k53, _sum11); + _sum10 = _mm256_fmadd_ps(_r134, _k54, _sum10); + _sum11 = _mm256_fmadd_ps(_r135, _k55, _sum11); + _sum10 = _mm256_fmadd_ps(_r136, _k56, _sum10); + _sum11 = _mm256_fmadd_ps(_r137, _k57, _sum11); + + __m256 _r200 = _mm256_broadcast_ss(r2 + 0); + __m256 _r201 = _mm256_broadcast_ss(r2 + 1); + __m256 _r202 = _mm256_broadcast_ss(r2 + 2); + __m256 _r203 = _mm256_broadcast_ss(r2 + 3); + __m256 _r204 = _mm256_broadcast_ss(r2 + 4); + __m256 _r205 = _mm256_broadcast_ss(r2 + 5); + __m256 _r206 = _mm256_broadcast_ss(r2 + 6); + __m256 _r207 = _mm256_broadcast_ss(r2 + 7); + + __m256 _k60 = _mm256_loadu_ps(kptr); + __m256 _k61 = _mm256_loadu_ps(kptr + 8); + __m256 _k62 = _mm256_loadu_ps(kptr + 16); + __m256 _k63 = _mm256_loadu_ps(kptr + 24); + __m256 _k64 = _mm256_loadu_ps(kptr + 32); + __m256 _k65 = _mm256_loadu_ps(kptr + 40); + __m256 _k66 = _mm256_loadu_ps(kptr + 48); + __m256 _k67 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r200, _k60, _sum00); + _sum01 = _mm256_fmadd_ps(_r201, _k61, _sum01); + _sum00 = _mm256_fmadd_ps(_r202, _k62, _sum00); + _sum01 = _mm256_fmadd_ps(_r203, _k63, _sum01); + _sum00 = _mm256_fmadd_ps(_r204, _k64, _sum00); + _sum01 = _mm256_fmadd_ps(_r205, _k65, _sum01); + _sum00 = _mm256_fmadd_ps(_r206, _k66, _sum00); + _sum01 = _mm256_fmadd_ps(_r207, _k67, _sum01); + + __m256 _r210 = _mm256_broadcast_ss(r2 + 8); + __m256 _r211 = _mm256_broadcast_ss(r2 + 9); + __m256 _r212 = _mm256_broadcast_ss(r2 + 10); + __m256 _r213 = _mm256_broadcast_ss(r2 + 11); + __m256 _r214 = _mm256_broadcast_ss(r2 + 12); + __m256 _r215 = _mm256_broadcast_ss(r2 + 13); + __m256 _r216 = _mm256_broadcast_ss(r2 + 14); + __m256 _r217 = _mm256_broadcast_ss(r2 + 15); + + _sum10 = _mm256_fmadd_ps(_r210, _k60, _sum10); + _sum11 = _mm256_fmadd_ps(_r211, _k61, _sum11); + _sum10 = _mm256_fmadd_ps(_r212, _k62, _sum10); + _sum11 = _mm256_fmadd_ps(_r213, _k63, _sum11); + _sum10 = _mm256_fmadd_ps(_r214, _k64, _sum10); + _sum11 = _mm256_fmadd_ps(_r215, _k65, _sum11); + _sum10 = _mm256_fmadd_ps(_r216, _k66, _sum10); + _sum11 = _mm256_fmadd_ps(_r217, _k67, _sum11); + + __m256 _k70 = _mm256_loadu_ps(kptr); + __m256 _k71 = _mm256_loadu_ps(kptr + 8); + __m256 _k72 = _mm256_loadu_ps(kptr + 16); + __m256 _k73 = _mm256_loadu_ps(kptr + 24); + __m256 _k74 = _mm256_loadu_ps(kptr + 32); + __m256 _k75 = _mm256_loadu_ps(kptr + 40); + __m256 _k76 = _mm256_loadu_ps(kptr + 48); + __m256 _k77 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum00 = _mm256_fmadd_ps(_r210, _k70, _sum00); + _sum01 = _mm256_fmadd_ps(_r211, _k71, _sum01); + _sum00 = _mm256_fmadd_ps(_r212, _k72, _sum00); + _sum01 = _mm256_fmadd_ps(_r213, _k73, _sum01); + _sum00 = _mm256_fmadd_ps(_r214, _k74, _sum00); + _sum01 = _mm256_fmadd_ps(_r215, _k75, _sum01); + _sum00 = _mm256_fmadd_ps(_r216, _k76, _sum00); + _sum01 = _mm256_fmadd_ps(_r217, _k77, _sum01); + + __m256 _r220 = _mm256_broadcast_ss(r2 + 16); + __m256 _r221 = _mm256_broadcast_ss(r2 + 17); + __m256 _r222 = _mm256_broadcast_ss(r2 + 18); + __m256 _r223 = _mm256_broadcast_ss(r2 + 19); + __m256 _r224 = _mm256_broadcast_ss(r2 + 20); + __m256 _r225 = _mm256_broadcast_ss(r2 + 21); + __m256 _r226 = _mm256_broadcast_ss(r2 + 22); + __m256 _r227 = _mm256_broadcast_ss(r2 + 23); + + _sum10 = _mm256_fmadd_ps(_r220, _k70, _sum10); + _sum11 = _mm256_fmadd_ps(_r221, _k71, _sum11); + _sum10 = _mm256_fmadd_ps(_r222, _k72, _sum10); + _sum11 = _mm256_fmadd_ps(_r223, _k73, _sum11); + _sum10 = _mm256_fmadd_ps(_r224, _k74, _sum10); + _sum11 = _mm256_fmadd_ps(_r225, _k75, _sum11); + _sum10 = _mm256_fmadd_ps(_r226, _k76, _sum10); + _sum11 = _mm256_fmadd_ps(_r227, _k77, _sum11); + + __m256 _k80 = _mm256_loadu_ps(kptr); + __m256 _k81 = _mm256_loadu_ps(kptr + 8); + __m256 _k82 = _mm256_loadu_ps(kptr + 16); + __m256 _k83 = _mm256_loadu_ps(kptr + 24); + __m256 _k84 = _mm256_loadu_ps(kptr + 32); + __m256 _k85 = _mm256_loadu_ps(kptr + 40); + __m256 _k86 = _mm256_loadu_ps(kptr + 48); + __m256 _k87 = _mm256_loadu_ps(kptr + 56); + + _sum00 = _mm256_fmadd_ps(_r220, _k80, _sum00); + _sum01 = _mm256_fmadd_ps(_r221, _k81, _sum01); + _sum00 = _mm256_fmadd_ps(_r222, _k82, _sum00); + _sum01 = _mm256_fmadd_ps(_r223, _k83, _sum01); + _sum00 = _mm256_fmadd_ps(_r224, _k84, _sum00); + _sum01 = _mm256_fmadd_ps(_r225, _k85, _sum01); + _sum00 = _mm256_fmadd_ps(_r226, _k86, _sum00); + _sum01 = _mm256_fmadd_ps(_r227, _k87, _sum01); + + __m256 _r230 = _mm256_broadcast_ss(r2 + 24); + __m256 _r231 = _mm256_broadcast_ss(r2 + 25); + __m256 _r232 = _mm256_broadcast_ss(r2 + 26); + __m256 _r233 = _mm256_broadcast_ss(r2 + 27); + __m256 _r234 = _mm256_broadcast_ss(r2 + 28); + __m256 _r235 = _mm256_broadcast_ss(r2 + 29); + __m256 _r236 = _mm256_broadcast_ss(r2 + 30); + __m256 _r237 = _mm256_broadcast_ss(r2 + 31); + + _sum10 = _mm256_fmadd_ps(_r230, _k80, _sum10); + _sum11 = _mm256_fmadd_ps(_r231, _k81, _sum11); + _sum10 = _mm256_fmadd_ps(_r232, _k82, _sum10); + _sum11 = _mm256_fmadd_ps(_r233, _k83, _sum11); + _sum10 = _mm256_fmadd_ps(_r234, _k84, _sum10); + _sum11 = _mm256_fmadd_ps(_r235, _k85, _sum11); + _sum10 = _mm256_fmadd_ps(_r236, _k86, _sum10); + _sum11 = _mm256_fmadd_ps(_r237, _k87, _sum11); + + kptr -= 64 * 8; + + _sum00 = _mm256_add_ps(_sum00, _sum01); + _sum10 = _mm256_add_ps(_sum10, _sum11); + + _mm256_storeu_ps(outptr, _sum00); + _mm256_storeu_ps(outptr + 8, _sum10); + + r0 += 16; + r1 += 16; + r2 += 16; + outptr += 16; + } + for (; j < outw; j++) + { + __m256 _sum0 = _mm256_loadu_ps(outptr); + __m256 _sum1 = _mm256_setzero_ps(); + + __m256 _r000 = _mm256_broadcast_ss(r0 + 0); + __m256 _r001 = _mm256_broadcast_ss(r0 + 1); + __m256 _r002 = _mm256_broadcast_ss(r0 + 2); + __m256 _r003 = _mm256_broadcast_ss(r0 + 3); + __m256 _r004 = _mm256_broadcast_ss(r0 + 4); + __m256 _r005 = _mm256_broadcast_ss(r0 + 5); + __m256 _r006 = _mm256_broadcast_ss(r0 + 6); + __m256 _r007 = _mm256_broadcast_ss(r0 + 7); + + __m256 _k00 = _mm256_loadu_ps(kptr); + __m256 _k01 = _mm256_loadu_ps(kptr + 8); + __m256 _k02 = _mm256_loadu_ps(kptr + 16); + __m256 _k03 = _mm256_loadu_ps(kptr + 24); + __m256 _k04 = _mm256_loadu_ps(kptr + 32); + __m256 _k05 = _mm256_loadu_ps(kptr + 40); + __m256 _k06 = _mm256_loadu_ps(kptr + 48); + __m256 _k07 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r000, _k00, _sum0); + _sum1 = _mm256_fmadd_ps(_r001, _k01, _sum1); + _sum0 = _mm256_fmadd_ps(_r002, _k02, _sum0); + _sum1 = _mm256_fmadd_ps(_r003, _k03, _sum1); + _sum0 = _mm256_fmadd_ps(_r004, _k04, _sum0); + _sum1 = _mm256_fmadd_ps(_r005, _k05, _sum1); + _sum0 = _mm256_fmadd_ps(_r006, _k06, _sum0); + _sum1 = _mm256_fmadd_ps(_r007, _k07, _sum1); + + __m256 _r010 = _mm256_broadcast_ss(r0 + 8); + __m256 _r011 = _mm256_broadcast_ss(r0 + 9); + __m256 _r012 = _mm256_broadcast_ss(r0 + 10); + __m256 _r013 = _mm256_broadcast_ss(r0 + 11); + __m256 _r014 = _mm256_broadcast_ss(r0 + 12); + __m256 _r015 = _mm256_broadcast_ss(r0 + 13); + __m256 _r016 = _mm256_broadcast_ss(r0 + 14); + __m256 _r017 = _mm256_broadcast_ss(r0 + 15); + + __m256 _k10 = _mm256_loadu_ps(kptr); + __m256 _k11 = _mm256_loadu_ps(kptr + 8); + __m256 _k12 = _mm256_loadu_ps(kptr + 16); + __m256 _k13 = _mm256_loadu_ps(kptr + 24); + __m256 _k14 = _mm256_loadu_ps(kptr + 32); + __m256 _k15 = _mm256_loadu_ps(kptr + 40); + __m256 _k16 = _mm256_loadu_ps(kptr + 48); + __m256 _k17 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r010, _k10, _sum0); + _sum1 = _mm256_fmadd_ps(_r011, _k11, _sum1); + _sum0 = _mm256_fmadd_ps(_r012, _k12, _sum0); + _sum1 = _mm256_fmadd_ps(_r013, _k13, _sum1); + _sum0 = _mm256_fmadd_ps(_r014, _k14, _sum0); + _sum1 = _mm256_fmadd_ps(_r015, _k15, _sum1); + _sum0 = _mm256_fmadd_ps(_r016, _k16, _sum0); + _sum1 = _mm256_fmadd_ps(_r017, _k17, _sum1); + + __m256 _r020 = _mm256_broadcast_ss(r0 + 16); + __m256 _r021 = _mm256_broadcast_ss(r0 + 17); + __m256 _r022 = _mm256_broadcast_ss(r0 + 18); + __m256 _r023 = _mm256_broadcast_ss(r0 + 19); + __m256 _r024 = _mm256_broadcast_ss(r0 + 20); + __m256 _r025 = _mm256_broadcast_ss(r0 + 21); + __m256 _r026 = _mm256_broadcast_ss(r0 + 22); + __m256 _r027 = _mm256_broadcast_ss(r0 + 23); + + __m256 _k20 = _mm256_loadu_ps(kptr); + __m256 _k21 = _mm256_loadu_ps(kptr + 8); + __m256 _k22 = _mm256_loadu_ps(kptr + 16); + __m256 _k23 = _mm256_loadu_ps(kptr + 24); + __m256 _k24 = _mm256_loadu_ps(kptr + 32); + __m256 _k25 = _mm256_loadu_ps(kptr + 40); + __m256 _k26 = _mm256_loadu_ps(kptr + 48); + __m256 _k27 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r020, _k20, _sum0); + _sum1 = _mm256_fmadd_ps(_r021, _k21, _sum1); + _sum0 = _mm256_fmadd_ps(_r022, _k22, _sum0); + _sum1 = _mm256_fmadd_ps(_r023, _k23, _sum1); + _sum0 = _mm256_fmadd_ps(_r024, _k24, _sum0); + _sum1 = _mm256_fmadd_ps(_r025, _k25, _sum1); + _sum0 = _mm256_fmadd_ps(_r026, _k26, _sum0); + _sum1 = _mm256_fmadd_ps(_r027, _k27, _sum1); + + __m256 _r100 = _mm256_broadcast_ss(r1 + 0); + __m256 _r101 = _mm256_broadcast_ss(r1 + 1); + __m256 _r102 = _mm256_broadcast_ss(r1 + 2); + __m256 _r103 = _mm256_broadcast_ss(r1 + 3); + __m256 _r104 = _mm256_broadcast_ss(r1 + 4); + __m256 _r105 = _mm256_broadcast_ss(r1 + 5); + __m256 _r106 = _mm256_broadcast_ss(r1 + 6); + __m256 _r107 = _mm256_broadcast_ss(r1 + 7); + + __m256 _k30 = _mm256_loadu_ps(kptr); + __m256 _k31 = _mm256_loadu_ps(kptr + 8); + __m256 _k32 = _mm256_loadu_ps(kptr + 16); + __m256 _k33 = _mm256_loadu_ps(kptr + 24); + __m256 _k34 = _mm256_loadu_ps(kptr + 32); + __m256 _k35 = _mm256_loadu_ps(kptr + 40); + __m256 _k36 = _mm256_loadu_ps(kptr + 48); + __m256 _k37 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r100, _k30, _sum0); + _sum1 = _mm256_fmadd_ps(_r101, _k31, _sum1); + _sum0 = _mm256_fmadd_ps(_r102, _k32, _sum0); + _sum1 = _mm256_fmadd_ps(_r103, _k33, _sum1); + _sum0 = _mm256_fmadd_ps(_r104, _k34, _sum0); + _sum1 = _mm256_fmadd_ps(_r105, _k35, _sum1); + _sum0 = _mm256_fmadd_ps(_r106, _k36, _sum0); + _sum1 = _mm256_fmadd_ps(_r107, _k37, _sum1); + + __m256 _r110 = _mm256_broadcast_ss(r1 + 8); + __m256 _r111 = _mm256_broadcast_ss(r1 + 9); + __m256 _r112 = _mm256_broadcast_ss(r1 + 10); + __m256 _r113 = _mm256_broadcast_ss(r1 + 11); + __m256 _r114 = _mm256_broadcast_ss(r1 + 12); + __m256 _r115 = _mm256_broadcast_ss(r1 + 13); + __m256 _r116 = _mm256_broadcast_ss(r1 + 14); + __m256 _r117 = _mm256_broadcast_ss(r1 + 15); + + __m256 _k40 = _mm256_loadu_ps(kptr); + __m256 _k41 = _mm256_loadu_ps(kptr + 8); + __m256 _k42 = _mm256_loadu_ps(kptr + 16); + __m256 _k43 = _mm256_loadu_ps(kptr + 24); + __m256 _k44 = _mm256_loadu_ps(kptr + 32); + __m256 _k45 = _mm256_loadu_ps(kptr + 40); + __m256 _k46 = _mm256_loadu_ps(kptr + 48); + __m256 _k47 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r110, _k40, _sum0); + _sum1 = _mm256_fmadd_ps(_r111, _k41, _sum1); + _sum0 = _mm256_fmadd_ps(_r112, _k42, _sum0); + _sum1 = _mm256_fmadd_ps(_r113, _k43, _sum1); + _sum0 = _mm256_fmadd_ps(_r114, _k44, _sum0); + _sum1 = _mm256_fmadd_ps(_r115, _k45, _sum1); + _sum0 = _mm256_fmadd_ps(_r116, _k46, _sum0); + _sum1 = _mm256_fmadd_ps(_r117, _k47, _sum1); + + __m256 _r120 = _mm256_broadcast_ss(r1 + 16); + __m256 _r121 = _mm256_broadcast_ss(r1 + 17); + __m256 _r122 = _mm256_broadcast_ss(r1 + 18); + __m256 _r123 = _mm256_broadcast_ss(r1 + 19); + __m256 _r124 = _mm256_broadcast_ss(r1 + 20); + __m256 _r125 = _mm256_broadcast_ss(r1 + 21); + __m256 _r126 = _mm256_broadcast_ss(r1 + 22); + __m256 _r127 = _mm256_broadcast_ss(r1 + 23); + + __m256 _k50 = _mm256_loadu_ps(kptr); + __m256 _k51 = _mm256_loadu_ps(kptr + 8); + __m256 _k52 = _mm256_loadu_ps(kptr + 16); + __m256 _k53 = _mm256_loadu_ps(kptr + 24); + __m256 _k54 = _mm256_loadu_ps(kptr + 32); + __m256 _k55 = _mm256_loadu_ps(kptr + 40); + __m256 _k56 = _mm256_loadu_ps(kptr + 48); + __m256 _k57 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r120, _k50, _sum0); + _sum1 = _mm256_fmadd_ps(_r121, _k51, _sum1); + _sum0 = _mm256_fmadd_ps(_r122, _k52, _sum0); + _sum1 = _mm256_fmadd_ps(_r123, _k53, _sum1); + _sum0 = _mm256_fmadd_ps(_r124, _k54, _sum0); + _sum1 = _mm256_fmadd_ps(_r125, _k55, _sum1); + _sum0 = _mm256_fmadd_ps(_r126, _k56, _sum0); + _sum1 = _mm256_fmadd_ps(_r127, _k57, _sum1); + + __m256 _r200 = _mm256_broadcast_ss(r2 + 0); + __m256 _r201 = _mm256_broadcast_ss(r2 + 1); + __m256 _r202 = _mm256_broadcast_ss(r2 + 2); + __m256 _r203 = _mm256_broadcast_ss(r2 + 3); + __m256 _r204 = _mm256_broadcast_ss(r2 + 4); + __m256 _r205 = _mm256_broadcast_ss(r2 + 5); + __m256 _r206 = _mm256_broadcast_ss(r2 + 6); + __m256 _r207 = _mm256_broadcast_ss(r2 + 7); + + __m256 _k60 = _mm256_loadu_ps(kptr); + __m256 _k61 = _mm256_loadu_ps(kptr + 8); + __m256 _k62 = _mm256_loadu_ps(kptr + 16); + __m256 _k63 = _mm256_loadu_ps(kptr + 24); + __m256 _k64 = _mm256_loadu_ps(kptr + 32); + __m256 _k65 = _mm256_loadu_ps(kptr + 40); + __m256 _k66 = _mm256_loadu_ps(kptr + 48); + __m256 _k67 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r200, _k60, _sum0); + _sum1 = _mm256_fmadd_ps(_r201, _k61, _sum1); + _sum0 = _mm256_fmadd_ps(_r202, _k62, _sum0); + _sum1 = _mm256_fmadd_ps(_r203, _k63, _sum1); + _sum0 = _mm256_fmadd_ps(_r204, _k64, _sum0); + _sum1 = _mm256_fmadd_ps(_r205, _k65, _sum1); + _sum0 = _mm256_fmadd_ps(_r206, _k66, _sum0); + _sum1 = _mm256_fmadd_ps(_r207, _k67, _sum1); + + __m256 _r210 = _mm256_broadcast_ss(r2 + 8); + __m256 _r211 = _mm256_broadcast_ss(r2 + 9); + __m256 _r212 = _mm256_broadcast_ss(r2 + 10); + __m256 _r213 = _mm256_broadcast_ss(r2 + 11); + __m256 _r214 = _mm256_broadcast_ss(r2 + 12); + __m256 _r215 = _mm256_broadcast_ss(r2 + 13); + __m256 _r216 = _mm256_broadcast_ss(r2 + 14); + __m256 _r217 = _mm256_broadcast_ss(r2 + 15); + + __m256 _k70 = _mm256_loadu_ps(kptr); + __m256 _k71 = _mm256_loadu_ps(kptr + 8); + __m256 _k72 = _mm256_loadu_ps(kptr + 16); + __m256 _k73 = _mm256_loadu_ps(kptr + 24); + __m256 _k74 = _mm256_loadu_ps(kptr + 32); + __m256 _k75 = _mm256_loadu_ps(kptr + 40); + __m256 _k76 = _mm256_loadu_ps(kptr + 48); + __m256 _k77 = _mm256_loadu_ps(kptr + 56); + + kptr += 64; + + _sum0 = _mm256_fmadd_ps(_r210, _k70, _sum0); + _sum1 = _mm256_fmadd_ps(_r211, _k71, _sum1); + _sum0 = _mm256_fmadd_ps(_r212, _k72, _sum0); + _sum1 = _mm256_fmadd_ps(_r213, _k73, _sum1); + _sum0 = _mm256_fmadd_ps(_r214, _k74, _sum0); + _sum1 = _mm256_fmadd_ps(_r215, _k75, _sum1); + _sum0 = _mm256_fmadd_ps(_r216, _k76, _sum0); + _sum1 = _mm256_fmadd_ps(_r217, _k77, _sum1); + + __m256 _r220 = _mm256_broadcast_ss(r2 + 16); + __m256 _r221 = _mm256_broadcast_ss(r2 + 17); + __m256 _r222 = _mm256_broadcast_ss(r2 + 18); + __m256 _r223 = _mm256_broadcast_ss(r2 + 19); + __m256 _r224 = _mm256_broadcast_ss(r2 + 20); + __m256 _r225 = _mm256_broadcast_ss(r2 + 21); + __m256 _r226 = _mm256_broadcast_ss(r2 + 22); + __m256 _r227 = _mm256_broadcast_ss(r2 + 23); + + __m256 _k80 = _mm256_loadu_ps(kptr); + __m256 _k81 = _mm256_loadu_ps(kptr + 8); + __m256 _k82 = _mm256_loadu_ps(kptr + 16); + __m256 _k83 = _mm256_loadu_ps(kptr + 24); + __m256 _k84 = _mm256_loadu_ps(kptr + 32); + __m256 _k85 = _mm256_loadu_ps(kptr + 40); + __m256 _k86 = _mm256_loadu_ps(kptr + 48); + __m256 _k87 = _mm256_loadu_ps(kptr + 56); + + _sum0 = _mm256_fmadd_ps(_r220, _k80, _sum0); + _sum1 = _mm256_fmadd_ps(_r221, _k81, _sum1); + _sum0 = _mm256_fmadd_ps(_r222, _k82, _sum0); + _sum1 = _mm256_fmadd_ps(_r223, _k83, _sum1); + _sum0 = _mm256_fmadd_ps(_r224, _k84, _sum0); + _sum1 = _mm256_fmadd_ps(_r225, _k85, _sum1); + _sum0 = _mm256_fmadd_ps(_r226, _k86, _sum0); + _sum1 = _mm256_fmadd_ps(_r227, _k87, _sum1); + + kptr -= 64 * 8; + + _sum0 = _mm256_add_ps(_sum0, _sum1); + + _mm256_storeu_ps(outptr, _sum0); + + r0 += 8; + r1 += 8; + r2 += 8; + outptr += 8; + } + + r0 += 16; + r1 += 16; + r2 += 16; + } + } + } +} + static void conv3x3s1_winograd64_transform_kernel_pack8_avx(const Mat& kernel, Mat& kernel_tm_pack8, int inch, int outch) { // winograd63 transform kernel diff --git a/src/layer/x86/convolution_x86.cpp b/src/layer/x86/convolution_x86.cpp index 4e843418f..a8612005f 100644 --- a/src/layer/x86/convolution_x86.cpp +++ b/src/layer/x86/convolution_x86.cpp @@ -268,7 +268,7 @@ int Convolution_x86::create_pipeline(const Option& opt) { conv2x2s1_weight_fp16_pack8_avx(weight_data, weight_data_packed, num_input, num_output); } - else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) { conv3x3s1_winograd64_transform_kernel_pack8_avx(weight_data, weight_data_packed, num_input, num_output); } @@ -433,10 +433,16 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_winograd64_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + if (num_input >= 16 && num_output >= 16) + { + conv3x3s1_winograd64_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + } + else + { + conv3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + } if (activation) { diff --git a/tests/test_convolution.cpp b/tests/test_convolution.cpp index 65968f0ce..14df16a7c 100644 --- a/tests/test_convolution.cpp +++ b/tests/test_convolution.cpp @@ -114,6 +114,7 @@ static int test_convolution_0() } return 0 + || test_convolution(11, 11, 8, 16, 3, 1, 1, 1, 1) || test_convolution(13, 16, 16, 24, 3, 1, 1, 1, 1) || test_convolution(8, 8, 16, 24, 3, 1, 1, 1, 0) || test_convolution(4, 8, 16, 24, 3, 1, 1, 1, 1)