diff --git a/src/layer/x86/convolution_3x3_pack8to1_int8.h b/src/layer/x86/convolution_3x3_pack8to1_int8.h index 4f687ac25..d5957faf6 100644 --- a/src/layer/x86/convolution_3x3_pack8to1_int8.h +++ b/src/layer/x86/convolution_3x3_pack8to1_int8.h @@ -125,11 +125,6 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to1_int8_sse(const Mat& k int p = 0; for (; p + 3 < outch; p += 4) { - const Mat k0 = kernel_tm.channel(p); - const Mat k1 = kernel_tm.channel(p + 1); - const Mat k2 = kernel_tm.channel(p + 2); - const Mat k3 = kernel_tm.channel(p + 3); - Mat g0 = kernel_tm_pack8to1.channel(p / 4); for (int k = 0; k < 36; k++) @@ -138,41 +133,15 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to1_int8_sse(const Mat& k for (int q = 0; q + 7 < inch; q += 8) { -#if __AVXVNNI__ || __AVX512VNNI__ || __XOP__ for (int i = 0; i < 4; i++) { - const short* k00 = k0.row(q + i * 2); - const short* k10 = k1.row(q + i * 2); - const short* k20 = k2.row(q + i * 2); - const short* k30 = k3.row(q + i * 2); - - const short* k01 = k0.row(q + i * 2 + 1); - const short* k11 = k1.row(q + i * 2 + 1); - const short* k21 = k2.row(q + i * 2 + 1); - const short* k31 = k3.row(q + i * 2 + 1); - - g00[0] = k00[k]; - g00[1] = k01[k]; - g00[2] = k10[k]; - g00[3] = k11[k]; - g00[4] = k20[k]; - g00[5] = k21[k]; - g00[6] = k30[k]; - g00[7] = k31[k]; - - g00 += 8; - } -#else - for (int i = 0; i < 8; i++) - { - g00[0] = k0.row(q + i)[k]; - g00[1] = k1.row(q + i)[k]; - g00[2] = k2.row(q + i)[k]; - g00[3] = k3.row(q + i)[k]; - - g00 += 4; + for (int j = 0; j < 8; j++) + { + const short* k00 = kernel_tm.channel(p + i).row(q + j); + g00[0] = k00[k]; + g00 += 1; + } } -#endif } } } @@ -508,114 +477,97 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& int nn = inch; // inch always > 0 - __m256i _sum0_1 = _mm256_setzero_si256(); - __m256i _sum2_3 = _mm256_setzero_si256(); - __m256i _sum4_5 = _mm256_setzero_si256(); - __m256i _sum6_7 = _mm256_setzero_si256(); + __m256i _sum00_11 = _mm256_setzero_si256(); + __m256i _sum10_01 = _mm256_setzero_si256(); + __m256i _sum02_13 = _mm256_setzero_si256(); + __m256i _sum12_03 = _mm256_setzero_si256(); + + __m256i _sum04_15 = _mm256_setzero_si256(); + __m256i _sum14_05 = _mm256_setzero_si256(); + __m256i _sum06_17 = _mm256_setzero_si256(); + __m256i _sum16_07 = _mm256_setzero_si256(); for (int j = 0; j < nn; j++) { // 0 1 2 3 4 5 6 7 8 9 a b c d e f - __m256i _val0 = _mm256_loadu_si256((const __m256i*)r0); + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _val10 = _mm256_permute4x64_epi64(_val01, 78); + #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val0_0123); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val0_89ab); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val0_4567); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val0_cdef); + _sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01); + _sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01); + _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23); + _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23); #else - // 0 0 1 1 2 2 3 3 8 8 9 9 a a b b - // 4 4 5 5 6 6 7 7 c c d d e e f f - __m256i _val0_0123_89ab = _mm256_unpacklo_epi16(_val0, _val0); - __m256i _val0_4567_cdef = _mm256_unpackhi_epi16(_val0, _val0); - - __m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val0_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val0_0123); - __m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val0_89ab); - __m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val0_89ab); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val0_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val0_4567); - __m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val0_cdef); - __m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val0_cdef); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23)); #endif - __m256i _val1 = _mm256_loadu_si256((const __m256i*)(r0 + 16)); + __m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16)); + + __m256i _val32 = _mm256_permute4x64_epi64(_val23, 78); #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w01, _val1_0123); - _sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w01, _val1_89ab); - _sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w23, _val1_4567); - _sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w23, _val1_cdef); + _sum04_15 = _mm256_dpwssd_epi32(_sum04_15, _val23, _w01); + _sum14_05 = _mm256_dpwssd_epi32(_sum14_05, _val32, _w01); + _sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23, _w23); + _sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32, _w23); #else - __m256i _val1_0123_89ab = _mm256_unpacklo_epi16(_val1, _val1); - __m256i _val1_4567_cdef = _mm256_unpackhi_epi16(_val1, _val1); - - __m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl04_05 = _mm256_mullo_epi16(_w01, _val1_0123); - __m256i _sh04_05 = _mm256_mulhi_epi16(_w01, _val1_0123); - __m256i _sl14_15 = _mm256_mullo_epi16(_w01, _val1_89ab); - __m256i _sh14_15 = _mm256_mulhi_epi16(_w01, _val1_89ab); - __m256i _sl06_07 = _mm256_mullo_epi16(_w23, _val1_4567); - __m256i _sh06_07 = _mm256_mulhi_epi16(_w23, _val1_4567); - __m256i _sl16_17 = _mm256_mullo_epi16(_w23, _val1_cdef); - __m256i _sh16_17 = _mm256_mulhi_epi16(_w23, _val1_cdef); - - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl04_05, _sh04_05)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl14_15, _sh14_15)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl06_07, _sh06_07)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl16_17, _sh16_17)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl04_05, _sh04_05)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl14_15, _sh14_15)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl06_07, _sh06_07)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl16_17, _sh16_17)); + _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23, _w01)); + _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32, _w01)); + _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23, _w23)); + _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32, _w23)); #endif r0 += 32; k0 += 32; } - __m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1)); - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); + // transpose 4x8 + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01); + _tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03); + _tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01); + _tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum04_15, _sum14_05); + _tmp1 = _mm256_unpacklo_epi32(_sum06_17, _sum16_07); + _tmp2 = _mm256_unpackhi_epi32(_sum04_15, _sum14_05); + _tmp3 = _mm256_unpackhi_epi32(_sum06_17, _sum16_07); + _sum04_15 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum14_05 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum06_17 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum16_07 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } + + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01); + _sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13); - __m256i _sum4_6 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum5_7 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 3, 0, 1)); - _sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7); + _sum04_15 = _mm256_add_epi32(_sum04_15, _sum14_05); + _sum06_17 = _mm256_add_epi32(_sum06_17, _sum16_07); + _sum04_15 = _mm256_add_epi32(_sum04_15, _sum06_17); + + __m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0); + _sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask); + _sum04_15 = _mm256_permutevar8x32_epi32(_sum04_15, _perm_mask); int sum[16]; - _mm256_storeu_si256((__m256i*)sum, _sum0_2); - _mm256_storeu_si256((__m256i*)(sum + 8), _sum4_6); + _mm256_storeu_si256((__m256i*)sum, _sum00_11); + _mm256_storeu_si256((__m256i*)(sum + 8), _sum04_15); output0_tm[0] = sum[0]; output1_tm[0] = sum[1]; @@ -651,60 +603,42 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& int nn = inch; // inch always > 0 #if __AVX2__ - __m256i _sum0_1 = _mm256_setzero_si256(); - __m256i _sum2_3 = _mm256_setzero_si256(); + __m256i _sum00_11 = _mm256_setzero_si256(); + __m256i _sum10_01 = _mm256_setzero_si256(); + __m256i _sum02_13 = _mm256_setzero_si256(); + __m256i _sum12_03 = _mm256_setzero_si256(); #else - __m128i _sum0 = _mm_setzero_si128(); - __m128i _sum1 = _mm_setzero_si128(); - __m128i _sum2 = _mm_setzero_si128(); - __m128i _sum3 = _mm_setzero_si128(); + __m128i _sum00 = _mm_setzero_si128(); + __m128i _sum01 = _mm_setzero_si128(); + __m128i _sum02 = _mm_setzero_si128(); + __m128i _sum03 = _mm_setzero_si128(); + __m128i _sum10 = _mm_setzero_si128(); + __m128i _sum11 = _mm_setzero_si128(); + __m128i _sum12 = _mm_setzero_si128(); + __m128i _sum13 = _mm_setzero_si128(); #endif for (int j = 0; j < nn; j++) { #if __AVX2__ // 0 1 2 3 4 5 6 7 8 9 a b c d e f - __m256i _val = _mm256_loadu_si256((const __m256i*)r0); + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _val10 = _mm256_permute4x64_epi64(_val01, 78); + #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val_0123 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val_4567 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val_89ab = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val_cdef = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val_89ab); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val_cdef); + _sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01); + _sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01); + _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23); + _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23); #else - __m256i _val_0123_89ab = _mm256_unpacklo_epi16(_val, _val); - __m256i _val_4567_cdef = _mm256_unpackhi_epi16(_val, _val); - - __m256i _val_0123 = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val_4567 = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val_89ab = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val_cdef = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123); - __m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val_89ab); - __m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val_89ab); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567); - __m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val_cdef); - __m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val_cdef); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23)); #endif #else // 0 1 2 3 4 5 6 7 @@ -717,75 +651,23 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& __m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24)); #if __XOP__ - __m128i _val0_01 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val0_23 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val0_45 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val0_67 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(3, 3, 3, 3)); - __m128i _val1_01 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val1_23 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val1_45 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val1_67 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(3, 3, 3, 3)); - - _sum0 = _mm_maddd_epi16(_val0_01, _w0, _sum0); - _sum1 = _mm_maddd_epi16(_val0_23, _w1, _sum1); - _sum2 = _mm_maddd_epi16(_val1_01, _w0, _sum2); - _sum3 = _mm_maddd_epi16(_val1_23, _w1, _sum3); - _sum0 = _mm_maddd_epi16(_val0_45, _w2, _sum0); - _sum1 = _mm_maddd_epi16(_val0_67, _w3, _sum1); - _sum2 = _mm_maddd_epi16(_val1_45, _w2, _sum2); - _sum3 = _mm_maddd_epi16(_val1_67, _w3, _sum3); + _sum00 = _mm_maddd_epi16(_val0, _w0, _sum00); + _sum01 = _mm_maddd_epi16(_val0, _w1, _sum01); + _sum02 = _mm_maddd_epi16(_val0, _w2, _sum02); + _sum03 = _mm_maddd_epi16(_val0, _w3, _sum03); + _sum10 = _mm_maddd_epi16(_val1, _w0, _sum10); + _sum11 = _mm_maddd_epi16(_val1, _w1, _sum11); + _sum12 = _mm_maddd_epi16(_val1, _w2, _sum12); + _sum13 = _mm_maddd_epi16(_val1, _w3, _sum13); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m128i _val0_0123 = _mm_unpacklo_epi16(_val0, _val0); - __m128i _val0_4567 = _mm_unpackhi_epi16(_val0, _val0); - - __m128i _val1_0123 = _mm_unpacklo_epi16(_val1, _val1); - __m128i _val1_4567 = _mm_unpackhi_epi16(_val1, _val1); - - __m128i _val0_01 = _mm_unpacklo_epi32(_val0_0123, _val0_0123); - __m128i _val0_23 = _mm_unpackhi_epi32(_val0_0123, _val0_0123); - __m128i _val0_45 = _mm_unpacklo_epi32(_val0_4567, _val0_4567); - __m128i _val0_67 = _mm_unpackhi_epi32(_val0_4567, _val0_4567); - - __m128i _val1_01 = _mm_unpacklo_epi32(_val1_0123, _val1_0123); - __m128i _val1_23 = _mm_unpackhi_epi32(_val1_0123, _val1_0123); - __m128i _val1_45 = _mm_unpacklo_epi32(_val1_4567, _val1_4567); - __m128i _val1_67 = _mm_unpackhi_epi32(_val1_4567, _val1_4567); - - __m128i _sl00 = _mm_mullo_epi16(_w0, _val0_01); - __m128i _sh00 = _mm_mulhi_epi16(_w0, _val0_01); - __m128i _sl10 = _mm_mullo_epi16(_w0, _val1_01); - __m128i _sh10 = _mm_mulhi_epi16(_w0, _val1_01); - __m128i _sl01 = _mm_mullo_epi16(_w1, _val0_23); - __m128i _sh01 = _mm_mulhi_epi16(_w1, _val0_23); - __m128i _sl11 = _mm_mullo_epi16(_w1, _val1_23); - __m128i _sh11 = _mm_mulhi_epi16(_w1, _val1_23); - __m128i _sl02 = _mm_mullo_epi16(_w2, _val0_45); - __m128i _sh02 = _mm_mulhi_epi16(_w2, _val0_45); - __m128i _sl12 = _mm_mullo_epi16(_w2, _val1_45); - __m128i _sh12 = _mm_mulhi_epi16(_w2, _val1_45); - __m128i _sl03 = _mm_mullo_epi16(_w3, _val0_67); - __m128i _sh03 = _mm_mulhi_epi16(_w3, _val0_67); - __m128i _sl13 = _mm_mullo_epi16(_w3, _val1_67); - __m128i _sh13 = _mm_mulhi_epi16(_w3, _val1_67); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl11, _sh11)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl02, _sh02)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl02, _sh02)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl12, _sh12)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl12, _sh12)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl03, _sh03)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl03, _sh03)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl13, _sh13)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl13, _sh13)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02); + _sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); + _sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12); + _sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13); #endif #endif @@ -794,19 +676,64 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& } #if __AVX2__ - __m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1)); - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); + // transpose 4x8 + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01); + _tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03); + _tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01); + _tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } + + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01); + _sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13); + + __m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0); + _sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask); int sum[8]; - _mm256_storeu_si256((__m256i*)sum, _sum0_2); + _mm256_storeu_si256((__m256i*)sum, _sum00_11); #else - _sum0 = _mm_add_epi32(_sum0, _sum1); - _sum2 = _mm_add_epi32(_sum2, _sum3); + // transpose 4x4 + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum00, _sum01); + _tmp1 = _mm_unpacklo_epi32(_sum02, _sum03); + _tmp2 = _mm_unpackhi_epi32(_sum00, _sum01); + _tmp3 = _mm_unpackhi_epi32(_sum02, _sum03); + _sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum10, _sum11); + _tmp1 = _mm_unpacklo_epi32(_sum12, _sum13); + _tmp2 = _mm_unpackhi_epi32(_sum10, _sum11); + _tmp3 = _mm_unpackhi_epi32(_sum12, _sum13); + _sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } + + _sum00 = _mm_add_epi32(_sum00, _sum01); + _sum02 = _mm_add_epi32(_sum02, _sum03); + _sum10 = _mm_add_epi32(_sum10, _sum11); + _sum12 = _mm_add_epi32(_sum12, _sum13); + + _sum00 = _mm_add_epi32(_sum00, _sum02); + _sum10 = _mm_add_epi32(_sum10, _sum12); int sum[8]; - _mm_storeu_si128((__m128i*)sum, _sum0); - _mm_storeu_si128((__m128i*)(sum + 4), _sum2); + _mm_storeu_si128((__m128i*)sum, _sum00); + _mm_storeu_si128((__m128i*)(sum + 4), _sum10); #endif output0_tm[0] = sum[0]; @@ -835,9 +762,12 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& #if __AVX2__ __m256i _sum0_1 = _mm256_setzero_si256(); + __m256i _sum2_3 = _mm256_setzero_si256(); #else __m128i _sum0 = _mm_setzero_si128(); __m128i _sum1 = _mm_setzero_si128(); + __m128i _sum2 = _mm_setzero_si128(); + __m128i _sum3 = _mm_setzero_si128(); #endif for (int j = 0; j < nn; j++) @@ -849,37 +779,14 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _valval = _mm256_inserti128_si256(_mm256_castsi128_si256(_val), _val, 1); + #if __AVXVNNI__ || __AVX512VNNI__ - // 0 1 0 1 x x x x - // 0 1 0 1 0 1 0 1 - __m128i _val_01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val_23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val_45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val_67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3)); - - __m256i _val_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_01), _val_23, 1); - __m256i _val_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_45), _val_67, 1); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567); + _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01); + _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m256i _val_0123 = _mm256_castsi128_si256(_mm_unpacklo_epi16(_val, _val)); - __m256i _val_4567 = _mm256_castsi128_si256(_mm_unpackhi_epi16(_val, _val)); - - _val_0123 = _mm256_permutevar8x32_epi32(_val_0123, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - _val_4567 = _mm256_permutevar8x32_epi32(_val_4567, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); + _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01)); + _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23)); #endif #else __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); @@ -888,43 +795,15 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& __m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24)); #if __XOP__ - __m128i _val01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3)); - - _sum0 = _mm_maddd_epi16(_val01, _w0, _sum0); - _sum1 = _mm_maddd_epi16(_val23, _w1, _sum1); - _sum0 = _mm_maddd_epi16(_val45, _w2, _sum0); - _sum1 = _mm_maddd_epi16(_val67, _w3, _sum1); + _sum0 = _mm_maddd_epi16(_val, _w0, _sum0); + _sum1 = _mm_maddd_epi16(_val, _w1, _sum1); + _sum2 = _mm_maddd_epi16(_val, _w2, _sum2); + _sum3 = _mm_maddd_epi16(_val, _w3, _sum3); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m128i _val_0123 = _mm_unpacklo_epi16(_val, _val); - __m128i _val_4567 = _mm_unpackhi_epi16(_val, _val); - - __m128i _val01 = _mm_unpacklo_epi32(_val_0123, _val_0123); - __m128i _val23 = _mm_unpackhi_epi32(_val_0123, _val_0123); - __m128i _val45 = _mm_unpacklo_epi32(_val_4567, _val_4567); - __m128i _val67 = _mm_unpackhi_epi32(_val_4567, _val_4567); - - __m128i _sl0 = _mm_mullo_epi16(_w0, _val01); - __m128i _sh0 = _mm_mulhi_epi16(_w0, _val01); - __m128i _sl1 = _mm_mullo_epi16(_w1, _val23); - __m128i _sh1 = _mm_mulhi_epi16(_w1, _val23); - __m128i _sl2 = _mm_mullo_epi16(_w2, _val45); - __m128i _sh2 = _mm_mulhi_epi16(_w2, _val45); - __m128i _sl3 = _mm_mullo_epi16(_w3, _val67); - __m128i _sh3 = _mm_mulhi_epi16(_w3, _val67); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl2, _sh2)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl2, _sh2)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl3, _sh3)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl3, _sh3)); + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1); + _sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2); + _sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3); #endif #endif @@ -935,8 +814,27 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& #if __AVX2__ __m128i _sum0 = _mm256_extracti128_si256(_sum0_1, 0); __m128i _sum1 = _mm256_extracti128_si256(_sum0_1, 1); + __m128i _sum2 = _mm256_extracti128_si256(_sum2_3, 0); + __m128i _sum3 = _mm256_extracti128_si256(_sum2_3, 1); #endif + + // transpose 4x4 + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum0, _sum1); + _tmp1 = _mm_unpacklo_epi32(_sum2, _sum3); + _tmp2 = _mm_unpackhi_epi32(_sum0, _sum1); + _tmp3 = _mm_unpackhi_epi32(_sum2, _sum3); + _sum0 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum1 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum2 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum3 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } + _sum0 = _mm_add_epi32(_sum0, _sum1); + _sum2 = _mm_add_epi32(_sum2, _sum3); + + _sum0 = _mm_add_epi32(_sum0, _sum2); int sum[4]; _mm_storeu_si128((__m128i*)sum, _sum0); @@ -973,55 +871,38 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& const short* r0 = bb2.row(i / 4); const short* k0 = kernel0_tm.row(r); - __m128i _sum0 = _mm_setzero_si128(); - __m128i _sum1 = _mm_setzero_si128(); - __m128i _sum2 = _mm_setzero_si128(); - __m128i _sum3 = _mm_setzero_si128(); - __m128i _sum4 = _mm_setzero_si128(); - __m128i _sum5 = _mm_setzero_si128(); - __m128i _sum6 = _mm_setzero_si128(); - __m128i _sum7 = _mm_setzero_si128(); + __m256i _sum01 = _mm256_setzero_si256(); + __m256i _sum23 = _mm256_setzero_si256(); for (int q = 0; q < inch; q++) { - __m128i _val0 = _mm_loadu_si128((const __m128i*)r0); - __m128i _val1 = _mm_loadu_si128((const __m128i*)(r0 + 8)); - __m128i _val2 = _mm_loadu_si128((const __m128i*)(r0 + 16)); - __m128i _val3 = _mm_loadu_si128((const __m128i*)(r0 + 24)); + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); + __m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16)); __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); + __m256i _w01 = _mm256_inserti128_si256(_mm256_castsi128_si256(_w0), _w0, 1); - __m128i _sl0 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh0 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl1 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh1 = _mm_mulhi_epi16(_val1, _w0); - __m128i _sl2 = _mm_mullo_epi16(_val2, _w0); - __m128i _sh2 = _mm_mulhi_epi16(_val2, _w0); - __m128i _sl3 = _mm_mullo_epi16(_val3, _w0); - __m128i _sh3 = _mm_mulhi_epi16(_val3, _w0); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl1, _sh1)); - _sum4 = _mm_add_epi32(_sum4, _mm_unpacklo_epi16(_sl2, _sh2)); - _sum5 = _mm_add_epi32(_sum5, _mm_unpackhi_epi16(_sl2, _sh2)); - _sum6 = _mm_add_epi32(_sum6, _mm_unpacklo_epi16(_sl3, _sh3)); - _sum7 = _mm_add_epi32(_sum7, _mm_unpackhi_epi16(_sl3, _sh3)); +#if __AVXVNNI__ || __AVX512VNNI__ + _sum01 = _mm256_dpwssd_epi32(_sum01, _val01, _w01); + _sum23 = _mm256_dpwssd_epi32(_sum23, _val23, _w01); +#else + _sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01, _w01)); + _sum23 = _mm256_add_epi32(_sum23, _mm256_madd_epi16(_val23, _w01)); +#endif k0 += 8; r0 += 32; } - _sum0 = _mm_add_epi32(_sum0, _sum1); - _sum2 = _mm_add_epi32(_sum2, _sum3); - _sum4 = _mm_add_epi32(_sum4, _sum5); - _sum6 = _mm_add_epi32(_sum6, _sum7); + __m128i _sum0 = _mm256_extracti128_si256(_sum01, 0); + __m128i _sum1 = _mm256_extracti128_si256(_sum01, 1); + __m128i _sum2 = _mm256_extracti128_si256(_sum23, 0); + __m128i _sum3 = _mm256_extracti128_si256(_sum23, 1); output0_tm[0] = _mm_reduce_add_epi32(_sum0); - output0_tm[1] = _mm_reduce_add_epi32(_sum2); - output0_tm[2] = _mm_reduce_add_epi32(_sum4); - output0_tm[3] = _mm_reduce_add_epi32(_sum6); + output0_tm[1] = _mm_reduce_add_epi32(_sum1); + output0_tm[2] = _mm_reduce_add_epi32(_sum2); + output0_tm[3] = _mm_reduce_add_epi32(_sum3); output0_tm += 4; } #endif @@ -1034,37 +915,52 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& #endif const short* k0 = kernel0_tm.row(r); +#if __AVX2__ + __m256i _sum01 = _mm256_setzero_si256(); +#else __m128i _sum0 = _mm_setzero_si128(); __m128i _sum1 = _mm_setzero_si128(); - __m128i _sum2 = _mm_setzero_si128(); - __m128i _sum3 = _mm_setzero_si128(); +#endif for (int q = 0; q < inch; q++) { +#if __AVX2__ + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); + + __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); + __m256i _w01 = _mm256_inserti128_si256(_mm256_castsi128_si256(_w0), _w0, 1); + +#if __AVXVNNI__ || __AVX512VNNI__ + _sum01 = _mm256_dpwssd_epi32(_sum01, _val01, _w01); +#else + _sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01, _w01)); +#endif +#else __m128i _val0 = _mm_loadu_si128((const __m128i*)r0); __m128i _val1 = _mm_loadu_si128((const __m128i*)(r0 + 8)); __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); - __m128i _sl0 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh0 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl1 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh1 = _mm_mulhi_epi16(_val1, _w0); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl1, _sh1)); +#if __XOP__ + _sum0 = _mm_maddd_epi16(_val0, _w0, _sum0); + _sum1 = _mm_maddd_epi16(_val1, _w0, _sum1); +#else + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum1); +#endif +#endif k0 += 8; r0 += 16; } - _sum0 = _mm_add_epi32(_sum0, _sum1); - _sum2 = _mm_add_epi32(_sum2, _sum3); +#if __AVX2__ + __m128i _sum0 = _mm256_extracti128_si256(_sum01, 0); + __m128i _sum1 = _mm256_extracti128_si256(_sum01, 1); +#endif output0_tm[0] = _mm_reduce_add_epi32(_sum0); - output0_tm[1] = _mm_reduce_add_epi32(_sum2); + output0_tm[1] = _mm_reduce_add_epi32(_sum1); output0_tm += 2; } for (; i < tiles; i++) @@ -1077,26 +973,23 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat& const short* k0 = kernel0_tm.row(r); __m128i _sum0 = _mm_setzero_si128(); - __m128i _sum1 = _mm_setzero_si128(); for (int q = 0; q < inch; q++) { - __m128i _val = _mm_loadu_si128((const __m128i*)r0); + __m128i _val0 = _mm_loadu_si128((const __m128i*)r0); __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); - __m128i _sl0 = _mm_mullo_epi16(_val, _w0); - __m128i _sh0 = _mm_mulhi_epi16(_val, _w0); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0)); +#if __XOP__ + _sum0 = _mm_maddd_epi16(_val0, _w0, _sum0); +#else + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0); +#endif k0 += 8; r0 += 8; } - _sum0 = _mm_add_epi32(_sum0, _sum1); - output0_tm[0] = _mm_reduce_add_epi32(_sum0); output0_tm++; } diff --git a/src/layer/x86/convolution_3x3_pack8to4_int8.h b/src/layer/x86/convolution_3x3_pack8to4_int8.h index 547de8774..2bb48ce19 100644 --- a/src/layer/x86/convolution_3x3_pack8to4_int8.h +++ b/src/layer/x86/convolution_3x3_pack8to4_int8.h @@ -125,59 +125,23 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to4_int8_sse(const Mat& k int q = 0; for (; q + 3 < outch; q += 4) { - const Mat k0 = kernel_tm.channel(q); - const Mat k1 = kernel_tm.channel(q + 1); - const Mat k2 = kernel_tm.channel(q + 2); - const Mat k3 = kernel_tm.channel(q + 3); - - Mat kernel_tm = kernel_tm_pack8.channel(q / 4); + Mat g0 = kernel_tm_pack8.channel(q / 4); for (int k = 0; k < 36; k++) { - short* g00 = kernel_tm.row(k); + short* g00 = g0.row(k); for (int p = 0; p + 7 < inch; p += 8) { -#if __AVXVNNI__ || __AVX512VNNI__ || __XOP__ for (int i = 0; i < 4; i++) { - const short* k00 = k0.row(p + i * 2); - const short* k10 = k1.row(p + i * 2); - const short* k20 = k2.row(p + i * 2); - const short* k30 = k3.row(p + i * 2); - - const short* k01 = k0.row(p + i * 2 + 1); - const short* k11 = k1.row(p + i * 2 + 1); - const short* k21 = k2.row(p + i * 2 + 1); - const short* k31 = k3.row(p + i * 2 + 1); - - g00[0] = k00[k]; - g00[1] = k01[k]; - g00[2] = k10[k]; - g00[3] = k11[k]; - g00[4] = k20[k]; - g00[5] = k21[k]; - g00[6] = k30[k]; - g00[7] = k31[k]; - - g00 += 8; - } -#else - for (int i = 0; i < 8; i++) - { - const short* k00 = k0.row(p + i); - const short* k10 = k1.row(p + i); - const short* k20 = k2.row(p + i); - const short* k30 = k3.row(p + i); - - g00[0] = k00[k]; - g00[1] = k10[k]; - g00[2] = k20[k]; - g00[3] = k30[k]; - - g00 += 4; + for (int j = 0; j < 8; j++) + { + const short* k00 = kernel_tm.channel(q + i).row(p + j); + g00[0] = k00[k]; + g00 += 1; + } } -#endif } } } @@ -482,113 +446,96 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& int nn = inch; // inch always > 0 - __m256i _sum0_1 = _mm256_setzero_si256(); - __m256i _sum2_3 = _mm256_setzero_si256(); - __m256i _sum4_5 = _mm256_setzero_si256(); - __m256i _sum6_7 = _mm256_setzero_si256(); + __m256i _sum00_11 = _mm256_setzero_si256(); + __m256i _sum10_01 = _mm256_setzero_si256(); + __m256i _sum02_13 = _mm256_setzero_si256(); + __m256i _sum12_03 = _mm256_setzero_si256(); + + __m256i _sum04_15 = _mm256_setzero_si256(); + __m256i _sum14_05 = _mm256_setzero_si256(); + __m256i _sum06_17 = _mm256_setzero_si256(); + __m256i _sum16_07 = _mm256_setzero_si256(); for (int j = 0; j < nn; j++) { // 0 1 2 3 4 5 6 7 8 9 a b c d e f - __m256i _val0 = _mm256_loadu_si256((const __m256i*)r0); + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _val10 = _mm256_permute4x64_epi64(_val01, 78); + #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val0_0123); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val0_89ab); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val0_4567); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val0_cdef); + _sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01); + _sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01); + _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23); + _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23); #else - // 0 0 1 1 2 2 3 3 8 8 9 9 a a b b - // 4 4 5 5 6 6 7 7 c c d d e e f f - __m256i _val0_0123_89ab = _mm256_unpacklo_epi16(_val0, _val0); - __m256i _val0_4567_cdef = _mm256_unpackhi_epi16(_val0, _val0); - - __m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val0_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val0_0123); - __m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val0_89ab); - __m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val0_89ab); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val0_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val0_4567); - __m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val0_cdef); - __m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val0_cdef); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23)); #endif - __m256i _val1 = _mm256_loadu_si256((const __m256i*)(r0 + 16)); + __m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16)); + + __m256i _val32 = _mm256_permute4x64_epi64(_val23, 78); #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w01, _val1_0123); - _sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w01, _val1_89ab); - _sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w23, _val1_4567); - _sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w23, _val1_cdef); + _sum04_15 = _mm256_dpwssd_epi32(_sum04_15, _val23, _w01); + _sum14_05 = _mm256_dpwssd_epi32(_sum14_05, _val32, _w01); + _sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23, _w23); + _sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32, _w23); #else - __m256i _val1_0123_89ab = _mm256_unpacklo_epi16(_val1, _val1); - __m256i _val1_4567_cdef = _mm256_unpackhi_epi16(_val1, _val1); - - __m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl04_05 = _mm256_mullo_epi16(_w01, _val1_0123); - __m256i _sh04_05 = _mm256_mulhi_epi16(_w01, _val1_0123); - __m256i _sl14_15 = _mm256_mullo_epi16(_w01, _val1_89ab); - __m256i _sh14_15 = _mm256_mulhi_epi16(_w01, _val1_89ab); - __m256i _sl06_07 = _mm256_mullo_epi16(_w23, _val1_4567); - __m256i _sh06_07 = _mm256_mulhi_epi16(_w23, _val1_4567); - __m256i _sl16_17 = _mm256_mullo_epi16(_w23, _val1_cdef); - __m256i _sh16_17 = _mm256_mulhi_epi16(_w23, _val1_cdef); - - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl04_05, _sh04_05)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl14_15, _sh14_15)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl06_07, _sh06_07)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl16_17, _sh16_17)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl04_05, _sh04_05)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl14_15, _sh14_15)); - _sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl06_07, _sh06_07)); - _sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl16_17, _sh16_17)); + _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23, _w01)); + _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32, _w01)); + _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23, _w23)); + _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32, _w23)); #endif r0 += 32; k0 += 32; } - __m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1)); - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); + // transpose 4x8 + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01); + _tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03); + _tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01); + _tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum04_15, _sum14_05); + _tmp1 = _mm256_unpacklo_epi32(_sum06_17, _sum16_07); + _tmp2 = _mm256_unpackhi_epi32(_sum04_15, _sum14_05); + _tmp3 = _mm256_unpackhi_epi32(_sum06_17, _sum16_07); + _sum04_15 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum14_05 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum06_17 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum16_07 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } + + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01); + _sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13); - __m256i _sum4_6 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum5_7 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 3, 0, 1)); - _sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7); + _sum04_15 = _mm256_add_epi32(_sum04_15, _sum14_05); + _sum06_17 = _mm256_add_epi32(_sum06_17, _sum16_07); + _sum04_15 = _mm256_add_epi32(_sum04_15, _sum06_17); - _mm256_storeu_si256((__m256i*)output0_tm, _sum0_2); - _mm256_storeu_si256((__m256i*)(output0_tm + 8), _sum4_6); + __m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0); + _sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask); + _sum04_15 = _mm256_permutevar8x32_epi32(_sum04_15, _perm_mask); + + _mm256_storeu_si256((__m256i*)output0_tm, _sum00_11); + _mm256_storeu_si256((__m256i*)(output0_tm + 8), _sum04_15); output0_tm += 16; } #endif @@ -604,60 +551,42 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& int nn = inch; // inch always > 0 #if __AVX2__ - __m256i _sum0_1 = _mm256_setzero_si256(); - __m256i _sum2_3 = _mm256_setzero_si256(); + __m256i _sum00_11 = _mm256_setzero_si256(); + __m256i _sum10_01 = _mm256_setzero_si256(); + __m256i _sum02_13 = _mm256_setzero_si256(); + __m256i _sum12_03 = _mm256_setzero_si256(); #else - __m128i _sum0 = _mm_setzero_si128(); - __m128i _sum1 = _mm_setzero_si128(); - __m128i _sum2 = _mm_setzero_si128(); - __m128i _sum3 = _mm_setzero_si128(); + __m128i _sum00 = _mm_setzero_si128(); + __m128i _sum01 = _mm_setzero_si128(); + __m128i _sum02 = _mm_setzero_si128(); + __m128i _sum03 = _mm_setzero_si128(); + __m128i _sum10 = _mm_setzero_si128(); + __m128i _sum11 = _mm_setzero_si128(); + __m128i _sum12 = _mm_setzero_si128(); + __m128i _sum13 = _mm_setzero_si128(); #endif for (int j = 0; j < nn; j++) { #if __AVX2__ // 0 1 2 3 4 5 6 7 8 9 a b c d e f - __m256i _val = _mm256_loadu_si256((const __m256i*)r0); + __m256i _val01 = _mm256_loadu_si256((const __m256i*)r0); __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _val10 = _mm256_permute4x64_epi64(_val01, 78); + #if __AVXVNNI__ || __AVX512VNNI__ - __m256i _val_0123 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0)); - __m256i _val_4567 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2)); - __m256i _val_89ab = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4)); - __m256i _val_cdef = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6)); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val_89ab); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567); - _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val_cdef); + _sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01); + _sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01); + _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23); + _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23); #else - __m256i _val_0123_89ab = _mm256_unpacklo_epi16(_val, _val); - __m256i _val_4567_cdef = _mm256_unpackhi_epi16(_val, _val); - - __m256i _val_0123 = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val_4567 = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - __m256i _val_89ab = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - __m256i _val_cdef = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123); - __m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val_89ab); - __m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val_89ab); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567); - __m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val_cdef); - __m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val_cdef); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23)); #endif #else // 0 1 2 3 4 5 6 7 @@ -670,75 +599,23 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& __m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24)); #if __XOP__ - __m128i _val0_01 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val0_23 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val0_45 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val0_67 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(3, 3, 3, 3)); - __m128i _val1_01 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val1_23 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val1_45 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val1_67 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(3, 3, 3, 3)); - - _sum0 = _mm_maddd_epi16(_val0_01, _w0, _sum0); - _sum1 = _mm_maddd_epi16(_val0_23, _w1, _sum1); - _sum2 = _mm_maddd_epi16(_val1_01, _w0, _sum2); - _sum3 = _mm_maddd_epi16(_val1_23, _w1, _sum3); - _sum0 = _mm_maddd_epi16(_val0_45, _w2, _sum0); - _sum1 = _mm_maddd_epi16(_val0_67, _w3, _sum1); - _sum2 = _mm_maddd_epi16(_val1_45, _w2, _sum2); - _sum3 = _mm_maddd_epi16(_val1_67, _w3, _sum3); + _sum00 = _mm_maddd_epi16(_val0, _w0, _sum00); + _sum01 = _mm_maddd_epi16(_val0, _w1, _sum01); + _sum02 = _mm_maddd_epi16(_val0, _w2, _sum02); + _sum03 = _mm_maddd_epi16(_val0, _w3, _sum03); + _sum10 = _mm_maddd_epi16(_val1, _w0, _sum10); + _sum11 = _mm_maddd_epi16(_val1, _w1, _sum11); + _sum12 = _mm_maddd_epi16(_val1, _w2, _sum12); + _sum13 = _mm_maddd_epi16(_val1, _w3, _sum13); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m128i _val0_0123 = _mm_unpacklo_epi16(_val0, _val0); - __m128i _val0_4567 = _mm_unpackhi_epi16(_val0, _val0); - - __m128i _val1_0123 = _mm_unpacklo_epi16(_val1, _val1); - __m128i _val1_4567 = _mm_unpackhi_epi16(_val1, _val1); - - __m128i _val0_01 = _mm_unpacklo_epi32(_val0_0123, _val0_0123); - __m128i _val0_23 = _mm_unpackhi_epi32(_val0_0123, _val0_0123); - __m128i _val0_45 = _mm_unpacklo_epi32(_val0_4567, _val0_4567); - __m128i _val0_67 = _mm_unpackhi_epi32(_val0_4567, _val0_4567); - - __m128i _val1_01 = _mm_unpacklo_epi32(_val1_0123, _val1_0123); - __m128i _val1_23 = _mm_unpackhi_epi32(_val1_0123, _val1_0123); - __m128i _val1_45 = _mm_unpacklo_epi32(_val1_4567, _val1_4567); - __m128i _val1_67 = _mm_unpackhi_epi32(_val1_4567, _val1_4567); - - __m128i _sl00 = _mm_mullo_epi16(_w0, _val0_01); - __m128i _sh00 = _mm_mulhi_epi16(_w0, _val0_01); - __m128i _sl10 = _mm_mullo_epi16(_w0, _val1_01); - __m128i _sh10 = _mm_mulhi_epi16(_w0, _val1_01); - __m128i _sl01 = _mm_mullo_epi16(_w1, _val0_23); - __m128i _sh01 = _mm_mulhi_epi16(_w1, _val0_23); - __m128i _sl11 = _mm_mullo_epi16(_w1, _val1_23); - __m128i _sh11 = _mm_mulhi_epi16(_w1, _val1_23); - __m128i _sl02 = _mm_mullo_epi16(_w2, _val0_45); - __m128i _sh02 = _mm_mulhi_epi16(_w2, _val0_45); - __m128i _sl12 = _mm_mullo_epi16(_w2, _val1_45); - __m128i _sh12 = _mm_mulhi_epi16(_w2, _val1_45); - __m128i _sl03 = _mm_mullo_epi16(_w3, _val0_67); - __m128i _sh03 = _mm_mulhi_epi16(_w3, _val0_67); - __m128i _sl13 = _mm_mullo_epi16(_w3, _val1_67); - __m128i _sh13 = _mm_mulhi_epi16(_w3, _val1_67); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl11, _sh11)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl02, _sh02)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl02, _sh02)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl12, _sh12)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl12, _sh12)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl03, _sh03)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl03, _sh03)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl13, _sh13)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl13, _sh13)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02); + _sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); + _sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12); + _sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13); #endif #endif @@ -747,17 +624,62 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& } #if __AVX2__ - __m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0)); - __m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1)); - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); + // transpose 4x8 + { + __m256i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01); + _tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03); + _tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01); + _tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1); + _sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1); + _sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); + _sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); + } - _mm256_storeu_si256((__m256i*)output0_tm, _sum0_2); + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01); + _sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03); + _sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13); + + __m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0); + _sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask); + + _mm256_storeu_si256((__m256i*)output0_tm, _sum00_11); #else - _sum0 = _mm_add_epi32(_sum0, _sum1); - _sum2 = _mm_add_epi32(_sum2, _sum3); + // transpose 4x4 + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum00, _sum01); + _tmp1 = _mm_unpacklo_epi32(_sum02, _sum03); + _tmp2 = _mm_unpackhi_epi32(_sum00, _sum01); + _tmp3 = _mm_unpackhi_epi32(_sum02, _sum03); + _sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum10, _sum11); + _tmp1 = _mm_unpacklo_epi32(_sum12, _sum13); + _tmp2 = _mm_unpackhi_epi32(_sum10, _sum11); + _tmp3 = _mm_unpackhi_epi32(_sum12, _sum13); + _sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } - _mm_storeu_si128((__m128i*)output0_tm, _sum0); - _mm_storeu_si128((__m128i*)(output0_tm + 4), _sum2); + _sum00 = _mm_add_epi32(_sum00, _sum01); + _sum02 = _mm_add_epi32(_sum02, _sum03); + _sum10 = _mm_add_epi32(_sum10, _sum11); + _sum12 = _mm_add_epi32(_sum12, _sum13); + + _sum00 = _mm_add_epi32(_sum00, _sum02); + _sum10 = _mm_add_epi32(_sum10, _sum12); + + _mm_storeu_si128((__m128i*)output0_tm, _sum00); + _mm_storeu_si128((__m128i*)(output0_tm + 4), _sum10); #endif output0_tm += 8; } @@ -774,9 +696,12 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& #if __AVX2__ __m256i _sum0_1 = _mm256_setzero_si256(); + __m256i _sum2_3 = _mm256_setzero_si256(); #else __m128i _sum0 = _mm_setzero_si128(); __m128i _sum1 = _mm_setzero_si128(); + __m128i _sum2 = _mm_setzero_si128(); + __m128i _sum3 = _mm_setzero_si128(); #endif for (int j = 0; j < nn; j++) @@ -787,37 +712,14 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& __m256i _w01 = _mm256_loadu_si256((const __m256i*)k0); __m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16)); + __m256i _valval = _mm256_inserti128_si256(_mm256_castsi128_si256(_val), _val, 1); + #if __AVXVNNI__ || __AVX512VNNI__ - // 0 1 0 1 x x x x - // 0 1 0 1 0 1 0 1 - __m128i _val_01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val_23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val_45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val_67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3)); - - __m256i _val_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_01), _val_23, 1); - __m256i _val_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_45), _val_67, 1); - - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123); - _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567); + _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01); + _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m256i _val_0123 = _mm256_castsi128_si256(_mm_unpacklo_epi16(_val, _val)); - __m256i _val_4567 = _mm256_castsi128_si256(_mm_unpackhi_epi16(_val, _val)); - - _val_0123 = _mm256_permutevar8x32_epi32(_val_0123, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - _val_4567 = _mm256_permutevar8x32_epi32(_val_4567, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - - __m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123); - __m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123); - __m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567); - __m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03)); + _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01)); + _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23)); #endif #else __m128i _w0 = _mm_loadu_si128((const __m128i*)k0); @@ -826,43 +728,15 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& __m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24)); #if __XOP__ - __m128i _val01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0)); - __m128i _val23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1)); - __m128i _val45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2)); - __m128i _val67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3)); - - _sum0 = _mm_maddd_epi16(_val01, _w0, _sum0); - _sum1 = _mm_maddd_epi16(_val23, _w1, _sum1); - _sum0 = _mm_maddd_epi16(_val45, _w2, _sum0); - _sum1 = _mm_maddd_epi16(_val67, _w3, _sum1); + _sum0 = _mm_maddd_epi16(_val, _w0, _sum0); + _sum1 = _mm_maddd_epi16(_val, _w1, _sum1); + _sum2 = _mm_maddd_epi16(_val, _w2, _sum2); + _sum3 = _mm_maddd_epi16(_val, _w3, _sum3); #else - // 0 0 1 1 2 2 3 3 - // 4 4 5 5 6 6 7 7 - __m128i _val_0123 = _mm_unpacklo_epi16(_val, _val); - __m128i _val_4567 = _mm_unpackhi_epi16(_val, _val); - - __m128i _val01 = _mm_unpacklo_epi32(_val_0123, _val_0123); - __m128i _val23 = _mm_unpackhi_epi32(_val_0123, _val_0123); - __m128i _val45 = _mm_unpacklo_epi32(_val_4567, _val_4567); - __m128i _val67 = _mm_unpackhi_epi32(_val_4567, _val_4567); - - __m128i _sl0 = _mm_mullo_epi16(_w0, _val01); - __m128i _sh0 = _mm_mulhi_epi16(_w0, _val01); - __m128i _sl1 = _mm_mullo_epi16(_w1, _val23); - __m128i _sh1 = _mm_mulhi_epi16(_w1, _val23); - __m128i _sl2 = _mm_mullo_epi16(_w2, _val45); - __m128i _sh2 = _mm_mulhi_epi16(_w2, _val45); - __m128i _sl3 = _mm_mullo_epi16(_w3, _val67); - __m128i _sh3 = _mm_mulhi_epi16(_w3, _val67); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl2, _sh2)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl2, _sh2)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl3, _sh3)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl3, _sh3)); + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1); + _sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2); + _sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3); #endif #endif @@ -873,8 +747,27 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat& #if __AVX2__ __m128i _sum0 = _mm256_extracti128_si256(_sum0_1, 0); __m128i _sum1 = _mm256_extracti128_si256(_sum0_1, 1); + __m128i _sum2 = _mm256_extracti128_si256(_sum2_3, 0); + __m128i _sum3 = _mm256_extracti128_si256(_sum2_3, 1); #endif + + // transpose 4x4 + { + __m128i _tmp0, _tmp1, _tmp2, _tmp3; + _tmp0 = _mm_unpacklo_epi32(_sum0, _sum1); + _tmp1 = _mm_unpacklo_epi32(_sum2, _sum3); + _tmp2 = _mm_unpackhi_epi32(_sum0, _sum1); + _tmp3 = _mm_unpackhi_epi32(_sum2, _sum3); + _sum0 = _mm_unpacklo_epi64(_tmp0, _tmp1); + _sum1 = _mm_unpackhi_epi64(_tmp0, _tmp1); + _sum2 = _mm_unpacklo_epi64(_tmp2, _tmp3); + _sum3 = _mm_unpackhi_epi64(_tmp2, _tmp3); + } + _sum0 = _mm_add_epi32(_sum0, _sum1); + _sum2 = _mm_add_epi32(_sum2, _sum3); + + _sum0 = _mm_add_epi32(_sum0, _sum2); _mm_storeu_si128((__m128i*)output0_tm, _sum0); output0_tm += 4; diff --git a/src/layer/x86/convolution_sgemm_int8.h b/src/layer/x86/convolution_sgemm_int8.h index a533ce79d..34097f057 100644 --- a/src/layer/x86/convolution_sgemm_int8.h +++ b/src/layer/x86/convolution_sgemm_int8.h @@ -338,17 +338,8 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const if (nn4 > 0) { -#if __AVXVNNI__ || __AVX512VNNI__ - __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum30_22 = _mm256_setzero_si256(); -#else __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum01_13 = _mm256_setzero_si256(); - __m256i _sum11_03 = _mm256_setzero_si256(); __m256i _sum30_22 = _mm256_setzero_si256(); - __m256i _sum21_33 = _mm256_setzero_si256(); - __m256i _sum31_23 = _mm256_setzero_si256(); -#endif int j = 0; for (; j < nn4; j++) @@ -371,72 +362,21 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const _sum20_32 = _mm256_dpwssd_epi32(_sum20_32, _val23_16, _w01_16); _sum30_22 = _mm256_dpwssd_epi32(_sum30_22, _val32_16, _w01_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl20_31 = _mm256_mullo_epi16(_val23_16, _w01_16); - __m256i _sh20_31 = _mm256_mulhi_epi16(_val23_16, _w01_16); - __m256i _sl30_21 = _mm256_mullo_epi16(_val32_16, _w01_16); - __m256i _sh30_21 = _mm256_mulhi_epi16(_val32_16, _w01_16); - - _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_unpacklo_epi16(_sl20_31, _sh20_31)); - _sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_unpacklo_epi16(_sl30_21, _sh30_21)); - _sum21_33 = _mm256_add_epi32(_sum21_33, _mm256_unpackhi_epi16(_sl20_31, _sh20_31)); - _sum31_23 = _mm256_add_epi32(_sum31_23, _mm256_unpackhi_epi16(_sl30_21, _sh30_21)); + _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_madd_epi16(_val23_16, _w01_16)); + _sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_madd_epi16(_val32_16, _w01_16)); #endif tmpptr += 16; kptr0 += 16; } -#if __AVXVNNI__ || __AVX512VNNI__ _sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02); _sum20_32 = _mm256_hadd_epi32(_sum20_32, _sum30_22); _sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0)); _sum20_32 = _mm256_permute4x64_epi64(_sum20_32, _MM_SHUFFLE(2, 1, 3, 0)); -#else - // transpose 4x8 - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02); - _tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03); - _tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02); - _tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum20_32, _sum30_22); - _tmp1 = _mm256_unpacklo_epi32(_sum21_33, _sum31_23); - _tmp2 = _mm256_unpackhi_epi32(_sum20_32, _sum30_22); - _tmp3 = _mm256_unpackhi_epi32(_sum21_33, _sum31_23); - _sum20_32 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum30_22 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum21_33 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum31_23 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02); - _sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13); - - _sum20_32 = _mm256_add_epi32(_sum20_32, _sum30_22); - _sum21_33 = _mm256_add_epi32(_sum21_33, _sum31_23); - _sum20_32 = _mm256_add_epi32(_sum20_32, _sum21_33); - - __m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0); - _sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask); - _sum20_32 = _mm256_permutevar8x32_epi32(_sum20_32, _perm_mask); -#endif } __m128i _sum00 = _mm256_extracti128_si256(_sum00_12, 0); @@ -532,25 +472,10 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const if (nn4 > 0) { #if __AVX2__ -#if __AVXVNNI__ || __AVX512VNNI__ - __m256i _sum10_02 = _mm256_setzero_si256(); -#else __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum01_13 = _mm256_setzero_si256(); - __m256i _sum11_03 = _mm256_setzero_si256(); -#endif -#else -#if __XOP__ - __m128i _sum01 = _mm_setzero_si128(); - __m128i _sum11 = _mm_setzero_si128(); #else __m128i _sum01 = _mm_setzero_si128(); - __m128i _sum02 = _mm_setzero_si128(); - __m128i _sum03 = _mm_setzero_si128(); __m128i _sum11 = _mm_setzero_si128(); - __m128i _sum12 = _mm_setzero_si128(); - __m128i _sum13 = _mm_setzero_si128(); -#endif #endif int j = 0; @@ -571,15 +496,8 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const _sum00_12 = _mm256_dpwssd_epi32(_sum00_12, _val01_16, _w01_16); _sum10_02 = _mm256_dpwssd_epi32(_sum10_02, _val10_16, _w01_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - - _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); + _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16)); #endif #else __m128i _val01 = _mm_loadl_epi64((const __m128i*)tmpptr); @@ -604,23 +522,10 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const _sum10 = _mm_maddd_epi16(_val1, _w0, _sum10); _sum11 = _mm_maddd_epi16(_val1, _w1, _sum11); #else - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl01 = _mm_mullo_epi16(_val0, _w1); - __m128i _sh01 = _mm_mulhi_epi16(_val0, _w1); - __m128i _sl10 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh10 = _mm_mulhi_epi16(_val1, _w0); - __m128i _sl11 = _mm_mullo_epi16(_val1, _w1); - __m128i _sh11 = _mm_mulhi_epi16(_val1, _w1); - - _sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl11, _sh11)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); #endif #endif @@ -629,67 +534,26 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const } #if __AVX2__ -#if __AVXVNNI__ || __AVX512VNNI__ _sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02); _sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0)); #else - // transpose 4x8 - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02); - _tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03); - _tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02); - _tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02); - _sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13); - - __m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0); - _sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask); -#endif -#else -#if __XOP__ +#if __SSSE3__ _sum00 = _mm_hadd_epi32(_sum00, _sum01); _sum10 = _mm_hadd_epi32(_sum10, _sum11); #else - // transpose 4x4 - { - __m128i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm_unpacklo_epi32(_sum00, _sum01); - _tmp1 = _mm_unpacklo_epi32(_sum02, _sum03); - _tmp2 = _mm_unpackhi_epi32(_sum00, _sum01); - _tmp3 = _mm_unpackhi_epi32(_sum02, _sum03); - _sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1); - _sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1); - _sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3); - _sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3); - } - { - __m128i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm_unpacklo_epi32(_sum10, _sum11); - _tmp1 = _mm_unpacklo_epi32(_sum12, _sum13); - _tmp2 = _mm_unpackhi_epi32(_sum10, _sum11); - _tmp3 = _mm_unpackhi_epi32(_sum12, _sum13); - _sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1); - _sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1); - _sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3); - _sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3); - } + __m128i _sum00_sh = _mm_shuffle_epi32(_sum00, 216); + __m128i _sum01_sh = _mm_shuffle_epi32(_sum01, 216); + __m128i _sum10_sh = _mm_shuffle_epi32(_sum10, 216); + __m128i _sum11_sh = _mm_shuffle_epi32(_sum11, 216); + + _sum00 = _mm_unpacklo_epi64(_sum00_sh, _sum01_sh); + _sum01 = _mm_unpackhi_epi64(_sum00_sh, _sum01_sh); + _sum10 = _mm_unpacklo_epi64(_sum10_sh, _sum11_sh); + _sum11 = _mm_unpackhi_epi64(_sum10_sh, _sum11_sh); _sum00 = _mm_add_epi32(_sum00, _sum01); - _sum02 = _mm_add_epi32(_sum02, _sum03); _sum10 = _mm_add_epi32(_sum10, _sum11); - _sum12 = _mm_add_epi32(_sum12, _sum13); - - _sum00 = _mm_add_epi32(_sum00, _sum02); - _sum10 = _mm_add_epi32(_sum10, _sum12); #endif #endif } diff --git a/src/layer/x86/convolution_sgemm_pack1to4_int8.h b/src/layer/x86/convolution_sgemm_pack1to4_int8.h index ba567ce33..fd0849872 100644 --- a/src/layer/x86/convolution_sgemm_pack1to4_int8.h +++ b/src/layer/x86/convolution_sgemm_pack1to4_int8.h @@ -301,17 +301,8 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl if (nn4 > 0) { -#if __AVXVNNI__ || __AVX512VNNI__ - __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum30_22 = _mm256_setzero_si256(); -#else __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum01_13 = _mm256_setzero_si256(); - __m256i _sum11_03 = _mm256_setzero_si256(); __m256i _sum30_22 = _mm256_setzero_si256(); - __m256i _sum21_33 = _mm256_setzero_si256(); - __m256i _sum31_23 = _mm256_setzero_si256(); -#endif int j = 0; for (; j < nn4; j++) @@ -334,72 +325,21 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum20_32 = _mm256_dpwssd_epi32(_sum20_32, _val23_16, _w01_16); _sum30_22 = _mm256_dpwssd_epi32(_sum30_22, _val32_16, _w01_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl20_31 = _mm256_mullo_epi16(_val23_16, _w01_16); - __m256i _sh20_31 = _mm256_mulhi_epi16(_val23_16, _w01_16); - __m256i _sl30_21 = _mm256_mullo_epi16(_val32_16, _w01_16); - __m256i _sh30_21 = _mm256_mulhi_epi16(_val32_16, _w01_16); - - _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_unpacklo_epi16(_sl20_31, _sh20_31)); - _sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_unpacklo_epi16(_sl30_21, _sh30_21)); - _sum21_33 = _mm256_add_epi32(_sum21_33, _mm256_unpackhi_epi16(_sl20_31, _sh20_31)); - _sum31_23 = _mm256_add_epi32(_sum31_23, _mm256_unpackhi_epi16(_sl30_21, _sh30_21)); + _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_madd_epi16(_val23_16, _w01_16)); + _sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_madd_epi16(_val32_16, _w01_16)); #endif tmpptr += 16; kptr0 += 16; } -#if __AVXVNNI__ || __AVX512VNNI__ _sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02); _sum20_32 = _mm256_hadd_epi32(_sum20_32, _sum30_22); _sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0)); _sum20_32 = _mm256_permute4x64_epi64(_sum20_32, _MM_SHUFFLE(2, 1, 3, 0)); -#else - // transpose 4x8 - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02); - _tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03); - _tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02); - _tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum20_32, _sum30_22); - _tmp1 = _mm256_unpacklo_epi32(_sum21_33, _sum31_23); - _tmp2 = _mm256_unpackhi_epi32(_sum20_32, _sum30_22); - _tmp3 = _mm256_unpackhi_epi32(_sum21_33, _sum31_23); - _sum20_32 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum30_22 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum21_33 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum31_23 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02); - _sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13); - - _sum20_32 = _mm256_add_epi32(_sum20_32, _sum30_22); - _sum21_33 = _mm256_add_epi32(_sum21_33, _sum31_23); - _sum20_32 = _mm256_add_epi32(_sum20_32, _sum21_33); - - __m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0); - _sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask); - _sum20_32 = _mm256_permutevar8x32_epi32(_sum20_32, _perm_mask); -#endif } __m128i _sum00 = _mm256_extracti128_si256(_sum00_12, 0); @@ -458,25 +398,10 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl if (nn4 > 0) { #if __AVX2__ -#if __AVXVNNI__ || __AVX512VNNI__ - __m256i _sum10_02 = _mm256_setzero_si256(); -#else __m256i _sum10_02 = _mm256_setzero_si256(); - __m256i _sum01_13 = _mm256_setzero_si256(); - __m256i _sum11_03 = _mm256_setzero_si256(); -#endif -#else -#if __XOP__ - __m128i _sum01 = _mm_setzero_si128(); - __m128i _sum11 = _mm_setzero_si128(); #else __m128i _sum01 = _mm_setzero_si128(); - __m128i _sum02 = _mm_setzero_si128(); - __m128i _sum03 = _mm_setzero_si128(); __m128i _sum11 = _mm_setzero_si128(); - __m128i _sum12 = _mm_setzero_si128(); - __m128i _sum13 = _mm_setzero_si128(); -#endif #endif int j = 0; @@ -497,15 +422,8 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum00_12 = _mm256_dpwssd_epi32(_sum00_12, _val01_16, _w01_16); _sum10_02 = _mm256_dpwssd_epi32(_sum10_02, _val10_16, _w01_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - - _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); + _sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16)); #endif #else __m128i _val01 = _mm_loadl_epi64((const __m128i*)tmpptr); @@ -530,23 +448,10 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum10 = _mm_maddd_epi16(_val1, _w0, _sum10); _sum11 = _mm_maddd_epi16(_val1, _w1, _sum11); #else - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl01 = _mm_mullo_epi16(_val0, _w1); - __m128i _sh01 = _mm_mulhi_epi16(_val0, _w1); - __m128i _sl10 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh10 = _mm_mulhi_epi16(_val1, _w0); - __m128i _sl11 = _mm_mullo_epi16(_val1, _w1); - __m128i _sh11 = _mm_mulhi_epi16(_val1, _w1); - - _sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl11, _sh11)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); #endif #endif @@ -555,67 +460,26 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl } #if __AVX2__ -#if __AVXVNNI__ || __AVX512VNNI__ _sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02); _sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0)); #else - // transpose 4x8 - { - __m256i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02); - _tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03); - _tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02); - _tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1); - _sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1); - _sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3); - _sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3); - } - - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02); - _sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03); - _sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13); - - __m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0); - _sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask); -#endif -#else -#if __XOP__ +#if __SSSE3__ _sum00 = _mm_hadd_epi32(_sum00, _sum01); _sum10 = _mm_hadd_epi32(_sum10, _sum11); #else - // transpose 4x4 - { - __m128i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm_unpacklo_epi32(_sum00, _sum01); - _tmp1 = _mm_unpacklo_epi32(_sum02, _sum03); - _tmp2 = _mm_unpackhi_epi32(_sum00, _sum01); - _tmp3 = _mm_unpackhi_epi32(_sum02, _sum03); - _sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1); - _sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1); - _sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3); - _sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3); - } - { - __m128i _tmp0, _tmp1, _tmp2, _tmp3; - _tmp0 = _mm_unpacklo_epi32(_sum10, _sum11); - _tmp1 = _mm_unpacklo_epi32(_sum12, _sum13); - _tmp2 = _mm_unpackhi_epi32(_sum10, _sum11); - _tmp3 = _mm_unpackhi_epi32(_sum12, _sum13); - _sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1); - _sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1); - _sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3); - _sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3); - } + __m128i _sum00_sh = _mm_shuffle_epi32(_sum00, 216); + __m128i _sum01_sh = _mm_shuffle_epi32(_sum01, 216); + __m128i _sum10_sh = _mm_shuffle_epi32(_sum10, 216); + __m128i _sum11_sh = _mm_shuffle_epi32(_sum11, 216); + + _sum00 = _mm_unpacklo_epi64(_sum00_sh, _sum01_sh); + _sum01 = _mm_unpackhi_epi64(_sum00_sh, _sum01_sh); + _sum10 = _mm_unpacklo_epi64(_sum10_sh, _sum11_sh); + _sum11 = _mm_unpackhi_epi64(_sum10_sh, _sum11_sh); _sum00 = _mm_add_epi32(_sum00, _sum01); - _sum02 = _mm_add_epi32(_sum02, _sum03); _sum10 = _mm_add_epi32(_sum10, _sum11); - _sum12 = _mm_add_epi32(_sum12, _sum13); - - _sum00 = _mm_add_epi32(_sum00, _sum02); - _sum10 = _mm_add_epi32(_sum10, _sum12); #endif #endif } diff --git a/src/layer/x86/convolution_sgemm_pack8to1_int8.h b/src/layer/x86/convolution_sgemm_pack8to1_int8.h index 9c080ffd5..b76b6e26f 100644 --- a/src/layer/x86/convolution_sgemm_pack8to1_int8.h +++ b/src/layer/x86/convolution_sgemm_pack8to1_int8.h @@ -225,23 +225,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16); _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16); - __m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16); - __m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16); - __m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16); - - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03)); - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16)); #endif __m128i _val23 = _mm_loadu_si128((const __m128i*)(tmpptr + 16)); @@ -254,23 +241,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23_16, _w23_16); _sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32_16, _w23_16); #else - __m256i _sl04_15 = _mm256_mullo_epi16(_val23_16, _w01_16); - __m256i _sh04_15 = _mm256_mulhi_epi16(_val23_16, _w01_16); - __m256i _sl14_05 = _mm256_mullo_epi16(_val32_16, _w01_16); - __m256i _sh14_05 = _mm256_mulhi_epi16(_val32_16, _w01_16); - __m256i _sl06_17 = _mm256_mullo_epi16(_val23_16, _w23_16); - __m256i _sh06_17 = _mm256_mulhi_epi16(_val23_16, _w23_16); - __m256i _sl16_07 = _mm256_mullo_epi16(_val32_16, _w23_16); - __m256i _sh16_07 = _mm256_mulhi_epi16(_val32_16, _w23_16); - - _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpacklo_epi16(_sl04_15, _sh04_15)); - _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpacklo_epi16(_sl14_05, _sh14_05)); - _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpacklo_epi16(_sl06_17, _sh06_17)); - _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpacklo_epi16(_sl16_07, _sh16_07)); - _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpackhi_epi16(_sl04_15, _sh04_15)); - _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpackhi_epi16(_sl14_05, _sh14_05)); - _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpackhi_epi16(_sl06_17, _sh06_17)); - _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpackhi_epi16(_sl16_07, _sh16_07)); + _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23_16, _w01_16)); + _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32_16, _w01_16)); + _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23_16, _w23_16)); + _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32_16, _w23_16)); #endif tmpptr += 32; @@ -386,23 +360,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16); _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16); - __m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16); - __m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16); - __m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16); - - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03)); - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16)); #endif #else __m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr); @@ -429,39 +390,14 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum12 = _mm_maddd_epi16(_val1, _w2, _sum12); _sum13 = _mm_maddd_epi16(_val1, _w3, _sum13); #else - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl01 = _mm_mullo_epi16(_val0, _w1); - __m128i _sh01 = _mm_mulhi_epi16(_val0, _w1); - __m128i _sl02 = _mm_mullo_epi16(_val0, _w2); - __m128i _sh02 = _mm_mulhi_epi16(_val0, _w2); - __m128i _sl03 = _mm_mullo_epi16(_val0, _w3); - __m128i _sh03 = _mm_mulhi_epi16(_val0, _w3); - __m128i _sl10 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh10 = _mm_mulhi_epi16(_val1, _w0); - __m128i _sl11 = _mm_mullo_epi16(_val1, _w1); - __m128i _sh11 = _mm_mulhi_epi16(_val1, _w1); - __m128i _sl12 = _mm_mullo_epi16(_val1, _w2); - __m128i _sh12 = _mm_mulhi_epi16(_val1, _w2); - __m128i _sl13 = _mm_mullo_epi16(_val1, _w3); - __m128i _sh13 = _mm_mulhi_epi16(_val1, _w3); - - _sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl02, _sh02)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpacklo_epi16(_sl03, _sh03)); - _sum00 = _mm_add_epi32(_sum00, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpackhi_epi16(_sl02, _sh02)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl03, _sh03)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl12, _sh12)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpacklo_epi16(_sl13, _sh13)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl11, _sh11)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpackhi_epi16(_sl12, _sh12)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl13, _sh13)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02); + _sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); + _sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12); + _sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13); #endif #endif @@ -582,15 +518,8 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01_16); _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23_16); #else - __m256i _sl0_1 = _mm256_mullo_epi16(_valval, _w01_16); - __m256i _sh0_1 = _mm256_mulhi_epi16(_valval, _w01_16); - __m256i _sl2_3 = _mm256_mullo_epi16(_valval, _w23_16); - __m256i _sh2_3 = _mm256_mulhi_epi16(_valval, _w23_16); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl0_1, _sh0_1)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl2_3, _sh2_3)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl0_1, _sh0_1)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl2_3, _sh2_3)); + _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01_16)); + _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23_16)); #endif #else __m128i _val = _mm_loadl_epi64((const __m128i*)tmpptr); @@ -615,23 +544,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum2 = _mm_maddd_epi16(_val, _w2, _sum2); _sum3 = _mm_maddd_epi16(_val, _w3, _sum3); #else - __m128i _sl0 = _mm_mullo_epi16(_val, _w0); - __m128i _sh0 = _mm_mulhi_epi16(_val, _w0); - __m128i _sl1 = _mm_mullo_epi16(_val, _w1); - __m128i _sh1 = _mm_mulhi_epi16(_val, _w1); - __m128i _sl2 = _mm_mullo_epi16(_val, _w2); - __m128i _sh2 = _mm_mulhi_epi16(_val, _w2); - __m128i _sl3 = _mm_mullo_epi16(_val, _w3); - __m128i _sh3 = _mm_mulhi_epi16(_val, _w3); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl2, _sh2)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpacklo_epi16(_sl3, _sh3)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpackhi_epi16(_sl2, _sh2)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl3, _sh3)); + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1); + _sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2); + _sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3); #endif #endif @@ -694,10 +610,8 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl int nn = inch * maxk; // inch always > 0 - __m256i _sum0_2 = _mm256_setzero_si256(); - __m256i _sum1_3 = _mm256_setzero_si256(); - __m256i _sum4_6 = _mm256_setzero_si256(); - __m256i _sum5_7 = _mm256_setzero_si256(); + __m256i _sum01 = _mm256_setzero_si256(); + __m256i _sum23 = _mm256_setzero_si256(); int j = 0; for (; j < nn; j++) @@ -711,31 +625,27 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl __m256i _w01_16 = _mm256_cvtepi8_epi16(_w01); _w01_16 = _mm256_permute4x64_epi64(_w01_16, _MM_SHUFFLE(1, 0, 1, 0)); - __m256i _sl00_10 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_10 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl20_30 = _mm256_mullo_epi16(_val23_16, _w01_16); - __m256i _sh20_30 = _mm256_mulhi_epi16(_val23_16, _w01_16); - - _sum0_2 = _mm256_add_epi32(_sum0_2, _mm256_unpacklo_epi16(_sl00_10, _sh00_10)); - _sum1_3 = _mm256_add_epi32(_sum1_3, _mm256_unpackhi_epi16(_sl00_10, _sh00_10)); - _sum4_6 = _mm256_add_epi32(_sum4_6, _mm256_unpacklo_epi16(_sl20_30, _sh20_30)); - _sum5_7 = _mm256_add_epi32(_sum5_7, _mm256_unpackhi_epi16(_sl20_30, _sh20_30)); +#if __AVXVNNI__ || __AVX512VNNI__ + _sum01 = _mm256_dpwssd_epi32(_sum01, _val01_16, _w01_16); + _sum23 = _mm256_dpwssd_epi32(_sum23, _val23_16, _w01_16); +#else + _sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum23 = _mm256_add_epi32(_sum23, _mm256_madd_epi16(_val23_16, _w01_16)); +#endif tmpptr += 32; kptr0 += 8; } - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); - _sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7); - __m128i _sum0 = _mm256_extracti128_si256(_sum0_2, 0); - __m128i _sum2 = _mm256_extracti128_si256(_sum0_2, 1); - __m128i _sum4 = _mm256_extracti128_si256(_sum4_6, 0); - __m128i _sum6 = _mm256_extracti128_si256(_sum4_6, 1); + __m128i _sum0 = _mm256_extracti128_si256(_sum01, 0); + __m128i _sum1 = _mm256_extracti128_si256(_sum01, 1); + __m128i _sum2 = _mm256_extracti128_si256(_sum23, 0); + __m128i _sum3 = _mm256_extracti128_si256(_sum23, 1); outptr0[0] = _mm_reduce_add_epi32(_sum0); - outptr0[1] = _mm_reduce_add_epi32(_sum2); - outptr0[2] = _mm_reduce_add_epi32(_sum4); - outptr0[3] = _mm_reduce_add_epi32(_sum6); + outptr0[1] = _mm_reduce_add_epi32(_sum1); + outptr0[2] = _mm_reduce_add_epi32(_sum2); + outptr0[3] = _mm_reduce_add_epi32(_sum3); outptr0 += 4; } #endif @@ -751,13 +661,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl int nn = inch * maxk; // inch always > 0 #if __AVX2__ - __m256i _sum0_2 = _mm256_setzero_si256(); - __m256i _sum1_3 = _mm256_setzero_si256(); + __m256i _sum01 = _mm256_setzero_si256(); #else __m128i _sum0 = _mm_setzero_si128(); __m128i _sum1 = _mm_setzero_si128(); - __m128i _sum2 = _mm_setzero_si128(); - __m128i _sum3 = _mm_setzero_si128(); #endif int j = 0; @@ -771,11 +678,11 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl __m256i _w01_16 = _mm256_cvtepi8_epi16(_w01); _w01_16 = _mm256_permute4x64_epi64(_w01_16, _MM_SHUFFLE(1, 0, 1, 0)); - __m256i _sl00_10 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_10 = _mm256_mulhi_epi16(_val01_16, _w01_16); - - _sum0_2 = _mm256_add_epi32(_sum0_2, _mm256_unpacklo_epi16(_sl00_10, _sh00_10)); - _sum1_3 = _mm256_add_epi32(_sum1_3, _mm256_unpackhi_epi16(_sl00_10, _sh00_10)); +#if __AVXVNNI__ || __AVX512VNNI__ + _sum01 = _mm256_dpwssd_epi32(_sum01, _val01_16, _w01_16); +#else + _sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01_16, _w01_16)); +#endif #else __m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr); __m128i _extval01 = _mm_cmpgt_epi8(_mm_setzero_si128(), _val01); @@ -790,15 +697,13 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl __m128i _w0 = _mm_unpacklo_epi8(_w01, _extw01); #endif - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl10 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh10 = _mm_mulhi_epi16(_val1, _w0); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10)); +#if __XOP__ + _sum0 = _mm_maddd_epi16(_val0, _w0, _sum0); + _sum1 = _mm_maddd_epi16(_val1, _w0, _sum1); +#else + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum1); +#endif #endif tmpptr += 16; @@ -806,16 +711,12 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl } #if __AVX2__ - _sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3); - __m128i _sum0 = _mm256_extracti128_si256(_sum0_2, 0); - __m128i _sum2 = _mm256_extracti128_si256(_sum0_2, 1); -#else - _sum0 = _mm_add_epi32(_sum0, _sum1); - _sum2 = _mm_add_epi32(_sum2, _sum3); + __m128i _sum0 = _mm256_extracti128_si256(_sum01, 0); + __m128i _sum1 = _mm256_extracti128_si256(_sum01, 1); #endif outptr0[0] = _mm_reduce_add_epi32(_sum0); - outptr0[1] = _mm_reduce_add_epi32(_sum2); + outptr0[1] = _mm_reduce_add_epi32(_sum1); outptr0 += 2; } for (; i < size; i++) @@ -830,7 +731,6 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl int nn = inch * maxk; // inch always > 0 __m128i _sum0 = _mm_setzero_si128(); - __m128i _sum1 = _mm_setzero_si128(); int j = 0; for (; j < nn; j++) @@ -851,18 +751,16 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl __m128i _w0 = _mm_unpacklo_epi8(_w01, _extw01); #endif - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00)); +#if __XOP__ + _sum0 = _mm_maddd_epi16(_val0, _w0, _sum0); +#else + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0); +#endif tmpptr += 8; kptr0 += 8; } - _sum0 = _mm_add_epi32(_sum0, _sum1); - outptr0[0] = _mm_reduce_add_epi32(_sum0); outptr0 += 1; } diff --git a/src/layer/x86/convolution_sgemm_pack8to4_int8.h b/src/layer/x86/convolution_sgemm_pack8to4_int8.h index 5e650c654..8fdaece96 100644 --- a/src/layer/x86/convolution_sgemm_pack8to4_int8.h +++ b/src/layer/x86/convolution_sgemm_pack8to4_int8.h @@ -215,23 +215,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16); _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16); - __m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16); - __m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16); - __m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16); - - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03)); - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16)); #endif __m128i _val23 = _mm_loadu_si128((const __m128i*)(tmpptr + 16)); @@ -244,23 +231,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23_16, _w23_16); _sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32_16, _w23_16); #else - __m256i _sl04_15 = _mm256_mullo_epi16(_val23_16, _w01_16); - __m256i _sh04_15 = _mm256_mulhi_epi16(_val23_16, _w01_16); - __m256i _sl14_05 = _mm256_mullo_epi16(_val32_16, _w01_16); - __m256i _sh14_05 = _mm256_mulhi_epi16(_val32_16, _w01_16); - __m256i _sl06_17 = _mm256_mullo_epi16(_val23_16, _w23_16); - __m256i _sh06_17 = _mm256_mulhi_epi16(_val23_16, _w23_16); - __m256i _sl16_07 = _mm256_mullo_epi16(_val32_16, _w23_16); - __m256i _sh16_07 = _mm256_mulhi_epi16(_val32_16, _w23_16); - - _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpacklo_epi16(_sl04_15, _sh04_15)); - _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpacklo_epi16(_sl14_05, _sh14_05)); - _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpacklo_epi16(_sl06_17, _sh06_17)); - _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpacklo_epi16(_sl16_07, _sh16_07)); - _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpackhi_epi16(_sl04_15, _sh04_15)); - _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpackhi_epi16(_sl14_05, _sh14_05)); - _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpackhi_epi16(_sl06_17, _sh06_17)); - _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpackhi_epi16(_sl16_07, _sh16_07)); + _sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23_16, _w01_16)); + _sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32_16, _w01_16)); + _sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23_16, _w23_16)); + _sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32_16, _w23_16)); #endif tmpptr += 32; @@ -355,23 +329,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16); _sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16); #else - __m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16); - __m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16); - __m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16); - __m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16); - __m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16); - __m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16); - __m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16); - __m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16); - - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03)); - _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11)); - _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01)); - _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13)); - _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03)); + _sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16)); + _sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16)); + _sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16)); + _sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16)); #endif #else __m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr); @@ -398,39 +359,14 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum12 = _mm_maddd_epi16(_val1, _w2, _sum12); _sum13 = _mm_maddd_epi16(_val1, _w3, _sum13); #else - __m128i _sl00 = _mm_mullo_epi16(_val0, _w0); - __m128i _sh00 = _mm_mulhi_epi16(_val0, _w0); - __m128i _sl01 = _mm_mullo_epi16(_val0, _w1); - __m128i _sh01 = _mm_mulhi_epi16(_val0, _w1); - __m128i _sl02 = _mm_mullo_epi16(_val0, _w2); - __m128i _sh02 = _mm_mulhi_epi16(_val0, _w2); - __m128i _sl03 = _mm_mullo_epi16(_val0, _w3); - __m128i _sh03 = _mm_mulhi_epi16(_val0, _w3); - __m128i _sl10 = _mm_mullo_epi16(_val1, _w0); - __m128i _sh10 = _mm_mulhi_epi16(_val1, _w0); - __m128i _sl11 = _mm_mullo_epi16(_val1, _w1); - __m128i _sh11 = _mm_mulhi_epi16(_val1, _w1); - __m128i _sl12 = _mm_mullo_epi16(_val1, _w2); - __m128i _sh12 = _mm_mulhi_epi16(_val1, _w2); - __m128i _sl13 = _mm_mullo_epi16(_val1, _w3); - __m128i _sh13 = _mm_mulhi_epi16(_val1, _w3); - - _sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpacklo_epi16(_sl01, _sh01)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl02, _sh02)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpacklo_epi16(_sl03, _sh03)); - _sum00 = _mm_add_epi32(_sum00, _mm_unpackhi_epi16(_sl00, _sh00)); - _sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl01, _sh01)); - _sum02 = _mm_add_epi32(_sum02, _mm_unpackhi_epi16(_sl02, _sh02)); - _sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl03, _sh03)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpacklo_epi16(_sl11, _sh11)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl12, _sh12)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpacklo_epi16(_sl13, _sh13)); - _sum10 = _mm_add_epi32(_sum10, _mm_unpackhi_epi16(_sl10, _sh10)); - _sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl11, _sh11)); - _sum12 = _mm_add_epi32(_sum12, _mm_unpackhi_epi16(_sl12, _sh12)); - _sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl13, _sh13)); + _sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00); + _sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01); + _sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02); + _sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03); + _sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10); + _sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11); + _sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12); + _sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13); #endif #endif @@ -537,15 +473,8 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01_16); _sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23_16); #else - __m256i _sl0_1 = _mm256_mullo_epi16(_valval, _w01_16); - __m256i _sh0_1 = _mm256_mulhi_epi16(_valval, _w01_16); - __m256i _sl2_3 = _mm256_mullo_epi16(_valval, _w23_16); - __m256i _sh2_3 = _mm256_mulhi_epi16(_valval, _w23_16); - - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl0_1, _sh0_1)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl2_3, _sh2_3)); - _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl0_1, _sh0_1)); - _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl2_3, _sh2_3)); + _sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01_16)); + _sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23_16)); #endif #else __m128i _val = _mm_loadl_epi64((const __m128i*)tmpptr); @@ -570,23 +499,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl _sum2 = _mm_maddd_epi16(_val, _w2, _sum2); _sum3 = _mm_maddd_epi16(_val, _w3, _sum3); #else - __m128i _sl0 = _mm_mullo_epi16(_val, _w0); - __m128i _sh0 = _mm_mulhi_epi16(_val, _w0); - __m128i _sl1 = _mm_mullo_epi16(_val, _w1); - __m128i _sh1 = _mm_mulhi_epi16(_val, _w1); - __m128i _sl2 = _mm_mullo_epi16(_val, _w2); - __m128i _sh2 = _mm_mulhi_epi16(_val, _w2); - __m128i _sl3 = _mm_mullo_epi16(_val, _w3); - __m128i _sh3 = _mm_mulhi_epi16(_val, _w3); - - _sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpacklo_epi16(_sl1, _sh1)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl2, _sh2)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpacklo_epi16(_sl3, _sh3)); - _sum0 = _mm_add_epi32(_sum0, _mm_unpackhi_epi16(_sl0, _sh0)); - _sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1)); - _sum2 = _mm_add_epi32(_sum2, _mm_unpackhi_epi16(_sl2, _sh2)); - _sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl3, _sh3)); + _sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0); + _sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1); + _sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2); + _sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3); #endif #endif diff --git a/src/layer/x86/convolution_x86.cpp b/src/layer/x86/convolution_x86.cpp index 60522b04e..9acceb288 100644 --- a/src/layer/x86/convolution_x86.cpp +++ b/src/layer/x86/convolution_x86.cpp @@ -16,12 +16,15 @@ #if __SSE2__ #include +#if __SSSE3__ +#include #if __SSE4_1__ #include #if __AVX__ #include #endif #endif // __SSE4_1__ +#endif // __SSSE3__ #endif // __SSE2__ #include "x86_activation.h" #include "x86_usability.h"