nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

x86 sse2/avx2 optimization for convolution sgemm/winograd int8 family (#4286)

tags/20221128
nihui GitHub 3 years ago
parent
c33cbc924a
commit
8eab5ea0ea
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 590 additions and 1259 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +255
    -362
      src/layer/x86/convolution_3x3_pack8to1_int8.h
  2. +203
    -310
      src/layer/x86/convolution_3x3_pack8to4_int8.h
  3. +20
    -156
      src/layer/x86/convolution_sgemm_int8.h
  4. +20
    -156
      src/layer/x86/convolution_sgemm_pack1to4_int8.h
  5. +63
    -165
      src/layer/x86/convolution_sgemm_pack8to1_int8.h
  6. +26
    -110
      src/layer/x86/convolution_sgemm_pack8to4_int8.h
  7. +3
    -0
      src/layer/x86/convolution_x86.cpp

+ 255
- 362
src/layer/x86/convolution_3x3_pack8to1_int8.h View File

@@ -125,11 +125,6 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to1_int8_sse(const Mat& k
int p = 0;
for (; p + 3 < outch; p += 4)
{
const Mat k0 = kernel_tm.channel(p);
const Mat k1 = kernel_tm.channel(p + 1);
const Mat k2 = kernel_tm.channel(p + 2);
const Mat k3 = kernel_tm.channel(p + 3);

Mat g0 = kernel_tm_pack8to1.channel(p / 4);

for (int k = 0; k < 36; k++)
@@ -138,41 +133,15 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to1_int8_sse(const Mat& k

for (int q = 0; q + 7 < inch; q += 8)
{
#if __AVXVNNI__ || __AVX512VNNI__ || __XOP__
for (int i = 0; i < 4; i++)
{
const short* k00 = k0.row<const short>(q + i * 2);
const short* k10 = k1.row<const short>(q + i * 2);
const short* k20 = k2.row<const short>(q + i * 2);
const short* k30 = k3.row<const short>(q + i * 2);

const short* k01 = k0.row<const short>(q + i * 2 + 1);
const short* k11 = k1.row<const short>(q + i * 2 + 1);
const short* k21 = k2.row<const short>(q + i * 2 + 1);
const short* k31 = k3.row<const short>(q + i * 2 + 1);

g00[0] = k00[k];
g00[1] = k01[k];
g00[2] = k10[k];
g00[3] = k11[k];
g00[4] = k20[k];
g00[5] = k21[k];
g00[6] = k30[k];
g00[7] = k31[k];

g00 += 8;
}
#else
for (int i = 0; i < 8; i++)
{
g00[0] = k0.row<const short>(q + i)[k];
g00[1] = k1.row<const short>(q + i)[k];
g00[2] = k2.row<const short>(q + i)[k];
g00[3] = k3.row<const short>(q + i)[k];

g00 += 4;
for (int j = 0; j < 8; j++)
{
const short* k00 = kernel_tm.channel(p + i).row<const short>(q + j);
g00[0] = k00[k];
g00 += 1;
}
}
#endif
}
}
}
@@ -508,114 +477,97 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&

int nn = inch; // inch always > 0

__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
__m256i _sum4_5 = _mm256_setzero_si256();
__m256i _sum6_7 = _mm256_setzero_si256();
__m256i _sum00_11 = _mm256_setzero_si256();
__m256i _sum10_01 = _mm256_setzero_si256();
__m256i _sum02_13 = _mm256_setzero_si256();
__m256i _sum12_03 = _mm256_setzero_si256();

__m256i _sum04_15 = _mm256_setzero_si256();
__m256i _sum14_05 = _mm256_setzero_si256();
__m256i _sum06_17 = _mm256_setzero_si256();
__m256i _sum16_07 = _mm256_setzero_si256();

for (int j = 0; j < nn; j++)
{
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
__m256i _val0 = _mm256_loadu_si256((const __m256i*)r0);
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);

__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _val10 = _mm256_permute4x64_epi64(_val01, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val0_0123);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val0_89ab);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val0_4567);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val0_cdef);
_sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01);
_sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01);
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23);
#else
// 0 0 1 1 2 2 3 3 8 8 9 9 a a b b
// 4 4 5 5 6 6 7 7 c c d d e e f f
__m256i _val0_0123_89ab = _mm256_unpacklo_epi16(_val0, _val0);
__m256i _val0_4567_cdef = _mm256_unpackhi_epi16(_val0, _val0);

__m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val0_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val0_0123);
__m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val0_89ab);
__m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val0_89ab);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val0_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val0_4567);
__m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val0_cdef);
__m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val0_cdef);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23));
#endif

__m256i _val1 = _mm256_loadu_si256((const __m256i*)(r0 + 16));
__m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16));

__m256i _val32 = _mm256_permute4x64_epi64(_val23, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w01, _val1_0123);
_sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w01, _val1_89ab);
_sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w23, _val1_4567);
_sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w23, _val1_cdef);
_sum04_15 = _mm256_dpwssd_epi32(_sum04_15, _val23, _w01);
_sum14_05 = _mm256_dpwssd_epi32(_sum14_05, _val32, _w01);
_sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23, _w23);
_sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32, _w23);
#else
__m256i _val1_0123_89ab = _mm256_unpacklo_epi16(_val1, _val1);
__m256i _val1_4567_cdef = _mm256_unpackhi_epi16(_val1, _val1);

__m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl04_05 = _mm256_mullo_epi16(_w01, _val1_0123);
__m256i _sh04_05 = _mm256_mulhi_epi16(_w01, _val1_0123);
__m256i _sl14_15 = _mm256_mullo_epi16(_w01, _val1_89ab);
__m256i _sh14_15 = _mm256_mulhi_epi16(_w01, _val1_89ab);
__m256i _sl06_07 = _mm256_mullo_epi16(_w23, _val1_4567);
__m256i _sh06_07 = _mm256_mulhi_epi16(_w23, _val1_4567);
__m256i _sl16_17 = _mm256_mullo_epi16(_w23, _val1_cdef);
__m256i _sh16_17 = _mm256_mulhi_epi16(_w23, _val1_cdef);

_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl04_05, _sh04_05));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl14_15, _sh14_15));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl06_07, _sh06_07));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl16_17, _sh16_17));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl04_05, _sh04_05));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl14_15, _sh14_15));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl06_07, _sh06_07));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl16_17, _sh16_17));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23, _w01));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32, _w01));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23, _w23));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32, _w23));
#endif

r0 += 32;
k0 += 32;
}

__m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1));
_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01);
_tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01);
_tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum04_15, _sum14_05);
_tmp1 = _mm256_unpacklo_epi32(_sum06_17, _sum16_07);
_tmp2 = _mm256_unpackhi_epi32(_sum04_15, _sum14_05);
_tmp3 = _mm256_unpackhi_epi32(_sum06_17, _sum16_07);
_sum04_15 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum14_05 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum06_17 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum16_07 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01);
_sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13);

__m256i _sum4_6 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum5_7 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 3, 0, 1));
_sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7);
_sum04_15 = _mm256_add_epi32(_sum04_15, _sum14_05);
_sum06_17 = _mm256_add_epi32(_sum06_17, _sum16_07);
_sum04_15 = _mm256_add_epi32(_sum04_15, _sum06_17);

__m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0);
_sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask);
_sum04_15 = _mm256_permutevar8x32_epi32(_sum04_15, _perm_mask);

int sum[16];
_mm256_storeu_si256((__m256i*)sum, _sum0_2);
_mm256_storeu_si256((__m256i*)(sum + 8), _sum4_6);
_mm256_storeu_si256((__m256i*)sum, _sum00_11);
_mm256_storeu_si256((__m256i*)(sum + 8), _sum04_15);

output0_tm[0] = sum[0];
output1_tm[0] = sum[1];
@@ -651,60 +603,42 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
int nn = inch; // inch always > 0

#if __AVX2__
__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
__m256i _sum00_11 = _mm256_setzero_si256();
__m256i _sum10_01 = _mm256_setzero_si256();
__m256i _sum02_13 = _mm256_setzero_si256();
__m256i _sum12_03 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
__m128i _sum00 = _mm_setzero_si128();
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum02 = _mm_setzero_si128();
__m128i _sum03 = _mm_setzero_si128();
__m128i _sum10 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
__m128i _sum12 = _mm_setzero_si128();
__m128i _sum13 = _mm_setzero_si128();
#endif

for (int j = 0; j < nn; j++)
{
#if __AVX2__
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
__m256i _val = _mm256_loadu_si256((const __m256i*)r0);
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);

__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _val10 = _mm256_permute4x64_epi64(_val01, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val_0123 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val_4567 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val_89ab = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val_cdef = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val_89ab);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val_cdef);
_sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01);
_sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01);
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23);
#else
__m256i _val_0123_89ab = _mm256_unpacklo_epi16(_val, _val);
__m256i _val_4567_cdef = _mm256_unpackhi_epi16(_val, _val);

__m256i _val_0123 = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val_4567 = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val_89ab = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val_cdef = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123);
__m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val_89ab);
__m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val_89ab);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567);
__m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val_cdef);
__m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val_cdef);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23));
#endif
#else
// 0 1 2 3 4 5 6 7
@@ -717,75 +651,23 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
__m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24));

#if __XOP__
__m128i _val0_01 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val0_23 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val0_45 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val0_67 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(3, 3, 3, 3));
__m128i _val1_01 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val1_23 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val1_45 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val1_67 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(3, 3, 3, 3));

_sum0 = _mm_maddd_epi16(_val0_01, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val0_23, _w1, _sum1);
_sum2 = _mm_maddd_epi16(_val1_01, _w0, _sum2);
_sum3 = _mm_maddd_epi16(_val1_23, _w1, _sum3);
_sum0 = _mm_maddd_epi16(_val0_45, _w2, _sum0);
_sum1 = _mm_maddd_epi16(_val0_67, _w3, _sum1);
_sum2 = _mm_maddd_epi16(_val1_45, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val1_67, _w3, _sum3);
_sum00 = _mm_maddd_epi16(_val0, _w0, _sum00);
_sum01 = _mm_maddd_epi16(_val0, _w1, _sum01);
_sum02 = _mm_maddd_epi16(_val0, _w2, _sum02);
_sum03 = _mm_maddd_epi16(_val0, _w3, _sum03);
_sum10 = _mm_maddd_epi16(_val1, _w0, _sum10);
_sum11 = _mm_maddd_epi16(_val1, _w1, _sum11);
_sum12 = _mm_maddd_epi16(_val1, _w2, _sum12);
_sum13 = _mm_maddd_epi16(_val1, _w3, _sum13);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m128i _val0_0123 = _mm_unpacklo_epi16(_val0, _val0);
__m128i _val0_4567 = _mm_unpackhi_epi16(_val0, _val0);

__m128i _val1_0123 = _mm_unpacklo_epi16(_val1, _val1);
__m128i _val1_4567 = _mm_unpackhi_epi16(_val1, _val1);

__m128i _val0_01 = _mm_unpacklo_epi32(_val0_0123, _val0_0123);
__m128i _val0_23 = _mm_unpackhi_epi32(_val0_0123, _val0_0123);
__m128i _val0_45 = _mm_unpacklo_epi32(_val0_4567, _val0_4567);
__m128i _val0_67 = _mm_unpackhi_epi32(_val0_4567, _val0_4567);

__m128i _val1_01 = _mm_unpacklo_epi32(_val1_0123, _val1_0123);
__m128i _val1_23 = _mm_unpackhi_epi32(_val1_0123, _val1_0123);
__m128i _val1_45 = _mm_unpacklo_epi32(_val1_4567, _val1_4567);
__m128i _val1_67 = _mm_unpackhi_epi32(_val1_4567, _val1_4567);

__m128i _sl00 = _mm_mullo_epi16(_w0, _val0_01);
__m128i _sh00 = _mm_mulhi_epi16(_w0, _val0_01);
__m128i _sl10 = _mm_mullo_epi16(_w0, _val1_01);
__m128i _sh10 = _mm_mulhi_epi16(_w0, _val1_01);
__m128i _sl01 = _mm_mullo_epi16(_w1, _val0_23);
__m128i _sh01 = _mm_mulhi_epi16(_w1, _val0_23);
__m128i _sl11 = _mm_mullo_epi16(_w1, _val1_23);
__m128i _sh11 = _mm_mulhi_epi16(_w1, _val1_23);
__m128i _sl02 = _mm_mullo_epi16(_w2, _val0_45);
__m128i _sh02 = _mm_mulhi_epi16(_w2, _val0_45);
__m128i _sl12 = _mm_mullo_epi16(_w2, _val1_45);
__m128i _sh12 = _mm_mulhi_epi16(_w2, _val1_45);
__m128i _sl03 = _mm_mullo_epi16(_w3, _val0_67);
__m128i _sh03 = _mm_mulhi_epi16(_w3, _val0_67);
__m128i _sl13 = _mm_mullo_epi16(_w3, _val1_67);
__m128i _sh13 = _mm_mulhi_epi16(_w3, _val1_67);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl01, _sh01));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl01, _sh01));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl11, _sh11));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl11, _sh11));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl02, _sh02));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl02, _sh02));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl12, _sh12));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl12, _sh12));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl03, _sh03));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl03, _sh03));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl13, _sh13));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl13, _sh13));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02);
_sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
_sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12);
_sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13);
#endif
#endif

@@ -794,19 +676,64 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
}

#if __AVX2__
__m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1));
_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01);
_tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01);
_tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01);
_sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13);

__m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0);
_sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask);

int sum[8];
_mm256_storeu_si256((__m256i*)sum, _sum0_2);
_mm256_storeu_si256((__m256i*)sum, _sum00_11);
#else
_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);
// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum00, _sum01);
_tmp1 = _mm_unpacklo_epi32(_sum02, _sum03);
_tmp2 = _mm_unpackhi_epi32(_sum00, _sum01);
_tmp3 = _mm_unpackhi_epi32(_sum02, _sum03);
_sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum10, _sum11);
_tmp1 = _mm_unpacklo_epi32(_sum12, _sum13);
_tmp2 = _mm_unpackhi_epi32(_sum10, _sum11);
_tmp3 = _mm_unpackhi_epi32(_sum12, _sum13);
_sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00 = _mm_add_epi32(_sum00, _sum01);
_sum02 = _mm_add_epi32(_sum02, _sum03);
_sum10 = _mm_add_epi32(_sum10, _sum11);
_sum12 = _mm_add_epi32(_sum12, _sum13);

_sum00 = _mm_add_epi32(_sum00, _sum02);
_sum10 = _mm_add_epi32(_sum10, _sum12);

int sum[8];
_mm_storeu_si128((__m128i*)sum, _sum0);
_mm_storeu_si128((__m128i*)(sum + 4), _sum2);
_mm_storeu_si128((__m128i*)sum, _sum00);
_mm_storeu_si128((__m128i*)(sum + 4), _sum10);
#endif

output0_tm[0] = sum[0];
@@ -835,9 +762,12 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&

#if __AVX2__
__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
#endif

for (int j = 0; j < nn; j++)
@@ -849,37 +779,14 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _valval = _mm256_inserti128_si256(_mm256_castsi128_si256(_val), _val, 1);

#if __AVXVNNI__ || __AVX512VNNI__
// 0 1 0 1 x x x x
// 0 1 0 1 0 1 0 1
__m128i _val_01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val_23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val_45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val_67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3));

__m256i _val_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_01), _val_23, 1);
__m256i _val_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_45), _val_67, 1);

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m256i _val_0123 = _mm256_castsi128_si256(_mm_unpacklo_epi16(_val, _val));
__m256i _val_4567 = _mm256_castsi128_si256(_mm_unpackhi_epi16(_val, _val));

_val_0123 = _mm256_permutevar8x32_epi32(_val_0123, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
_val_4567 = _mm256_permutevar8x32_epi32(_val_4567, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23));
#endif
#else
__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);
@@ -888,43 +795,15 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
__m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24));

#if __XOP__
__m128i _val01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3));

_sum0 = _mm_maddd_epi16(_val01, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val23, _w1, _sum1);
_sum0 = _mm_maddd_epi16(_val45, _w2, _sum0);
_sum1 = _mm_maddd_epi16(_val67, _w3, _sum1);
_sum0 = _mm_maddd_epi16(_val, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val, _w1, _sum1);
_sum2 = _mm_maddd_epi16(_val, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val, _w3, _sum3);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m128i _val_0123 = _mm_unpacklo_epi16(_val, _val);
__m128i _val_4567 = _mm_unpackhi_epi16(_val, _val);

__m128i _val01 = _mm_unpacklo_epi32(_val_0123, _val_0123);
__m128i _val23 = _mm_unpackhi_epi32(_val_0123, _val_0123);
__m128i _val45 = _mm_unpacklo_epi32(_val_4567, _val_4567);
__m128i _val67 = _mm_unpackhi_epi32(_val_4567, _val_4567);

__m128i _sl0 = _mm_mullo_epi16(_w0, _val01);
__m128i _sh0 = _mm_mulhi_epi16(_w0, _val01);
__m128i _sl1 = _mm_mullo_epi16(_w1, _val23);
__m128i _sh1 = _mm_mulhi_epi16(_w1, _val23);
__m128i _sl2 = _mm_mullo_epi16(_w2, _val45);
__m128i _sh2 = _mm_mulhi_epi16(_w2, _val45);
__m128i _sl3 = _mm_mullo_epi16(_w3, _val67);
__m128i _sh3 = _mm_mulhi_epi16(_w3, _val67);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl1, _sh1));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl2, _sh2));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl2, _sh2));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl3, _sh3));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1);
_sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2);
_sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3);
#endif
#endif

@@ -935,8 +814,27 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
#if __AVX2__
__m128i _sum0 = _mm256_extracti128_si256(_sum0_1, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum0_1, 1);
__m128i _sum2 = _mm256_extracti128_si256(_sum2_3, 0);
__m128i _sum3 = _mm256_extracti128_si256(_sum2_3, 1);
#endif

// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum0, _sum1);
_tmp1 = _mm_unpacklo_epi32(_sum2, _sum3);
_tmp2 = _mm_unpackhi_epi32(_sum0, _sum1);
_tmp3 = _mm_unpackhi_epi32(_sum2, _sum3);
_sum0 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum1 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum2 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum3 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}

_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);

_sum0 = _mm_add_epi32(_sum0, _sum2);

int sum[4];
_mm_storeu_si128((__m128i*)sum, _sum0);
@@ -973,55 +871,38 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
const short* r0 = bb2.row<const short>(i / 4);
const short* k0 = kernel0_tm.row<const short>(r);

__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
__m128i _sum4 = _mm_setzero_si128();
__m128i _sum5 = _mm_setzero_si128();
__m128i _sum6 = _mm_setzero_si128();
__m128i _sum7 = _mm_setzero_si128();
__m256i _sum01 = _mm256_setzero_si256();
__m256i _sum23 = _mm256_setzero_si256();

for (int q = 0; q < inch; q++)
{
__m128i _val0 = _mm_loadu_si128((const __m128i*)r0);
__m128i _val1 = _mm_loadu_si128((const __m128i*)(r0 + 8));
__m128i _val2 = _mm_loadu_si128((const __m128i*)(r0 + 16));
__m128i _val3 = _mm_loadu_si128((const __m128i*)(r0 + 24));
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);
__m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16));

__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);
__m256i _w01 = _mm256_inserti128_si256(_mm256_castsi128_si256(_w0), _w0, 1);

__m128i _sl0 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh0 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl1 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh1 = _mm_mulhi_epi16(_val1, _w0);
__m128i _sl2 = _mm_mullo_epi16(_val2, _w0);
__m128i _sh2 = _mm_mulhi_epi16(_val2, _w0);
__m128i _sl3 = _mm_mullo_epi16(_val3, _w0);
__m128i _sh3 = _mm_mulhi_epi16(_val3, _w0);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl1, _sh1));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl1, _sh1));
_sum4 = _mm_add_epi32(_sum4, _mm_unpacklo_epi16(_sl2, _sh2));
_sum5 = _mm_add_epi32(_sum5, _mm_unpackhi_epi16(_sl2, _sh2));
_sum6 = _mm_add_epi32(_sum6, _mm_unpacklo_epi16(_sl3, _sh3));
_sum7 = _mm_add_epi32(_sum7, _mm_unpackhi_epi16(_sl3, _sh3));
#if __AVXVNNI__ || __AVX512VNNI__
_sum01 = _mm256_dpwssd_epi32(_sum01, _val01, _w01);
_sum23 = _mm256_dpwssd_epi32(_sum23, _val23, _w01);
#else
_sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01, _w01));
_sum23 = _mm256_add_epi32(_sum23, _mm256_madd_epi16(_val23, _w01));
#endif

k0 += 8;
r0 += 32;
}

_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);
_sum4 = _mm_add_epi32(_sum4, _sum5);
_sum6 = _mm_add_epi32(_sum6, _sum7);
__m128i _sum0 = _mm256_extracti128_si256(_sum01, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum01, 1);
__m128i _sum2 = _mm256_extracti128_si256(_sum23, 0);
__m128i _sum3 = _mm256_extracti128_si256(_sum23, 1);

output0_tm[0] = _mm_reduce_add_epi32(_sum0);
output0_tm[1] = _mm_reduce_add_epi32(_sum2);
output0_tm[2] = _mm_reduce_add_epi32(_sum4);
output0_tm[3] = _mm_reduce_add_epi32(_sum6);
output0_tm[1] = _mm_reduce_add_epi32(_sum1);
output0_tm[2] = _mm_reduce_add_epi32(_sum2);
output0_tm[3] = _mm_reduce_add_epi32(_sum3);
output0_tm += 4;
}
#endif
@@ -1034,37 +915,52 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
#endif
const short* k0 = kernel0_tm.row<const short>(r);

#if __AVX2__
__m256i _sum01 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
#endif

for (int q = 0; q < inch; q++)
{
#if __AVX2__
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);

__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);
__m256i _w01 = _mm256_inserti128_si256(_mm256_castsi128_si256(_w0), _w0, 1);

#if __AVXVNNI__ || __AVX512VNNI__
_sum01 = _mm256_dpwssd_epi32(_sum01, _val01, _w01);
#else
_sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01, _w01));
#endif
#else
__m128i _val0 = _mm_loadu_si128((const __m128i*)r0);
__m128i _val1 = _mm_loadu_si128((const __m128i*)(r0 + 8));

__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);

__m128i _sl0 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh0 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl1 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh1 = _mm_mulhi_epi16(_val1, _w0);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl1, _sh1));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl1, _sh1));
#if __XOP__
_sum0 = _mm_maddd_epi16(_val0, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val1, _w0, _sum1);
#else
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum1);
#endif
#endif

k0 += 8;
r0 += 16;
}

_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);
#if __AVX2__
__m128i _sum0 = _mm256_extracti128_si256(_sum01, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum01, 1);
#endif

output0_tm[0] = _mm_reduce_add_epi32(_sum0);
output0_tm[1] = _mm_reduce_add_epi32(_sum2);
output0_tm[1] = _mm_reduce_add_epi32(_sum1);
output0_tm += 2;
}
for (; i < tiles; i++)
@@ -1077,26 +973,23 @@ static void conv3x3s1_winograd43_pack8to1_int8_sse(const Mat& bottom_blob, Mat&
const short* k0 = kernel0_tm.row<const short>(r);

__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();

for (int q = 0; q < inch; q++)
{
__m128i _val = _mm_loadu_si128((const __m128i*)r0);
__m128i _val0 = _mm_loadu_si128((const __m128i*)r0);

__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);

__m128i _sl0 = _mm_mullo_epi16(_val, _w0);
__m128i _sh0 = _mm_mulhi_epi16(_val, _w0);
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0));
#if __XOP__
_sum0 = _mm_maddd_epi16(_val0, _w0, _sum0);
#else
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0);
#endif

k0 += 8;
r0 += 8;
}

_sum0 = _mm_add_epi32(_sum0, _sum1);

output0_tm[0] = _mm_reduce_add_epi32(_sum0);
output0_tm++;
}


+ 203
- 310
src/layer/x86/convolution_3x3_pack8to4_int8.h View File

@@ -125,59 +125,23 @@ static void conv3x3s1_winograd43_transform_kernel_pack8to4_int8_sse(const Mat& k
int q = 0;
for (; q + 3 < outch; q += 4)
{
const Mat k0 = kernel_tm.channel(q);
const Mat k1 = kernel_tm.channel(q + 1);
const Mat k2 = kernel_tm.channel(q + 2);
const Mat k3 = kernel_tm.channel(q + 3);

Mat kernel_tm = kernel_tm_pack8.channel(q / 4);
Mat g0 = kernel_tm_pack8.channel(q / 4);

for (int k = 0; k < 36; k++)
{
short* g00 = kernel_tm.row<short>(k);
short* g00 = g0.row<short>(k);

for (int p = 0; p + 7 < inch; p += 8)
{
#if __AVXVNNI__ || __AVX512VNNI__ || __XOP__
for (int i = 0; i < 4; i++)
{
const short* k00 = k0.row<const short>(p + i * 2);
const short* k10 = k1.row<const short>(p + i * 2);
const short* k20 = k2.row<const short>(p + i * 2);
const short* k30 = k3.row<const short>(p + i * 2);

const short* k01 = k0.row<const short>(p + i * 2 + 1);
const short* k11 = k1.row<const short>(p + i * 2 + 1);
const short* k21 = k2.row<const short>(p + i * 2 + 1);
const short* k31 = k3.row<const short>(p + i * 2 + 1);

g00[0] = k00[k];
g00[1] = k01[k];
g00[2] = k10[k];
g00[3] = k11[k];
g00[4] = k20[k];
g00[5] = k21[k];
g00[6] = k30[k];
g00[7] = k31[k];

g00 += 8;
}
#else
for (int i = 0; i < 8; i++)
{
const short* k00 = k0.row<const short>(p + i);
const short* k10 = k1.row<const short>(p + i);
const short* k20 = k2.row<const short>(p + i);
const short* k30 = k3.row<const short>(p + i);

g00[0] = k00[k];
g00[1] = k10[k];
g00[2] = k20[k];
g00[3] = k30[k];

g00 += 4;
for (int j = 0; j < 8; j++)
{
const short* k00 = kernel_tm.channel(q + i).row<const short>(p + j);
g00[0] = k00[k];
g00 += 1;
}
}
#endif
}
}
}
@@ -482,113 +446,96 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&

int nn = inch; // inch always > 0

__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
__m256i _sum4_5 = _mm256_setzero_si256();
__m256i _sum6_7 = _mm256_setzero_si256();
__m256i _sum00_11 = _mm256_setzero_si256();
__m256i _sum10_01 = _mm256_setzero_si256();
__m256i _sum02_13 = _mm256_setzero_si256();
__m256i _sum12_03 = _mm256_setzero_si256();

__m256i _sum04_15 = _mm256_setzero_si256();
__m256i _sum14_05 = _mm256_setzero_si256();
__m256i _sum06_17 = _mm256_setzero_si256();
__m256i _sum16_07 = _mm256_setzero_si256();

for (int j = 0; j < nn; j++)
{
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
__m256i _val0 = _mm256_loadu_si256((const __m256i*)r0);
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);

__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _val10 = _mm256_permute4x64_epi64(_val01, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val0_0123);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val0_89ab);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val0_4567);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val0_cdef);
_sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01);
_sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01);
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23);
#else
// 0 0 1 1 2 2 3 3 8 8 9 9 a a b b
// 4 4 5 5 6 6 7 7 c c d d e e f f
__m256i _val0_0123_89ab = _mm256_unpacklo_epi16(_val0, _val0);
__m256i _val0_4567_cdef = _mm256_unpackhi_epi16(_val0, _val0);

__m256i _val0_0123 = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val0_4567 = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val0_89ab = _mm256_permutevar8x32_epi32(_val0_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val0_cdef = _mm256_permutevar8x32_epi32(_val0_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val0_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val0_0123);
__m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val0_89ab);
__m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val0_89ab);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val0_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val0_4567);
__m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val0_cdef);
__m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val0_cdef);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23));
#endif

__m256i _val1 = _mm256_loadu_si256((const __m256i*)(r0 + 16));
__m256i _val23 = _mm256_loadu_si256((const __m256i*)(r0 + 16));

__m256i _val32 = _mm256_permute4x64_epi64(_val23, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w01, _val1_0123);
_sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w01, _val1_89ab);
_sum4_5 = _mm256_dpwssd_epi32(_sum4_5, _w23, _val1_4567);
_sum6_7 = _mm256_dpwssd_epi32(_sum6_7, _w23, _val1_cdef);
_sum04_15 = _mm256_dpwssd_epi32(_sum04_15, _val23, _w01);
_sum14_05 = _mm256_dpwssd_epi32(_sum14_05, _val32, _w01);
_sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23, _w23);
_sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32, _w23);
#else
__m256i _val1_0123_89ab = _mm256_unpacklo_epi16(_val1, _val1);
__m256i _val1_4567_cdef = _mm256_unpackhi_epi16(_val1, _val1);

__m256i _val1_0123 = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val1_4567 = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val1_89ab = _mm256_permutevar8x32_epi32(_val1_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val1_cdef = _mm256_permutevar8x32_epi32(_val1_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl04_05 = _mm256_mullo_epi16(_w01, _val1_0123);
__m256i _sh04_05 = _mm256_mulhi_epi16(_w01, _val1_0123);
__m256i _sl14_15 = _mm256_mullo_epi16(_w01, _val1_89ab);
__m256i _sh14_15 = _mm256_mulhi_epi16(_w01, _val1_89ab);
__m256i _sl06_07 = _mm256_mullo_epi16(_w23, _val1_4567);
__m256i _sh06_07 = _mm256_mulhi_epi16(_w23, _val1_4567);
__m256i _sl16_17 = _mm256_mullo_epi16(_w23, _val1_cdef);
__m256i _sh16_17 = _mm256_mulhi_epi16(_w23, _val1_cdef);

_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl04_05, _sh04_05));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl14_15, _sh14_15));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpacklo_epi16(_sl06_07, _sh06_07));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpacklo_epi16(_sl16_17, _sh16_17));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl04_05, _sh04_05));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl14_15, _sh14_15));
_sum4_5 = _mm256_add_epi32(_sum4_5, _mm256_unpackhi_epi16(_sl06_07, _sh06_07));
_sum6_7 = _mm256_add_epi32(_sum6_7, _mm256_unpackhi_epi16(_sl16_17, _sh16_17));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23, _w01));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32, _w01));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23, _w23));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32, _w23));
#endif

r0 += 32;
k0 += 32;
}

__m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1));
_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01);
_tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01);
_tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum04_15, _sum14_05);
_tmp1 = _mm256_unpacklo_epi32(_sum06_17, _sum16_07);
_tmp2 = _mm256_unpackhi_epi32(_sum04_15, _sum14_05);
_tmp3 = _mm256_unpackhi_epi32(_sum06_17, _sum16_07);
_sum04_15 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum14_05 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum06_17 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum16_07 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01);
_sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13);

__m256i _sum4_6 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum5_7 = _mm256_permute2x128_si256(_sum4_5, _sum6_7, _MM_SHUFFLE(0, 3, 0, 1));
_sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7);
_sum04_15 = _mm256_add_epi32(_sum04_15, _sum14_05);
_sum06_17 = _mm256_add_epi32(_sum06_17, _sum16_07);
_sum04_15 = _mm256_add_epi32(_sum04_15, _sum06_17);

_mm256_storeu_si256((__m256i*)output0_tm, _sum0_2);
_mm256_storeu_si256((__m256i*)(output0_tm + 8), _sum4_6);
__m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0);
_sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask);
_sum04_15 = _mm256_permutevar8x32_epi32(_sum04_15, _perm_mask);

_mm256_storeu_si256((__m256i*)output0_tm, _sum00_11);
_mm256_storeu_si256((__m256i*)(output0_tm + 8), _sum04_15);
output0_tm += 16;
}
#endif
@@ -604,60 +551,42 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
int nn = inch; // inch always > 0

#if __AVX2__
__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
__m256i _sum00_11 = _mm256_setzero_si256();
__m256i _sum10_01 = _mm256_setzero_si256();
__m256i _sum02_13 = _mm256_setzero_si256();
__m256i _sum12_03 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
__m128i _sum00 = _mm_setzero_si128();
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum02 = _mm_setzero_si128();
__m128i _sum03 = _mm_setzero_si128();
__m128i _sum10 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
__m128i _sum12 = _mm_setzero_si128();
__m128i _sum13 = _mm_setzero_si128();
#endif

for (int j = 0; j < nn; j++)
{
#if __AVX2__
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
__m256i _val = _mm256_loadu_si256((const __m256i*)r0);
__m256i _val01 = _mm256_loadu_si256((const __m256i*)r0);

__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _val10 = _mm256_permute4x64_epi64(_val01, 78);

#if __AVXVNNI__ || __AVX512VNNI__
__m256i _val_0123 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0));
__m256i _val_4567 = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(3, 3, 3, 3, 2, 2, 2, 2));
__m256i _val_89ab = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(5, 5, 5, 5, 4, 4, 4, 4));
__m256i _val_cdef = _mm256_permutevar8x32_epi32(_val, _mm256_set_epi32(7, 7, 7, 7, 6, 6, 6, 6));

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w01, _val_89ab);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _w23, _val_cdef);
_sum00_11 = _mm256_dpwssd_epi32(_sum00_11, _val01, _w01);
_sum10_01 = _mm256_dpwssd_epi32(_sum10_01, _val10, _w01);
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01, _w23);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10, _w23);
#else
__m256i _val_0123_89ab = _mm256_unpacklo_epi16(_val, _val);
__m256i _val_4567_cdef = _mm256_unpackhi_epi16(_val, _val);

__m256i _val_0123 = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val_4567 = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
__m256i _val_89ab = _mm256_permutevar8x32_epi32(_val_0123_89ab, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));
__m256i _val_cdef = _mm256_permutevar8x32_epi32(_val_4567_cdef, _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123);
__m256i _sl10_11 = _mm256_mullo_epi16(_w01, _val_89ab);
__m256i _sh10_11 = _mm256_mulhi_epi16(_w01, _val_89ab);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567);
__m256i _sl12_13 = _mm256_mullo_epi16(_w23, _val_cdef);
__m256i _sh12_13 = _mm256_mulhi_epi16(_w23, _val_cdef);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl12_13, _sh12_13));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl10_11, _sh10_11));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl12_13, _sh12_13));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01, _w01));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10, _w01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01, _w23));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10, _w23));
#endif
#else
// 0 1 2 3 4 5 6 7
@@ -670,75 +599,23 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
__m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24));

#if __XOP__
__m128i _val0_01 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val0_23 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val0_45 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val0_67 = _mm_shuffle_epi32(_val0, _MM_SHUFFLE(3, 3, 3, 3));
__m128i _val1_01 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val1_23 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val1_45 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val1_67 = _mm_shuffle_epi32(_val1, _MM_SHUFFLE(3, 3, 3, 3));

_sum0 = _mm_maddd_epi16(_val0_01, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val0_23, _w1, _sum1);
_sum2 = _mm_maddd_epi16(_val1_01, _w0, _sum2);
_sum3 = _mm_maddd_epi16(_val1_23, _w1, _sum3);
_sum0 = _mm_maddd_epi16(_val0_45, _w2, _sum0);
_sum1 = _mm_maddd_epi16(_val0_67, _w3, _sum1);
_sum2 = _mm_maddd_epi16(_val1_45, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val1_67, _w3, _sum3);
_sum00 = _mm_maddd_epi16(_val0, _w0, _sum00);
_sum01 = _mm_maddd_epi16(_val0, _w1, _sum01);
_sum02 = _mm_maddd_epi16(_val0, _w2, _sum02);
_sum03 = _mm_maddd_epi16(_val0, _w3, _sum03);
_sum10 = _mm_maddd_epi16(_val1, _w0, _sum10);
_sum11 = _mm_maddd_epi16(_val1, _w1, _sum11);
_sum12 = _mm_maddd_epi16(_val1, _w2, _sum12);
_sum13 = _mm_maddd_epi16(_val1, _w3, _sum13);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m128i _val0_0123 = _mm_unpacklo_epi16(_val0, _val0);
__m128i _val0_4567 = _mm_unpackhi_epi16(_val0, _val0);

__m128i _val1_0123 = _mm_unpacklo_epi16(_val1, _val1);
__m128i _val1_4567 = _mm_unpackhi_epi16(_val1, _val1);

__m128i _val0_01 = _mm_unpacklo_epi32(_val0_0123, _val0_0123);
__m128i _val0_23 = _mm_unpackhi_epi32(_val0_0123, _val0_0123);
__m128i _val0_45 = _mm_unpacklo_epi32(_val0_4567, _val0_4567);
__m128i _val0_67 = _mm_unpackhi_epi32(_val0_4567, _val0_4567);

__m128i _val1_01 = _mm_unpacklo_epi32(_val1_0123, _val1_0123);
__m128i _val1_23 = _mm_unpackhi_epi32(_val1_0123, _val1_0123);
__m128i _val1_45 = _mm_unpacklo_epi32(_val1_4567, _val1_4567);
__m128i _val1_67 = _mm_unpackhi_epi32(_val1_4567, _val1_4567);

__m128i _sl00 = _mm_mullo_epi16(_w0, _val0_01);
__m128i _sh00 = _mm_mulhi_epi16(_w0, _val0_01);
__m128i _sl10 = _mm_mullo_epi16(_w0, _val1_01);
__m128i _sh10 = _mm_mulhi_epi16(_w0, _val1_01);
__m128i _sl01 = _mm_mullo_epi16(_w1, _val0_23);
__m128i _sh01 = _mm_mulhi_epi16(_w1, _val0_23);
__m128i _sl11 = _mm_mullo_epi16(_w1, _val1_23);
__m128i _sh11 = _mm_mulhi_epi16(_w1, _val1_23);
__m128i _sl02 = _mm_mullo_epi16(_w2, _val0_45);
__m128i _sh02 = _mm_mulhi_epi16(_w2, _val0_45);
__m128i _sl12 = _mm_mullo_epi16(_w2, _val1_45);
__m128i _sh12 = _mm_mulhi_epi16(_w2, _val1_45);
__m128i _sl03 = _mm_mullo_epi16(_w3, _val0_67);
__m128i _sh03 = _mm_mulhi_epi16(_w3, _val0_67);
__m128i _sl13 = _mm_mullo_epi16(_w3, _val1_67);
__m128i _sh13 = _mm_mulhi_epi16(_w3, _val1_67);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl01, _sh01));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl01, _sh01));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl11, _sh11));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl11, _sh11));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl02, _sh02));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl02, _sh02));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl12, _sh12));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl12, _sh12));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl03, _sh03));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl03, _sh03));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl13, _sh13));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl13, _sh13));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02);
_sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
_sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12);
_sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13);
#endif
#endif

@@ -747,17 +624,62 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
}

#if __AVX2__
__m256i _sum0_2 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 2, 0, 0));
__m256i _sum1_3 = _mm256_permute2x128_si256(_sum0_1, _sum2_3, _MM_SHUFFLE(0, 3, 0, 1));
_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_11, _sum10_01);
_tmp1 = _mm256_unpacklo_epi32(_sum02_13, _sum12_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_11, _sum10_01);
_tmp3 = _mm256_unpackhi_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_01 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum02_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum12_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_mm256_storeu_si256((__m256i*)output0_tm, _sum0_2);
_sum00_11 = _mm256_add_epi32(_sum00_11, _sum10_01);
_sum02_13 = _mm256_add_epi32(_sum02_13, _sum12_03);
_sum00_11 = _mm256_add_epi32(_sum00_11, _sum02_13);

__m256i _perm_mask = _mm256_set_epi32(6, 3, 4, 1, 7, 2, 5, 0);
_sum00_11 = _mm256_permutevar8x32_epi32(_sum00_11, _perm_mask);

_mm256_storeu_si256((__m256i*)output0_tm, _sum00_11);
#else
_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);
// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum00, _sum01);
_tmp1 = _mm_unpacklo_epi32(_sum02, _sum03);
_tmp2 = _mm_unpackhi_epi32(_sum00, _sum01);
_tmp3 = _mm_unpackhi_epi32(_sum02, _sum03);
_sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum10, _sum11);
_tmp1 = _mm_unpacklo_epi32(_sum12, _sum13);
_tmp2 = _mm_unpackhi_epi32(_sum10, _sum11);
_tmp3 = _mm_unpackhi_epi32(_sum12, _sum13);
_sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}

_mm_storeu_si128((__m128i*)output0_tm, _sum0);
_mm_storeu_si128((__m128i*)(output0_tm + 4), _sum2);
_sum00 = _mm_add_epi32(_sum00, _sum01);
_sum02 = _mm_add_epi32(_sum02, _sum03);
_sum10 = _mm_add_epi32(_sum10, _sum11);
_sum12 = _mm_add_epi32(_sum12, _sum13);

_sum00 = _mm_add_epi32(_sum00, _sum02);
_sum10 = _mm_add_epi32(_sum10, _sum12);

_mm_storeu_si128((__m128i*)output0_tm, _sum00);
_mm_storeu_si128((__m128i*)(output0_tm + 4), _sum10);
#endif
output0_tm += 8;
}
@@ -774,9 +696,12 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&

#if __AVX2__
__m256i _sum0_1 = _mm256_setzero_si256();
__m256i _sum2_3 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
#endif

for (int j = 0; j < nn; j++)
@@ -787,37 +712,14 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
__m256i _w01 = _mm256_loadu_si256((const __m256i*)k0);
__m256i _w23 = _mm256_loadu_si256((const __m256i*)(k0 + 16));

__m256i _valval = _mm256_inserti128_si256(_mm256_castsi128_si256(_val), _val, 1);

#if __AVXVNNI__ || __AVX512VNNI__
// 0 1 0 1 x x x x
// 0 1 0 1 0 1 0 1
__m128i _val_01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val_23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val_45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val_67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3));

__m256i _val_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_01), _val_23, 1);
__m256i _val_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(_val_45), _val_67, 1);

_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w01, _val_0123);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _w23, _val_4567);
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m256i _val_0123 = _mm256_castsi128_si256(_mm_unpacklo_epi16(_val, _val));
__m256i _val_4567 = _mm256_castsi128_si256(_mm_unpackhi_epi16(_val, _val));

_val_0123 = _mm256_permutevar8x32_epi32(_val_0123, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));
_val_4567 = _mm256_permutevar8x32_epi32(_val_4567, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0));

__m256i _sl00_01 = _mm256_mullo_epi16(_w01, _val_0123);
__m256i _sh00_01 = _mm256_mulhi_epi16(_w01, _val_0123);
__m256i _sl02_03 = _mm256_mullo_epi16(_w23, _val_4567);
__m256i _sh02_03 = _mm256_mulhi_epi16(_w23, _val_4567);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl00_01, _sh00_01));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl02_03, _sh02_03));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl00_01, _sh00_01));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl02_03, _sh02_03));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23));
#endif
#else
__m128i _w0 = _mm_loadu_si128((const __m128i*)k0);
@@ -826,43 +728,15 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
__m128i _w3 = _mm_loadu_si128((const __m128i*)(k0 + 24));

#if __XOP__
__m128i _val01 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(0, 0, 0, 0));
__m128i _val23 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(1, 1, 1, 1));
__m128i _val45 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(2, 2, 2, 2));
__m128i _val67 = _mm_shuffle_epi32(_val, _MM_SHUFFLE(3, 3, 3, 3));

_sum0 = _mm_maddd_epi16(_val01, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val23, _w1, _sum1);
_sum0 = _mm_maddd_epi16(_val45, _w2, _sum0);
_sum1 = _mm_maddd_epi16(_val67, _w3, _sum1);
_sum0 = _mm_maddd_epi16(_val, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val, _w1, _sum1);
_sum2 = _mm_maddd_epi16(_val, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val, _w3, _sum3);
#else
// 0 0 1 1 2 2 3 3
// 4 4 5 5 6 6 7 7
__m128i _val_0123 = _mm_unpacklo_epi16(_val, _val);
__m128i _val_4567 = _mm_unpackhi_epi16(_val, _val);

__m128i _val01 = _mm_unpacklo_epi32(_val_0123, _val_0123);
__m128i _val23 = _mm_unpackhi_epi32(_val_0123, _val_0123);
__m128i _val45 = _mm_unpacklo_epi32(_val_4567, _val_4567);
__m128i _val67 = _mm_unpackhi_epi32(_val_4567, _val_4567);

__m128i _sl0 = _mm_mullo_epi16(_w0, _val01);
__m128i _sh0 = _mm_mulhi_epi16(_w0, _val01);
__m128i _sl1 = _mm_mullo_epi16(_w1, _val23);
__m128i _sh1 = _mm_mulhi_epi16(_w1, _val23);
__m128i _sl2 = _mm_mullo_epi16(_w2, _val45);
__m128i _sh2 = _mm_mulhi_epi16(_w2, _val45);
__m128i _sl3 = _mm_mullo_epi16(_w3, _val67);
__m128i _sh3 = _mm_mulhi_epi16(_w3, _val67);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl0, _sh0));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl1, _sh1));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl2, _sh2));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl2, _sh2));
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl3, _sh3));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1);
_sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2);
_sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3);
#endif
#endif

@@ -873,8 +747,27 @@ static void conv3x3s1_winograd43_pack8to4_int8_sse(const Mat& bottom_blob, Mat&
#if __AVX2__
__m128i _sum0 = _mm256_extracti128_si256(_sum0_1, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum0_1, 1);
__m128i _sum2 = _mm256_extracti128_si256(_sum2_3, 0);
__m128i _sum3 = _mm256_extracti128_si256(_sum2_3, 1);
#endif

// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum0, _sum1);
_tmp1 = _mm_unpacklo_epi32(_sum2, _sum3);
_tmp2 = _mm_unpackhi_epi32(_sum0, _sum1);
_tmp3 = _mm_unpackhi_epi32(_sum2, _sum3);
_sum0 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum1 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum2 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum3 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}

_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);

_sum0 = _mm_add_epi32(_sum0, _sum2);

_mm_storeu_si128((__m128i*)output0_tm, _sum0);
output0_tm += 4;


+ 20
- 156
src/layer/x86/convolution_sgemm_int8.h View File

@@ -338,17 +338,8 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const

if (nn4 > 0)
{
#if __AVXVNNI__ || __AVX512VNNI__
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum30_22 = _mm256_setzero_si256();
#else
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum01_13 = _mm256_setzero_si256();
__m256i _sum11_03 = _mm256_setzero_si256();
__m256i _sum30_22 = _mm256_setzero_si256();
__m256i _sum21_33 = _mm256_setzero_si256();
__m256i _sum31_23 = _mm256_setzero_si256();
#endif

int j = 0;
for (; j < nn4; j++)
@@ -371,72 +362,21 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const
_sum20_32 = _mm256_dpwssd_epi32(_sum20_32, _val23_16, _w01_16);
_sum30_22 = _mm256_dpwssd_epi32(_sum30_22, _val32_16, _w01_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl20_31 = _mm256_mullo_epi16(_val23_16, _w01_16);
__m256i _sh20_31 = _mm256_mulhi_epi16(_val23_16, _w01_16);
__m256i _sl30_21 = _mm256_mullo_epi16(_val32_16, _w01_16);
__m256i _sh30_21 = _mm256_mulhi_epi16(_val32_16, _w01_16);

_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_unpacklo_epi16(_sl20_31, _sh20_31));
_sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_unpacklo_epi16(_sl30_21, _sh30_21));
_sum21_33 = _mm256_add_epi32(_sum21_33, _mm256_unpackhi_epi16(_sl20_31, _sh20_31));
_sum31_23 = _mm256_add_epi32(_sum31_23, _mm256_unpackhi_epi16(_sl30_21, _sh30_21));
_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16));
_sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_madd_epi16(_val23_16, _w01_16));
_sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_madd_epi16(_val32_16, _w01_16));
#endif

tmpptr += 16;
kptr0 += 16;
}

#if __AVXVNNI__ || __AVX512VNNI__
_sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02);
_sum20_32 = _mm256_hadd_epi32(_sum20_32, _sum30_22);

_sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0));
_sum20_32 = _mm256_permute4x64_epi64(_sum20_32, _MM_SHUFFLE(2, 1, 3, 0));
#else
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02);
_tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02);
_tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum20_32, _sum30_22);
_tmp1 = _mm256_unpacklo_epi32(_sum21_33, _sum31_23);
_tmp2 = _mm256_unpackhi_epi32(_sum20_32, _sum30_22);
_tmp3 = _mm256_unpackhi_epi32(_sum21_33, _sum31_23);
_sum20_32 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum30_22 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum21_33 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum31_23 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02);
_sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13);

_sum20_32 = _mm256_add_epi32(_sum20_32, _sum30_22);
_sum21_33 = _mm256_add_epi32(_sum21_33, _sum31_23);
_sum20_32 = _mm256_add_epi32(_sum20_32, _sum21_33);

__m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0);
_sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask);
_sum20_32 = _mm256_permutevar8x32_epi32(_sum20_32, _perm_mask);
#endif
}

__m128i _sum00 = _mm256_extracti128_si256(_sum00_12, 0);
@@ -532,25 +472,10 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const
if (nn4 > 0)
{
#if __AVX2__
#if __AVXVNNI__ || __AVX512VNNI__
__m256i _sum10_02 = _mm256_setzero_si256();
#else
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum01_13 = _mm256_setzero_si256();
__m256i _sum11_03 = _mm256_setzero_si256();
#endif
#else
#if __XOP__
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
#else
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum02 = _mm_setzero_si128();
__m128i _sum03 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
__m128i _sum12 = _mm_setzero_si128();
__m128i _sum13 = _mm_setzero_si128();
#endif
#endif

int j = 0;
@@ -571,15 +496,8 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const
_sum00_12 = _mm256_dpwssd_epi32(_sum00_12, _val01_16, _w01_16);
_sum10_02 = _mm256_dpwssd_epi32(_sum10_02, _val10_16, _w01_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);

_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16));
#endif
#else
__m128i _val01 = _mm_loadl_epi64((const __m128i*)tmpptr);
@@ -604,23 +522,10 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const
_sum10 = _mm_maddd_epi16(_val1, _w0, _sum10);
_sum11 = _mm_maddd_epi16(_val1, _w1, _sum11);
#else
__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl01 = _mm_mullo_epi16(_val0, _w1);
__m128i _sh01 = _mm_mulhi_epi16(_val0, _w1);
__m128i _sl10 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh10 = _mm_mulhi_epi16(_val1, _w0);
__m128i _sl11 = _mm_mullo_epi16(_val1, _w1);
__m128i _sh11 = _mm_mulhi_epi16(_val1, _w1);

_sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl00, _sh00));
_sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl01, _sh01));
_sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl01, _sh01));
_sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl10, _sh10));
_sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl11, _sh11));
_sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl11, _sh11));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
#endif
#endif

@@ -629,67 +534,26 @@ static void im2col_sgemm_int8_sse(const Mat& bottom_im2col, Mat& top_blob, const
}

#if __AVX2__
#if __AVXVNNI__ || __AVX512VNNI__
_sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02);

_sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0));
#else
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02);
_tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02);
_tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02);
_sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13);

__m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0);
_sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask);
#endif
#else
#if __XOP__
#if __SSSE3__
_sum00 = _mm_hadd_epi32(_sum00, _sum01);
_sum10 = _mm_hadd_epi32(_sum10, _sum11);
#else
// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum00, _sum01);
_tmp1 = _mm_unpacklo_epi32(_sum02, _sum03);
_tmp2 = _mm_unpackhi_epi32(_sum00, _sum01);
_tmp3 = _mm_unpackhi_epi32(_sum02, _sum03);
_sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum10, _sum11);
_tmp1 = _mm_unpacklo_epi32(_sum12, _sum13);
_tmp2 = _mm_unpackhi_epi32(_sum10, _sum11);
_tmp3 = _mm_unpackhi_epi32(_sum12, _sum13);
_sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
__m128i _sum00_sh = _mm_shuffle_epi32(_sum00, 216);
__m128i _sum01_sh = _mm_shuffle_epi32(_sum01, 216);
__m128i _sum10_sh = _mm_shuffle_epi32(_sum10, 216);
__m128i _sum11_sh = _mm_shuffle_epi32(_sum11, 216);

_sum00 = _mm_unpacklo_epi64(_sum00_sh, _sum01_sh);
_sum01 = _mm_unpackhi_epi64(_sum00_sh, _sum01_sh);
_sum10 = _mm_unpacklo_epi64(_sum10_sh, _sum11_sh);
_sum11 = _mm_unpackhi_epi64(_sum10_sh, _sum11_sh);

_sum00 = _mm_add_epi32(_sum00, _sum01);
_sum02 = _mm_add_epi32(_sum02, _sum03);
_sum10 = _mm_add_epi32(_sum10, _sum11);
_sum12 = _mm_add_epi32(_sum12, _sum13);

_sum00 = _mm_add_epi32(_sum00, _sum02);
_sum10 = _mm_add_epi32(_sum10, _sum12);
#endif
#endif
}


+ 20
- 156
src/layer/x86/convolution_sgemm_pack1to4_int8.h View File

@@ -301,17 +301,8 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl

if (nn4 > 0)
{
#if __AVXVNNI__ || __AVX512VNNI__
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum30_22 = _mm256_setzero_si256();
#else
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum01_13 = _mm256_setzero_si256();
__m256i _sum11_03 = _mm256_setzero_si256();
__m256i _sum30_22 = _mm256_setzero_si256();
__m256i _sum21_33 = _mm256_setzero_si256();
__m256i _sum31_23 = _mm256_setzero_si256();
#endif

int j = 0;
for (; j < nn4; j++)
@@ -334,72 +325,21 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum20_32 = _mm256_dpwssd_epi32(_sum20_32, _val23_16, _w01_16);
_sum30_22 = _mm256_dpwssd_epi32(_sum30_22, _val32_16, _w01_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl20_31 = _mm256_mullo_epi16(_val23_16, _w01_16);
__m256i _sh20_31 = _mm256_mulhi_epi16(_val23_16, _w01_16);
__m256i _sl30_21 = _mm256_mullo_epi16(_val32_16, _w01_16);
__m256i _sh30_21 = _mm256_mulhi_epi16(_val32_16, _w01_16);

_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_unpacklo_epi16(_sl20_31, _sh20_31));
_sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_unpacklo_epi16(_sl30_21, _sh30_21));
_sum21_33 = _mm256_add_epi32(_sum21_33, _mm256_unpackhi_epi16(_sl20_31, _sh20_31));
_sum31_23 = _mm256_add_epi32(_sum31_23, _mm256_unpackhi_epi16(_sl30_21, _sh30_21));
_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16));
_sum20_32 = _mm256_add_epi32(_sum20_32, _mm256_madd_epi16(_val23_16, _w01_16));
_sum30_22 = _mm256_add_epi32(_sum30_22, _mm256_madd_epi16(_val32_16, _w01_16));
#endif

tmpptr += 16;
kptr0 += 16;
}

#if __AVXVNNI__ || __AVX512VNNI__
_sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02);
_sum20_32 = _mm256_hadd_epi32(_sum20_32, _sum30_22);

_sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0));
_sum20_32 = _mm256_permute4x64_epi64(_sum20_32, _MM_SHUFFLE(2, 1, 3, 0));
#else
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02);
_tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02);
_tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum20_32, _sum30_22);
_tmp1 = _mm256_unpacklo_epi32(_sum21_33, _sum31_23);
_tmp2 = _mm256_unpackhi_epi32(_sum20_32, _sum30_22);
_tmp3 = _mm256_unpackhi_epi32(_sum21_33, _sum31_23);
_sum20_32 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum30_22 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum21_33 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum31_23 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02);
_sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13);

_sum20_32 = _mm256_add_epi32(_sum20_32, _sum30_22);
_sum21_33 = _mm256_add_epi32(_sum21_33, _sum31_23);
_sum20_32 = _mm256_add_epi32(_sum20_32, _sum21_33);

__m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0);
_sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask);
_sum20_32 = _mm256_permutevar8x32_epi32(_sum20_32, _perm_mask);
#endif
}

__m128i _sum00 = _mm256_extracti128_si256(_sum00_12, 0);
@@ -458,25 +398,10 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
if (nn4 > 0)
{
#if __AVX2__
#if __AVXVNNI__ || __AVX512VNNI__
__m256i _sum10_02 = _mm256_setzero_si256();
#else
__m256i _sum10_02 = _mm256_setzero_si256();
__m256i _sum01_13 = _mm256_setzero_si256();
__m256i _sum11_03 = _mm256_setzero_si256();
#endif
#else
#if __XOP__
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
#else
__m128i _sum01 = _mm_setzero_si128();
__m128i _sum02 = _mm_setzero_si128();
__m128i _sum03 = _mm_setzero_si128();
__m128i _sum11 = _mm_setzero_si128();
__m128i _sum12 = _mm_setzero_si128();
__m128i _sum13 = _mm_setzero_si128();
#endif
#endif

int j = 0;
@@ -497,15 +422,8 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum00_12 = _mm256_dpwssd_epi32(_sum00_12, _val01_16, _w01_16);
_sum10_02 = _mm256_dpwssd_epi32(_sum10_02, _val10_16, _w01_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);

_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum01_13 = _mm256_add_epi32(_sum01_13, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum11_03 = _mm256_add_epi32(_sum11_03, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum00_12 = _mm256_add_epi32(_sum00_12, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_02 = _mm256_add_epi32(_sum10_02, _mm256_madd_epi16(_val10_16, _w01_16));
#endif
#else
__m128i _val01 = _mm_loadl_epi64((const __m128i*)tmpptr);
@@ -530,23 +448,10 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum10 = _mm_maddd_epi16(_val1, _w0, _sum10);
_sum11 = _mm_maddd_epi16(_val1, _w1, _sum11);
#else
__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl01 = _mm_mullo_epi16(_val0, _w1);
__m128i _sh01 = _mm_mulhi_epi16(_val0, _w1);
__m128i _sl10 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh10 = _mm_mulhi_epi16(_val1, _w0);
__m128i _sl11 = _mm_mullo_epi16(_val1, _w1);
__m128i _sh11 = _mm_mulhi_epi16(_val1, _w1);

_sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl00, _sh00));
_sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl01, _sh01));
_sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl01, _sh01));
_sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl10, _sh10));
_sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl11, _sh11));
_sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl11, _sh11));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
#endif
#endif

@@ -555,67 +460,26 @@ static void im2col_sgemm_pack1to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
}

#if __AVX2__
#if __AVXVNNI__ || __AVX512VNNI__
_sum00_12 = _mm256_hadd_epi32(_sum00_12, _sum10_02);

_sum00_12 = _mm256_permute4x64_epi64(_sum00_12, _MM_SHUFFLE(2, 1, 3, 0));
#else
// transpose 4x8
{
__m256i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm256_unpacklo_epi32(_sum00_12, _sum10_02);
_tmp1 = _mm256_unpacklo_epi32(_sum01_13, _sum11_03);
_tmp2 = _mm256_unpackhi_epi32(_sum00_12, _sum10_02);
_tmp3 = _mm256_unpackhi_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_unpacklo_epi64(_tmp0, _tmp1);
_sum10_02 = _mm256_unpackhi_epi64(_tmp0, _tmp1);
_sum01_13 = _mm256_unpacklo_epi64(_tmp2, _tmp3);
_sum11_03 = _mm256_unpackhi_epi64(_tmp2, _tmp3);
}

_sum00_12 = _mm256_add_epi32(_sum00_12, _sum10_02);
_sum01_13 = _mm256_add_epi32(_sum01_13, _sum11_03);
_sum00_12 = _mm256_add_epi32(_sum00_12, _sum01_13);

__m256i _perm_mask = _mm256_set_epi32(6, 4, 3, 1, 7, 5, 2, 0);
_sum00_12 = _mm256_permutevar8x32_epi32(_sum00_12, _perm_mask);
#endif
#else
#if __XOP__
#if __SSSE3__
_sum00 = _mm_hadd_epi32(_sum00, _sum01);
_sum10 = _mm_hadd_epi32(_sum10, _sum11);
#else
// transpose 4x4
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum00, _sum01);
_tmp1 = _mm_unpacklo_epi32(_sum02, _sum03);
_tmp2 = _mm_unpackhi_epi32(_sum00, _sum01);
_tmp3 = _mm_unpackhi_epi32(_sum02, _sum03);
_sum00 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum01 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum02 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum03 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
{
__m128i _tmp0, _tmp1, _tmp2, _tmp3;
_tmp0 = _mm_unpacklo_epi32(_sum10, _sum11);
_tmp1 = _mm_unpacklo_epi32(_sum12, _sum13);
_tmp2 = _mm_unpackhi_epi32(_sum10, _sum11);
_tmp3 = _mm_unpackhi_epi32(_sum12, _sum13);
_sum10 = _mm_unpacklo_epi64(_tmp0, _tmp1);
_sum11 = _mm_unpackhi_epi64(_tmp0, _tmp1);
_sum12 = _mm_unpacklo_epi64(_tmp2, _tmp3);
_sum13 = _mm_unpackhi_epi64(_tmp2, _tmp3);
}
__m128i _sum00_sh = _mm_shuffle_epi32(_sum00, 216);
__m128i _sum01_sh = _mm_shuffle_epi32(_sum01, 216);
__m128i _sum10_sh = _mm_shuffle_epi32(_sum10, 216);
__m128i _sum11_sh = _mm_shuffle_epi32(_sum11, 216);

_sum00 = _mm_unpacklo_epi64(_sum00_sh, _sum01_sh);
_sum01 = _mm_unpackhi_epi64(_sum00_sh, _sum01_sh);
_sum10 = _mm_unpacklo_epi64(_sum10_sh, _sum11_sh);
_sum11 = _mm_unpackhi_epi64(_sum10_sh, _sum11_sh);

_sum00 = _mm_add_epi32(_sum00, _sum01);
_sum02 = _mm_add_epi32(_sum02, _sum03);
_sum10 = _mm_add_epi32(_sum10, _sum11);
_sum12 = _mm_add_epi32(_sum12, _sum13);

_sum00 = _mm_add_epi32(_sum00, _sum02);
_sum10 = _mm_add_epi32(_sum10, _sum12);
#endif
#endif
}


+ 63
- 165
src/layer/x86/convolution_sgemm_pack8to1_int8.h View File

@@ -225,23 +225,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16);
__m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16);
__m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16);
__m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16);

_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16));
#endif

__m128i _val23 = _mm_loadu_si128((const __m128i*)(tmpptr + 16));
@@ -254,23 +241,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23_16, _w23_16);
_sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32_16, _w23_16);
#else
__m256i _sl04_15 = _mm256_mullo_epi16(_val23_16, _w01_16);
__m256i _sh04_15 = _mm256_mulhi_epi16(_val23_16, _w01_16);
__m256i _sl14_05 = _mm256_mullo_epi16(_val32_16, _w01_16);
__m256i _sh14_05 = _mm256_mulhi_epi16(_val32_16, _w01_16);
__m256i _sl06_17 = _mm256_mullo_epi16(_val23_16, _w23_16);
__m256i _sh06_17 = _mm256_mulhi_epi16(_val23_16, _w23_16);
__m256i _sl16_07 = _mm256_mullo_epi16(_val32_16, _w23_16);
__m256i _sh16_07 = _mm256_mulhi_epi16(_val32_16, _w23_16);

_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpacklo_epi16(_sl04_15, _sh04_15));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpacklo_epi16(_sl14_05, _sh14_05));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpacklo_epi16(_sl06_17, _sh06_17));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpacklo_epi16(_sl16_07, _sh16_07));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpackhi_epi16(_sl04_15, _sh04_15));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpackhi_epi16(_sl14_05, _sh14_05));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpackhi_epi16(_sl06_17, _sh06_17));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpackhi_epi16(_sl16_07, _sh16_07));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23_16, _w01_16));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32_16, _w01_16));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23_16, _w23_16));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32_16, _w23_16));
#endif

tmpptr += 32;
@@ -386,23 +360,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16);
__m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16);
__m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16);
__m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16);

_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16));
#endif
#else
__m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr);
@@ -429,39 +390,14 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum12 = _mm_maddd_epi16(_val1, _w2, _sum12);
_sum13 = _mm_maddd_epi16(_val1, _w3, _sum13);
#else
__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl01 = _mm_mullo_epi16(_val0, _w1);
__m128i _sh01 = _mm_mulhi_epi16(_val0, _w1);
__m128i _sl02 = _mm_mullo_epi16(_val0, _w2);
__m128i _sh02 = _mm_mulhi_epi16(_val0, _w2);
__m128i _sl03 = _mm_mullo_epi16(_val0, _w3);
__m128i _sh03 = _mm_mulhi_epi16(_val0, _w3);
__m128i _sl10 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh10 = _mm_mulhi_epi16(_val1, _w0);
__m128i _sl11 = _mm_mullo_epi16(_val1, _w1);
__m128i _sh11 = _mm_mulhi_epi16(_val1, _w1);
__m128i _sl12 = _mm_mullo_epi16(_val1, _w2);
__m128i _sh12 = _mm_mulhi_epi16(_val1, _w2);
__m128i _sl13 = _mm_mullo_epi16(_val1, _w3);
__m128i _sh13 = _mm_mulhi_epi16(_val1, _w3);

_sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpacklo_epi16(_sl01, _sh01));
_sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl02, _sh02));
_sum03 = _mm_add_epi32(_sum03, _mm_unpacklo_epi16(_sl03, _sh03));
_sum00 = _mm_add_epi32(_sum00, _mm_unpackhi_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl01, _sh01));
_sum02 = _mm_add_epi32(_sum02, _mm_unpackhi_epi16(_sl02, _sh02));
_sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl03, _sh03));
_sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpacklo_epi16(_sl11, _sh11));
_sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl12, _sh12));
_sum13 = _mm_add_epi32(_sum13, _mm_unpacklo_epi16(_sl13, _sh13));
_sum10 = _mm_add_epi32(_sum10, _mm_unpackhi_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl11, _sh11));
_sum12 = _mm_add_epi32(_sum12, _mm_unpackhi_epi16(_sl12, _sh12));
_sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl13, _sh13));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02);
_sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
_sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12);
_sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13);
#endif
#endif

@@ -582,15 +518,8 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01_16);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23_16);
#else
__m256i _sl0_1 = _mm256_mullo_epi16(_valval, _w01_16);
__m256i _sh0_1 = _mm256_mulhi_epi16(_valval, _w01_16);
__m256i _sl2_3 = _mm256_mullo_epi16(_valval, _w23_16);
__m256i _sh2_3 = _mm256_mulhi_epi16(_valval, _w23_16);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl0_1, _sh0_1));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl2_3, _sh2_3));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl0_1, _sh0_1));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl2_3, _sh2_3));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01_16));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23_16));
#endif
#else
__m128i _val = _mm_loadl_epi64((const __m128i*)tmpptr);
@@ -615,23 +544,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum2 = _mm_maddd_epi16(_val, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val, _w3, _sum3);
#else
__m128i _sl0 = _mm_mullo_epi16(_val, _w0);
__m128i _sh0 = _mm_mulhi_epi16(_val, _w0);
__m128i _sl1 = _mm_mullo_epi16(_val, _w1);
__m128i _sh1 = _mm_mulhi_epi16(_val, _w1);
__m128i _sl2 = _mm_mullo_epi16(_val, _w2);
__m128i _sh2 = _mm_mulhi_epi16(_val, _w2);
__m128i _sl3 = _mm_mullo_epi16(_val, _w3);
__m128i _sh3 = _mm_mulhi_epi16(_val, _w3);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpacklo_epi16(_sl1, _sh1));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl2, _sh2));
_sum3 = _mm_add_epi32(_sum3, _mm_unpacklo_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_sum0, _mm_unpackhi_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1));
_sum2 = _mm_add_epi32(_sum2, _mm_unpackhi_epi16(_sl2, _sh2));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1);
_sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2);
_sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3);
#endif
#endif

@@ -694,10 +610,8 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl

int nn = inch * maxk; // inch always > 0

__m256i _sum0_2 = _mm256_setzero_si256();
__m256i _sum1_3 = _mm256_setzero_si256();
__m256i _sum4_6 = _mm256_setzero_si256();
__m256i _sum5_7 = _mm256_setzero_si256();
__m256i _sum01 = _mm256_setzero_si256();
__m256i _sum23 = _mm256_setzero_si256();

int j = 0;
for (; j < nn; j++)
@@ -711,31 +625,27 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
__m256i _w01_16 = _mm256_cvtepi8_epi16(_w01);
_w01_16 = _mm256_permute4x64_epi64(_w01_16, _MM_SHUFFLE(1, 0, 1, 0));

__m256i _sl00_10 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_10 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl20_30 = _mm256_mullo_epi16(_val23_16, _w01_16);
__m256i _sh20_30 = _mm256_mulhi_epi16(_val23_16, _w01_16);

_sum0_2 = _mm256_add_epi32(_sum0_2, _mm256_unpacklo_epi16(_sl00_10, _sh00_10));
_sum1_3 = _mm256_add_epi32(_sum1_3, _mm256_unpackhi_epi16(_sl00_10, _sh00_10));
_sum4_6 = _mm256_add_epi32(_sum4_6, _mm256_unpacklo_epi16(_sl20_30, _sh20_30));
_sum5_7 = _mm256_add_epi32(_sum5_7, _mm256_unpackhi_epi16(_sl20_30, _sh20_30));
#if __AVXVNNI__ || __AVX512VNNI__
_sum01 = _mm256_dpwssd_epi32(_sum01, _val01_16, _w01_16);
_sum23 = _mm256_dpwssd_epi32(_sum23, _val23_16, _w01_16);
#else
_sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01_16, _w01_16));
_sum23 = _mm256_add_epi32(_sum23, _mm256_madd_epi16(_val23_16, _w01_16));
#endif

tmpptr += 32;
kptr0 += 8;
}

_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
_sum4_6 = _mm256_add_epi32(_sum4_6, _sum5_7);
__m128i _sum0 = _mm256_extracti128_si256(_sum0_2, 0);
__m128i _sum2 = _mm256_extracti128_si256(_sum0_2, 1);
__m128i _sum4 = _mm256_extracti128_si256(_sum4_6, 0);
__m128i _sum6 = _mm256_extracti128_si256(_sum4_6, 1);
__m128i _sum0 = _mm256_extracti128_si256(_sum01, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum01, 1);
__m128i _sum2 = _mm256_extracti128_si256(_sum23, 0);
__m128i _sum3 = _mm256_extracti128_si256(_sum23, 1);

outptr0[0] = _mm_reduce_add_epi32(_sum0);
outptr0[1] = _mm_reduce_add_epi32(_sum2);
outptr0[2] = _mm_reduce_add_epi32(_sum4);
outptr0[3] = _mm_reduce_add_epi32(_sum6);
outptr0[1] = _mm_reduce_add_epi32(_sum1);
outptr0[2] = _mm_reduce_add_epi32(_sum2);
outptr0[3] = _mm_reduce_add_epi32(_sum3);
outptr0 += 4;
}
#endif
@@ -751,13 +661,10 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
int nn = inch * maxk; // inch always > 0

#if __AVX2__
__m256i _sum0_2 = _mm256_setzero_si256();
__m256i _sum1_3 = _mm256_setzero_si256();
__m256i _sum01 = _mm256_setzero_si256();
#else
__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();
__m128i _sum2 = _mm_setzero_si128();
__m128i _sum3 = _mm_setzero_si128();
#endif

int j = 0;
@@ -771,11 +678,11 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
__m256i _w01_16 = _mm256_cvtepi8_epi16(_w01);
_w01_16 = _mm256_permute4x64_epi64(_w01_16, _MM_SHUFFLE(1, 0, 1, 0));

__m256i _sl00_10 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_10 = _mm256_mulhi_epi16(_val01_16, _w01_16);
_sum0_2 = _mm256_add_epi32(_sum0_2, _mm256_unpacklo_epi16(_sl00_10, _sh00_10));
_sum1_3 = _mm256_add_epi32(_sum1_3, _mm256_unpackhi_epi16(_sl00_10, _sh00_10));
#if __AVXVNNI__ || __AVX512VNNI__
_sum01 = _mm256_dpwssd_epi32(_sum01, _val01_16, _w01_16);
#else
_sum01 = _mm256_add_epi32(_sum01, _mm256_madd_epi16(_val01_16, _w01_16));
#endif
#else
__m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr);
__m128i _extval01 = _mm_cmpgt_epi8(_mm_setzero_si128(), _val01);
@@ -790,15 +697,13 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
__m128i _w0 = _mm_unpacklo_epi8(_w01, _extw01);
#endif

__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl10 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh10 = _mm_mulhi_epi16(_val1, _w0);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl10, _sh10));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl10, _sh10));
#if __XOP__
_sum0 = _mm_maddd_epi16(_val0, _w0, _sum0);
_sum1 = _mm_maddd_epi16(_val1, _w0, _sum1);
#else
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum1);
#endif
#endif

tmpptr += 16;
@@ -806,16 +711,12 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
}

#if __AVX2__
_sum0_2 = _mm256_add_epi32(_sum0_2, _sum1_3);
__m128i _sum0 = _mm256_extracti128_si256(_sum0_2, 0);
__m128i _sum2 = _mm256_extracti128_si256(_sum0_2, 1);
#else
_sum0 = _mm_add_epi32(_sum0, _sum1);
_sum2 = _mm_add_epi32(_sum2, _sum3);
__m128i _sum0 = _mm256_extracti128_si256(_sum01, 0);
__m128i _sum1 = _mm256_extracti128_si256(_sum01, 1);
#endif

outptr0[0] = _mm_reduce_add_epi32(_sum0);
outptr0[1] = _mm_reduce_add_epi32(_sum2);
outptr0[1] = _mm_reduce_add_epi32(_sum1);
outptr0 += 2;
}
for (; i < size; i++)
@@ -830,7 +731,6 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
int nn = inch * maxk; // inch always > 0

__m128i _sum0 = _mm_setzero_si128();
__m128i _sum1 = _mm_setzero_si128();

int j = 0;
for (; j < nn; j++)
@@ -851,18 +751,16 @@ static void im2col_sgemm_pack8to1_int8_sse(const Mat& bottom_im2col, Mat& top_bl
__m128i _w0 = _mm_unpacklo_epi8(_w01, _extw01);
#endif

__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl00, _sh00));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl00, _sh00));
#if __XOP__
_sum0 = _mm_maddd_epi16(_val0, _w0, _sum0);
#else
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum0);
#endif

tmpptr += 8;
kptr0 += 8;
}

_sum0 = _mm_add_epi32(_sum0, _sum1);

outptr0[0] = _mm_reduce_add_epi32(_sum0);
outptr0 += 1;
}


+ 26
- 110
src/layer/x86/convolution_sgemm_pack8to4_int8.h View File

@@ -215,23 +215,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16);
__m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16);
__m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16);
__m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16);

_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16));
#endif

__m128i _val23 = _mm_loadu_si128((const __m128i*)(tmpptr + 16));
@@ -244,23 +231,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum06_17 = _mm256_dpwssd_epi32(_sum06_17, _val23_16, _w23_16);
_sum16_07 = _mm256_dpwssd_epi32(_sum16_07, _val32_16, _w23_16);
#else
__m256i _sl04_15 = _mm256_mullo_epi16(_val23_16, _w01_16);
__m256i _sh04_15 = _mm256_mulhi_epi16(_val23_16, _w01_16);
__m256i _sl14_05 = _mm256_mullo_epi16(_val32_16, _w01_16);
__m256i _sh14_05 = _mm256_mulhi_epi16(_val32_16, _w01_16);
__m256i _sl06_17 = _mm256_mullo_epi16(_val23_16, _w23_16);
__m256i _sh06_17 = _mm256_mulhi_epi16(_val23_16, _w23_16);
__m256i _sl16_07 = _mm256_mullo_epi16(_val32_16, _w23_16);
__m256i _sh16_07 = _mm256_mulhi_epi16(_val32_16, _w23_16);

_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpacklo_epi16(_sl04_15, _sh04_15));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpacklo_epi16(_sl14_05, _sh14_05));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpacklo_epi16(_sl06_17, _sh06_17));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpacklo_epi16(_sl16_07, _sh16_07));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_unpackhi_epi16(_sl04_15, _sh04_15));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_unpackhi_epi16(_sl14_05, _sh14_05));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_unpackhi_epi16(_sl06_17, _sh06_17));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_unpackhi_epi16(_sl16_07, _sh16_07));
_sum04_15 = _mm256_add_epi32(_sum04_15, _mm256_madd_epi16(_val23_16, _w01_16));
_sum14_05 = _mm256_add_epi32(_sum14_05, _mm256_madd_epi16(_val32_16, _w01_16));
_sum06_17 = _mm256_add_epi32(_sum06_17, _mm256_madd_epi16(_val23_16, _w23_16));
_sum16_07 = _mm256_add_epi32(_sum16_07, _mm256_madd_epi16(_val32_16, _w23_16));
#endif

tmpptr += 32;
@@ -355,23 +329,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum02_13 = _mm256_dpwssd_epi32(_sum02_13, _val01_16, _w23_16);
_sum12_03 = _mm256_dpwssd_epi32(_sum12_03, _val10_16, _w23_16);
#else
__m256i _sl00_11 = _mm256_mullo_epi16(_val01_16, _w01_16);
__m256i _sh00_11 = _mm256_mulhi_epi16(_val01_16, _w01_16);
__m256i _sl10_01 = _mm256_mullo_epi16(_val10_16, _w01_16);
__m256i _sh10_01 = _mm256_mulhi_epi16(_val10_16, _w01_16);
__m256i _sl02_13 = _mm256_mullo_epi16(_val01_16, _w23_16);
__m256i _sh02_13 = _mm256_mulhi_epi16(_val01_16, _w23_16);
__m256i _sl12_03 = _mm256_mullo_epi16(_val10_16, _w23_16);
__m256i _sh12_03 = _mm256_mulhi_epi16(_val10_16, _w23_16);

_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpacklo_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpacklo_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpacklo_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpacklo_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_unpackhi_epi16(_sl00_11, _sh00_11));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_unpackhi_epi16(_sl10_01, _sh10_01));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_unpackhi_epi16(_sl02_13, _sh02_13));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_unpackhi_epi16(_sl12_03, _sh12_03));
_sum00_11 = _mm256_add_epi32(_sum00_11, _mm256_madd_epi16(_val01_16, _w01_16));
_sum10_01 = _mm256_add_epi32(_sum10_01, _mm256_madd_epi16(_val10_16, _w01_16));
_sum02_13 = _mm256_add_epi32(_sum02_13, _mm256_madd_epi16(_val01_16, _w23_16));
_sum12_03 = _mm256_add_epi32(_sum12_03, _mm256_madd_epi16(_val10_16, _w23_16));
#endif
#else
__m128i _val01 = _mm_loadu_si128((const __m128i*)tmpptr);
@@ -398,39 +359,14 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum12 = _mm_maddd_epi16(_val1, _w2, _sum12);
_sum13 = _mm_maddd_epi16(_val1, _w3, _sum13);
#else
__m128i _sl00 = _mm_mullo_epi16(_val0, _w0);
__m128i _sh00 = _mm_mulhi_epi16(_val0, _w0);
__m128i _sl01 = _mm_mullo_epi16(_val0, _w1);
__m128i _sh01 = _mm_mulhi_epi16(_val0, _w1);
__m128i _sl02 = _mm_mullo_epi16(_val0, _w2);
__m128i _sh02 = _mm_mulhi_epi16(_val0, _w2);
__m128i _sl03 = _mm_mullo_epi16(_val0, _w3);
__m128i _sh03 = _mm_mulhi_epi16(_val0, _w3);
__m128i _sl10 = _mm_mullo_epi16(_val1, _w0);
__m128i _sh10 = _mm_mulhi_epi16(_val1, _w0);
__m128i _sl11 = _mm_mullo_epi16(_val1, _w1);
__m128i _sh11 = _mm_mulhi_epi16(_val1, _w1);
__m128i _sl12 = _mm_mullo_epi16(_val1, _w2);
__m128i _sh12 = _mm_mulhi_epi16(_val1, _w2);
__m128i _sl13 = _mm_mullo_epi16(_val1, _w3);
__m128i _sh13 = _mm_mulhi_epi16(_val1, _w3);

_sum00 = _mm_add_epi32(_sum00, _mm_unpacklo_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpacklo_epi16(_sl01, _sh01));
_sum02 = _mm_add_epi32(_sum02, _mm_unpacklo_epi16(_sl02, _sh02));
_sum03 = _mm_add_epi32(_sum03, _mm_unpacklo_epi16(_sl03, _sh03));
_sum00 = _mm_add_epi32(_sum00, _mm_unpackhi_epi16(_sl00, _sh00));
_sum01 = _mm_add_epi32(_sum01, _mm_unpackhi_epi16(_sl01, _sh01));
_sum02 = _mm_add_epi32(_sum02, _mm_unpackhi_epi16(_sl02, _sh02));
_sum03 = _mm_add_epi32(_sum03, _mm_unpackhi_epi16(_sl03, _sh03));
_sum10 = _mm_add_epi32(_sum10, _mm_unpacklo_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpacklo_epi16(_sl11, _sh11));
_sum12 = _mm_add_epi32(_sum12, _mm_unpacklo_epi16(_sl12, _sh12));
_sum13 = _mm_add_epi32(_sum13, _mm_unpacklo_epi16(_sl13, _sh13));
_sum10 = _mm_add_epi32(_sum10, _mm_unpackhi_epi16(_sl10, _sh10));
_sum11 = _mm_add_epi32(_sum11, _mm_unpackhi_epi16(_sl11, _sh11));
_sum12 = _mm_add_epi32(_sum12, _mm_unpackhi_epi16(_sl12, _sh12));
_sum13 = _mm_add_epi32(_sum13, _mm_unpackhi_epi16(_sl13, _sh13));
_sum00 = _mm_add_epi32(_mm_madd_epi16(_val0, _w0), _sum00);
_sum01 = _mm_add_epi32(_mm_madd_epi16(_val0, _w1), _sum01);
_sum02 = _mm_add_epi32(_mm_madd_epi16(_val0, _w2), _sum02);
_sum03 = _mm_add_epi32(_mm_madd_epi16(_val0, _w3), _sum03);
_sum10 = _mm_add_epi32(_mm_madd_epi16(_val1, _w0), _sum10);
_sum11 = _mm_add_epi32(_mm_madd_epi16(_val1, _w1), _sum11);
_sum12 = _mm_add_epi32(_mm_madd_epi16(_val1, _w2), _sum12);
_sum13 = _mm_add_epi32(_mm_madd_epi16(_val1, _w3), _sum13);
#endif
#endif

@@ -537,15 +473,8 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum0_1 = _mm256_dpwssd_epi32(_sum0_1, _valval, _w01_16);
_sum2_3 = _mm256_dpwssd_epi32(_sum2_3, _valval, _w23_16);
#else
__m256i _sl0_1 = _mm256_mullo_epi16(_valval, _w01_16);
__m256i _sh0_1 = _mm256_mulhi_epi16(_valval, _w01_16);
__m256i _sl2_3 = _mm256_mullo_epi16(_valval, _w23_16);
__m256i _sh2_3 = _mm256_mulhi_epi16(_valval, _w23_16);

_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpacklo_epi16(_sl0_1, _sh0_1));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpacklo_epi16(_sl2_3, _sh2_3));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_unpackhi_epi16(_sl0_1, _sh0_1));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_unpackhi_epi16(_sl2_3, _sh2_3));
_sum0_1 = _mm256_add_epi32(_sum0_1, _mm256_madd_epi16(_valval, _w01_16));
_sum2_3 = _mm256_add_epi32(_sum2_3, _mm256_madd_epi16(_valval, _w23_16));
#endif
#else
__m128i _val = _mm_loadl_epi64((const __m128i*)tmpptr);
@@ -570,23 +499,10 @@ static void im2col_sgemm_pack8to4_int8_sse(const Mat& bottom_im2col, Mat& top_bl
_sum2 = _mm_maddd_epi16(_val, _w2, _sum2);
_sum3 = _mm_maddd_epi16(_val, _w3, _sum3);
#else
__m128i _sl0 = _mm_mullo_epi16(_val, _w0);
__m128i _sh0 = _mm_mulhi_epi16(_val, _w0);
__m128i _sl1 = _mm_mullo_epi16(_val, _w1);
__m128i _sh1 = _mm_mulhi_epi16(_val, _w1);
__m128i _sl2 = _mm_mullo_epi16(_val, _w2);
__m128i _sh2 = _mm_mulhi_epi16(_val, _w2);
__m128i _sl3 = _mm_mullo_epi16(_val, _w3);
__m128i _sh3 = _mm_mulhi_epi16(_val, _w3);

_sum0 = _mm_add_epi32(_sum0, _mm_unpacklo_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpacklo_epi16(_sl1, _sh1));
_sum2 = _mm_add_epi32(_sum2, _mm_unpacklo_epi16(_sl2, _sh2));
_sum3 = _mm_add_epi32(_sum3, _mm_unpacklo_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_sum0, _mm_unpackhi_epi16(_sl0, _sh0));
_sum1 = _mm_add_epi32(_sum1, _mm_unpackhi_epi16(_sl1, _sh1));
_sum2 = _mm_add_epi32(_sum2, _mm_unpackhi_epi16(_sl2, _sh2));
_sum3 = _mm_add_epi32(_sum3, _mm_unpackhi_epi16(_sl3, _sh3));
_sum0 = _mm_add_epi32(_mm_madd_epi16(_val, _w0), _sum0);
_sum1 = _mm_add_epi32(_mm_madd_epi16(_val, _w1), _sum1);
_sum2 = _mm_add_epi32(_mm_madd_epi16(_val, _w2), _sum2);
_sum3 = _mm_add_epi32(_mm_madd_epi16(_val, _w3), _sum3);
#endif
#endif



+ 3
- 0
src/layer/x86/convolution_x86.cpp View File

@@ -16,12 +16,15 @@

#if __SSE2__
#include <emmintrin.h>
#if __SSSE3__
#include <tmmintrin.h>
#if __SSE4_1__
#include <smmintrin.h>
#if __AVX__
#include <immintrin.h>
#endif
#endif // __SSE4_1__
#endif // __SSSE3__
#endif // __SSE2__
#include "x86_activation.h"
#include "x86_usability.h"


Write Preview
Loading…
Cancel
Save