wangwei
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

feat(dnn/x86): add sse 8816 matmul 

GitOrigin-RevId: ed8d9ee5db
tags/v0.5.0
Megvii Engine Team Xu Xinran 5 years ago
parent
25b6a13148
commit
5dbf218d19
10 changed files with 294 additions and 113 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +89
    -6
      dnn/src/x86/matrix_mul/algos.cpp
  2. +16
    -1
      dnn/src/x86/matrix_mul/algos.h
  3. +82
    -81
      dnn/src/x86/matrix_mul/int8/kernel_sse_4x8x2.h
  4. +70
    -24
      dnn/src/x86/matrix_mul/int8/sse_strategy_4x8x2.cpp
  5. +4
    -0
      dnn/src/x86/matrix_mul/int8/strategy.h
  6. +2
    -0
      dnn/src/x86/matrix_mul/opr_impl.cpp
  7. +1
    -0
      dnn/src/x86/matrix_mul/opr_impl.h
  8. +8
    -1
      dnn/test/x86/conv_bias.cpp
  9. +1
    -0
      dnn/test/x86/convolution.cpp
  10. +21
    -0
      dnn/test/x86/matrix_mul.cpp

+ 89
- 6
dnn/src/x86/matrix_mul/algos.cpp View File

@@ -184,7 +184,8 @@ MatrixMulImpl::kern_t MatrixMulImpl::AlgoInt8x8x32Vnni::get_kern(
}

MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL_DETAIL(AlgoInt8x8x32Vnni,
megdnn_x86_matmul_kern, 5,
megdnn_x86_matmul_kern,
"AlgoInt8x8x32Vnni"_hash,
x86::matmul::gemm_int8_vnni_12x32x4,
dt_int8, dt_int32, dt_uint8);
#endif
@@ -318,6 +319,8 @@ void gemm_s8s8s32_sse_4x8x2(const MatrixMulImpl::KernParam& kern_param) {
}

} // namespace

/*************************AlgoInt8x8x16AVX2********************/
void MatrixMulImpl::AlgoInt8x8x16AVX2::gemm_s8s8s16_avx2_4x16x2(
const MatrixMulImpl::KernParam& kern_param) {
MEGDNN_MARK_USED_VAR(kern_param);
@@ -389,9 +392,86 @@ size_t MatrixMulImpl::AlgoInt8x8x16AVX2::get_workspace(
.get_workspace_size();
}
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL_DETAIL(
AlgoInt8x8x16AVX2, megdnn_x86_matmul_kern, 8,
AlgoInt8x8x16AVX2, megdnn_x86_matmul_kern, "AlgoInt8x8x16AVX2"_hash,
x86::matmul::gemm_avx2_s8s8s16_4x16x2, dt_int8, dt_int16, dt_int16);

/*************************AlgoInt8x8x16SSE********************/
void MatrixMulImpl::AlgoInt8x8x16SSE::gemm_s8s8s16_sse_4x8x2(
const MatrixMulImpl::KernParam& kern_param) {
MEGDNN_MARK_USED_VAR(kern_param);
MIDOUT_BEGIN(megdnn_x86_matmul_kern_sse_4x8x2, midout_iv(2)) {
constexpr int cacheline = 64;
const size_t m = kern_param.M;
const size_t n = kern_param.N;
const size_t k = kern_param.K;
const bool trans_a = kern_param.trA;
const bool trans_b = kern_param.trB;
const size_t lda = kern_param.LDA;
const size_t ldb = kern_param.LDB;
const size_t ldc = kern_param.LDC;
auto a_type = kern_param.A_type;
auto b_type = kern_param.B_type;
auto c_type = kern_param.C_type;
const auto a_ptr = kern_param.A<dt_int8>();
const auto b_ptr = kern_param.B<dt_int8>();
auto c_ptr = kern_param.C<dt_int16>();
x86::matmul::gemm_sse_s8s8s16_4x8x2 strategy(m, n, k, a_type, b_type,
c_type);

megdnn::matmul::GemmInterleaved<x86::matmul::gemm_sse_s8s8s16_4x8x2>(
m, n, k, trans_a, trans_b, strategy, cacheline)
.execute(a_ptr, lda, b_ptr, ldb, c_ptr, ldc,
kern_param.workspace_ptr);
}
MIDOUT_END();
}
MatrixMulImpl::kern_t MatrixMulImpl::AlgoInt8x8x16SSE::get_kern(
const KernSizeParam&) const {
return gemm_s8s8s16_sse_4x8x2;
}
bool MatrixMulImpl::AlgoInt8x8x16SSE::usable(
const KernSizeParam& kern_size_param) const {
bool is_ab_same =
kern_size_param.A_type.enumv() == kern_size_param.B_type.enumv();
bool is_type_ok =
((kern_size_param.A_type.enumv() == DTypeEnum::Int8 &&
kern_size_param.C_type.enumv() == DTypeEnum::Int16) ||
(kern_size_param.A_type.enumv() == DTypeEnum::QuantizedS8 &&
kern_size_param.C_type.enumv() == DTypeEnum::QuantizedS16));
bool is_mode_ok =
kern_size_param.compute_mode == Param::ComputeMode::DEFAULT &&
is_supported(SIMDType::SSE4_1);
bool is_param_ok = is_ab_same && is_type_ok && is_mode_ok;
return is_param_ok;
}
bool MatrixMulImpl::AlgoInt8x8x16SSE::preferred(const KernSizeParam&) const {
return true;
}
size_t MatrixMulImpl::AlgoInt8x8x16SSE::get_workspace(
const KernSizeParam& kern_param) const {
constexpr int cacheline = 64;
const size_t m = kern_param.M;
const size_t n = kern_param.N;
const size_t k = kern_param.K;
const bool trans_a = kern_param.trA;
const bool trans_b = kern_param.trB;
auto a_type = kern_param.A_type;
auto b_type = kern_param.B_type;
auto c_type = kern_param.C_type;
x86::matmul::gemm_sse_s8s8s16_4x8x2 strategy(m, n, k, a_type, b_type,
c_type);

return megdnn::matmul::GemmInterleaved<x86::matmul::gemm_sse_s8s8s16_4x8x2>(
m, n, k, trans_a, trans_b, strategy, cacheline)
.get_workspace_size();
}
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL_DETAIL(AlgoInt8x8x16SSE,
megdnn_x86_matmul_kern,
"AlgoInt8x8x16SSE"_hash,
x86::matmul::gemm_sse_s8s8s16_4x8x2,
dt_int8, dt_int16, dt_int16);

/*************************AlgoInt8x8x32AVX2M4N16K2********************/
MatrixMulImpl::kern_t MatrixMulImpl::AlgoInt8x8x32AVX2M4N16K2::get_kern(
const KernSizeParam&) const {
return gemm_s8s8s32_avx2_4x16x2;
@@ -426,8 +506,9 @@ size_t MatrixMulImpl::AlgoInt8x8x32AVX2M4N16K2::get_workspace(
.get_workspace_size();
}
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL_DETAIL(
AlgoInt8x8x32AVX2M4N16K2, megdnn_x86_matmul_kern, 8,
x86::matmul::gemm_avx2_s8s8s32_4x16x2, dt_int8, dt_int32, dt_int16);
AlgoInt8x8x32AVX2M4N16K2, megdnn_x86_matmul_kern,
"AlgoInt8x8x32AVX2M4N16K2"_hash, x86::matmul::gemm_avx2_s8s8s32_4x16x2,
dt_int8, dt_int32, dt_int16);

MatrixMulImpl::kern_t MatrixMulImpl::AlgoInt8x8x32AVX2M2N4K16::get_kern(
const KernSizeParam&) const {
@@ -463,7 +544,8 @@ size_t MatrixMulImpl::AlgoInt8x8x32AVX2M2N4K16::get_workspace(
.get_workspace_size();
}
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL(AlgoInt8x8x32AVX2M2N4K16,
megdnn_x86_matmul_kern, 8,
megdnn_x86_matmul_kern,
"AlgoInt8x8x32AVX2M2N4K16"_hash,
x86::matmul::gemm_avx2_s8s8s32_2x4x16,
dt_int8, dt_int32);

@@ -501,7 +583,8 @@ size_t MatrixMulImpl::AlgoInt8x8x32SSEM4N8K2::get_workspace(
.get_workspace_size();
}
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL_IMPL_DETAIL(AlgoInt8x8x32SSEM4N8K2,
megdnn_x86_matmul_kern, 9,
megdnn_x86_matmul_kern,
"AlgoInt8x8x32SSEM4N8K2"_hash,
x86::matmul::gemm_sse_s8s8s32_4x8x2,
dt_int8, dt_int32, dt_int16);



+ 16
- 1
dnn/src/x86/matrix_mul/algos.h View File

@@ -76,7 +76,6 @@ class MatrixMulImpl::AlgoInt8x8x16AVX2 : public AlgoBase {
private:
static void gemm_s8s8s16_avx2_4x16x2(
const MatrixMulImpl::KernParam& kern_param);
static MatrixMulImpl::AlgoInt8x8x32AVX2M4N16K2 m_algo;

public:
bool is_reproducible() const override { return true; }
@@ -89,6 +88,22 @@ public:
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL();
};

class MatrixMulImpl::AlgoInt8x8x16SSE : public AlgoBase {
private:
static void gemm_s8s8s16_sse_4x8x2(
const MatrixMulImpl::KernParam& kern_param);

public:
bool is_reproducible() const override { return true; }
const char* name() const override { return "X86_INT8X8X16_SSE"; }
bool usable(const KernSizeParam&) const override;
size_t get_workspace(const KernSizeParam&) const override;
kern_t get_kern(const KernSizeParam&) const override;
void* type() const override { return sm_x86_algo_type; }
bool preferred(const KernSizeParam&) const override;
MEGDNN_REG_GEMM_FUNC_FOR_IM2COL();
};

class MatrixMulImpl::AlgoInt8x8x32SSEM4N8K2 : public AlgoBase {
public:
bool is_reproducible() const override { return true; }


+ 82
- 81
dnn/src/x86/matrix_mul/int8/kernel_sse_4x8x2.h View File

@@ -6,10 +6,17 @@
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied.
*/

#include <immintrin.h>
#ifdef WIN32
#include <avx2intrin.h>
#include <avxintrin.h>
#include <fmaintrin.h>
#include <smmintrin.h>
#endif
#include <cmath>
#include <cstdint>
#include <type_traits>
@@ -21,10 +28,44 @@ namespace x86 {

namespace matmul_sse_4x8x2 {

template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
void store_overflow(void* ptr, __m128i a);

template <>
void store_overflow<int16_t>(void* ptr, __m128i a) {
a = _mm_shufflelo_epi16(a, 0x08);
a = _mm_shufflehi_epi16(a, 0x08);
a = _mm_shuffle_epi32(a, 0x08);
_mm_storel_epi64((__m128i*)ptr, a);
}
template <>
void store_overflow<int32_t>(void* ptr, __m128i a) {
_mm_storeu_si128((__m128i*)(ptr), a);
}
template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
void store_overflow(void* ptr, __m128i a, int remain);

template <>
void store_overflow<int16_t>(void* ptr, __m128i a, int remain) {
__m128i mask = _mm_continue_mask(remain * sizeof(int16_t));
a = _mm_shufflelo_epi16(a, 0x08);
a = _mm_shufflehi_epi16(a, 0x08);
a = _mm_shuffle_epi32(a, 0x08);
_mm_maskmoveu_si128(a, mask, reinterpret_cast<char*>(ptr));
}
template <>
void store_overflow<int32_t>(void* ptr, __m128i a, int remain) {
__m128i mask = _mm_continue_mask(remain * sizeof(int32_t));
_mm_maskmoveu_si128(a, mask, reinterpret_cast<char*>(ptr));
}

template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
static inline void kern_gemm_s8s8s32_sse_4x8x2(const int16_t* pack_a_ptr,
const int8_t* pack_b_ptr,
int32_t* c_ptr, const int ldc,
CType* c_ptr, const int ldc,
const int k) {
constexpr int k_step = 2;

@@ -102,20 +143,20 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2(const int16_t* pack_a_ptr,
pack_a_ptr += 8;
pack_b_ptr += 16;
}

_mm_storeu_si128((__m128i*)(c_ptr), c_vec[0]);
_mm_storeu_si128((__m128i*)(c_ptr + 4), c_vec[1]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc + 4), c_vec[3]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc), c_vec[4]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc + 4), c_vec[5]);
_mm_storeu_si128((__m128i*)(c_ptr + 3 * ldc), c_vec[6]);
_mm_storeu_si128((__m128i*)(c_ptr + 3 * ldc + 4), c_vec[7]);
store_overflow<CType>(c_ptr, c_vec[0]);
store_overflow<CType>(c_ptr + 4, c_vec[1]);
store_overflow<CType>(c_ptr + ldc, c_vec[2]);
store_overflow<CType>(c_ptr + ldc + 4, c_vec[3]);
store_overflow<CType>(c_ptr + 2 * ldc, c_vec[4]);
store_overflow<CType>(c_ptr + 2 * ldc + 4, c_vec[5]);
store_overflow<CType>(c_ptr + 3 * ldc, c_vec[6]);
store_overflow<CType>(c_ptr + 3 * ldc + 4, c_vec[7]);
}

template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_m(
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, int32_t* c_ptr,
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, CType* c_ptr,
const int ldc, const int k, const int remain_m) {
constexpr int k_step = 2;

@@ -194,34 +235,35 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_m(
pack_b_ptr += 16;
}

_mm_storeu_si128((__m128i*)(c_ptr), c_vec[0]);
_mm_storeu_si128((__m128i*)(c_ptr + 4), c_vec[1]);
store_overflow<CType>(c_ptr, c_vec[0]);
store_overflow<CType>(c_ptr + 4, c_vec[1]);
switch (remain_m) {
case 2:
_mm_storeu_si128((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc + 4), c_vec[3]);
store_overflow<CType>(c_ptr + ldc, c_vec[2]);
store_overflow<CType>(c_ptr + ldc + 4, c_vec[3]);
break;
case 3:
_mm_storeu_si128((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc + 4), c_vec[3]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc), c_vec[4]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc + 4), c_vec[5]);
store_overflow<CType>(c_ptr + ldc, c_vec[2]);
store_overflow<CType>(c_ptr + ldc + 4, c_vec[3]);
store_overflow<CType>(c_ptr + 2 * ldc, c_vec[4]);
store_overflow<CType>(c_ptr + 2 * ldc + 4, c_vec[5]);
break;
case 4:
_mm_storeu_si128((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc + 4), c_vec[3]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc), c_vec[4]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc + 4), c_vec[5]);
_mm_storeu_si128((__m128i*)(c_ptr + 3 * ldc), c_vec[6]);
_mm_storeu_si128((__m128i*)(c_ptr + 3 * ldc + 4), c_vec[7]);
store_overflow<CType>(c_ptr + ldc, c_vec[2]);
store_overflow<CType>(c_ptr + ldc + 4, c_vec[3]);
store_overflow<CType>(c_ptr + 2 * ldc, c_vec[4]);
store_overflow<CType>(c_ptr + 2 * ldc + 4, c_vec[5]);
store_overflow<CType>(c_ptr + 3 * ldc, c_vec[6]);
store_overflow<CType>(c_ptr + 3 * ldc + 4, c_vec[7]);
default:
break;
}
}

template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_n(
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, int32_t* c_ptr,
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, CType* c_ptr,
const int ldc, const int k, int remain_n) {
constexpr int k_step = 2;

@@ -301,10 +343,10 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_n(
}

if (remain_n >= 4) {
_mm_storeu_si128((__m128i*)(c_ptr), c_vec[0]);
_mm_storeu_si128((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storeu_si128((__m128i*)(c_ptr + 2 * ldc), c_vec[4]);
_mm_storeu_si128((__m128i*)(c_ptr + 3 * ldc), c_vec[6]);
store_overflow<CType>(c_ptr, c_vec[0]);
store_overflow<CType>(c_ptr + ldc, c_vec[2]);
store_overflow<CType>(c_ptr + 2 * ldc, c_vec[4]);
store_overflow<CType>(c_ptr + 3 * ldc, c_vec[6]);
c_ptr += 4;
remain_n -= 4;
c_vec[0] = c_vec[1];
@@ -312,35 +354,16 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_n(
c_vec[4] = c_vec[5];
c_vec[6] = c_vec[7];
}

switch (remain_n) {
case 0:
break;
case 1:
*(c_ptr) = _mm_extract_epi32(c_vec[0], 0);
*(c_ptr + ldc) = _mm_extract_epi32(c_vec[2], 0);
*(c_ptr + 2 * ldc) = _mm_extract_epi32(c_vec[4], 0);
*(c_ptr + 3 * ldc) = _mm_extract_epi32(c_vec[6], 0);
break;
case 2:
case 3:
_mm_storel_epi64((__m128i*)(c_ptr), c_vec[0]);
_mm_storel_epi64((__m128i*)(c_ptr + ldc), c_vec[2]);
_mm_storel_epi64((__m128i*)(c_ptr + 2 * ldc), c_vec[4]);
_mm_storel_epi64((__m128i*)(c_ptr + 3 * ldc), c_vec[6]);
break;
}
if (remain_n == 3) {
*(c_ptr + 2) = _mm_extract_epi32(c_vec[0], 2);
*(c_ptr + ldc + 2) = _mm_extract_epi32(c_vec[2], 2);
*(c_ptr + 2 * ldc + 2) = _mm_extract_epi32(c_vec[4], 2);
*(c_ptr + 3 * ldc + 2) = _mm_extract_epi32(c_vec[6], 2);
}
store_overflow<CType>(c_ptr, c_vec[0], remain_n);
store_overflow<CType>(c_ptr + ldc, c_vec[2], remain_n);
store_overflow<CType>(c_ptr + 2 * ldc, c_vec[4], remain_n);
store_overflow<CType>(c_ptr + 3 * ldc, c_vec[6], remain_n);
}

template <typename CType>
MEGDNN_ATTRIBUTE_TARGET("sse4.1")
static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_m_n(
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, int32_t* c_ptr,
const int16_t* pack_a_ptr, const int8_t* pack_b_ptr, CType* c_ptr,
const int ldc, const int k, int remain_m, int remain_n) {
constexpr int k_step = 2;

@@ -421,8 +444,7 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_m_n(
int index_array[4]{0, 2, 4, 6};
if (remain_n >= 4) {
for (int m = 0; m < remain_m; ++m) {
_mm_storeu_si128((__m128i*)(c_ptr + m * ldc),
c_vec[index_array[m]]);
store_overflow<CType>(c_ptr + m * ldc, c_vec[index_array[m]]);
}
c_ptr += 4;
remain_n -= 4;
@@ -431,29 +453,8 @@ static inline void kern_gemm_s8s8s32_sse_4x8x2_remain_m_n(
c_vec[4] = c_vec[5];
c_vec[6] = c_vec[7];
}

switch (remain_n) {
case 0:
break;
case 1:
for (int m = 0; m < remain_m; ++m) {
*(c_ptr + m * ldc) =
_mm_extract_epi32(c_vec[index_array[m]], 0);
}
break;
case 2:
case 3:
for (int m = 0; m < remain_m; ++m) {
_mm_storel_epi64((__m128i*)(c_ptr + m * ldc),
c_vec[index_array[m]]);
}
break;
}
if (remain_n == 3) {
for (int m = 0; m < remain_m; ++m) {
*(c_ptr + m * ldc + 2) =
_mm_extract_epi32(c_vec[index_array[m]], 2);
}
for (int m = 0; m < remain_m; ++m) {
store_overflow<CType>(c_ptr + m * ldc, c_vec[index_array[m]], remain_n);
}
}



+ 70
- 24
dnn/src/x86/matrix_mul/int8/sse_strategy_4x8x2.cpp View File

@@ -6,7 +6,8 @@
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied.
*/

#include "src/common/utils.h"
@@ -18,11 +19,9 @@ using namespace megdnn;
using namespace x86;
using namespace x86::matmul;

MEGDNN_REG_GEMM_STRATEGY_IMPL(gemm_sse_s8s8s32_4x8x2);

void gemm_sse_s8s8s32_4x8x2::pack_A(dt_int16* out, const dt_int8* in, int ldin,
int y0, int ymax, int k0, int kmax,
bool transpose) const {
static inline void gemm_packa(dt_int16* out, const dt_int8* in, int ldin,
int y0, int ymax, int k0, int kmax,
bool transpose) {
if (transpose) {
matmul_sse_4x8x2::gemm_s8s8s32_sse_4x8x2_pack_at(out, in, ldin, y0,
ymax, k0, kmax);
@@ -31,10 +30,8 @@ void gemm_sse_s8s8s32_4x8x2::pack_A(dt_int16* out, const dt_int8* in, int ldin,
ymax, k0, kmax);
}
}

void gemm_sse_s8s8s32_4x8x2::pack_B(dt_int8* out, const dt_int8* in, int ldin,
int x0, int xmax, int k0, int kmax,
bool transpose) const {
static inline void gemm_packb(dt_int8* out, const dt_int8* in, int ldin, int x0,
int xmax, int k0, int kmax, bool transpose) {
if (transpose) {
matmul_sse_4x8x2::gemm_s8s8s32_sse_4x8x2_pack_bt(out, in, ldin, x0,
xmax, k0, kmax);
@@ -43,20 +40,11 @@ void gemm_sse_s8s8s32_4x8x2::pack_B(dt_int8* out, const dt_int8* in, int ldin,
xmax, k0, kmax);
}
}

void gemm_sse_s8s8s32_4x8x2::kern(const dt_int16* pack_a_ptr,
const dt_int8* pack_b_ptr, size_t m, size_t n,
size_t k, dt_int32* c_ptr, size_t ldc,
bool is_first_k, const dt_int32*,
dt_int32*) const {
megdnn_assert(A_dtype.enumv() == B_dtype.enumv() &&
((A_dtype.enumv() == DTypeEnum::Int8 &&
C_dtype.enumv() == DTypeEnum::Int32) ||
(A_dtype.enumv() == DTypeEnum::QuantizedS8 &&
C_dtype.enumv() == DTypeEnum::QuantizedS32)),
"A: %s B: %s C: %s", A_dtype.name(), B_dtype.name(),
C_dtype.name());
megdnn_assert(is_first_k == true);
template <typename CType>
static inline void gemm_kern(const dt_int16* pack_a_ptr,
const dt_int8* pack_b_ptr, size_t m, size_t n,
size_t k, CType* c_ptr, size_t ldc,
bool is_first_k) {
constexpr int m_tile = 4;
constexpr int n_tile = 8;
constexpr int k_tile = 2;
@@ -99,4 +87,62 @@ void gemm_sse_s8s8s32_4x8x2::kern(const dt_int16* pack_a_ptr,
}
}
}
MEGDNN_REG_GEMM_STRATEGY_IMPL(gemm_sse_s8s8s32_4x8x2);
void gemm_sse_s8s8s32_4x8x2::pack_A(dt_int16* out, const dt_int8* in, int ldin,
int y0, int ymax, int k0, int kmax,
bool transpose) const {
gemm_packa(out, in, ldin, y0, ymax, k0, kmax, transpose);
}

void gemm_sse_s8s8s32_4x8x2::pack_B(dt_int8* out, const dt_int8* in, int ldin,
int x0, int xmax, int k0, int kmax,
bool transpose) const {
gemm_packb(out, in, ldin, x0, xmax, k0, kmax, transpose);
}

void gemm_sse_s8s8s32_4x8x2::kern(const dt_int16* pack_a_ptr,
const dt_int8* pack_b_ptr, size_t m, size_t n,
size_t k, dt_int32* c_ptr, size_t ldc,
bool is_first_k, const dt_int32*,
dt_int32*) const {
megdnn_assert(A_dtype.enumv() == B_dtype.enumv() &&
((A_dtype.enumv() == DTypeEnum::Int8 &&
C_dtype.enumv() == DTypeEnum::Int32) ||
(A_dtype.enumv() == DTypeEnum::QuantizedS8 &&
C_dtype.enumv() == DTypeEnum::QuantizedS32)),
"A: %s B: %s C: %s", A_dtype.name(), B_dtype.name(),
C_dtype.name());
megdnn_assert(is_first_k == true);
gemm_kern(pack_a_ptr, pack_b_ptr, m, n, k, c_ptr, ldc, is_first_k);
}

MEGDNN_REG_GEMM_STRATEGY_IMPL(gemm_sse_s8s8s16_4x8x2);
void gemm_sse_s8s8s16_4x8x2::pack_A(dt_int16* out, const dt_int8* in, int ldin,
int y0, int ymax, int k0, int kmax,
bool transpose) const {
gemm_packa(out, in, ldin, y0, ymax, k0, kmax, transpose);
}

void gemm_sse_s8s8s16_4x8x2::pack_B(dt_int8* out, const dt_int8* in, int ldin,
int x0, int xmax, int k0, int kmax,
bool transpose) const {
gemm_packb(out, in, ldin, x0, xmax, k0, kmax, transpose);
}

void gemm_sse_s8s8s16_4x8x2::kern(const dt_int16* pack_a_ptr,
const dt_int8* pack_b_ptr, size_t m, size_t n,
size_t k, dt_int16* c_ptr, size_t ldc,
bool is_first_k, const dt_int32*,
dt_int32*) const {
megdnn_assert(A_dtype.enumv() == B_dtype.enumv() &&
((A_dtype.enumv() == DTypeEnum::Int8 &&
C_dtype.enumv() == DTypeEnum::Int16) ||
(A_dtype.enumv() == DTypeEnum::QuantizedS8 &&
C_dtype.enumv() == DTypeEnum::QuantizedS16)),
"A: %s B: %s C: %s", A_dtype.name(), B_dtype.name(),
C_dtype.name());
megdnn_assert(is_first_k == true);
gemm_kern(pack_a_ptr, pack_b_ptr, m, n, k, c_ptr, ldc, is_first_k);
}

// vim: syntax=cpp.doxygen

+ 4
- 0
dnn/src/x86/matrix_mul/int8/strategy.h View File

@@ -38,6 +38,10 @@ MEGDNN_REG_GEMM_STRATEGY_WITH_PACK_A_TYPE(dt_int8, dt_int16, dt_int32, dt_int32,
4, 8, 2, false, false,
gemm_sse_s8s8s32_4x8x2);

MEGDNN_REG_GEMM_STRATEGY_WITH_PACK_A_TYPE(dt_int8, dt_int16, dt_int16, dt_int32,
4, 8, 2, false, false,
gemm_sse_s8s8s16_4x8x2);

} // namespace matmul
} // namespace x86
} // namespace megdnn


+ 2
- 0
dnn/src/x86/matrix_mul/opr_impl.cpp View File

@@ -38,6 +38,7 @@ class MatrixMulImpl::AlgoPack : NonCopyableObj {
AlgoInt8x8x32AVX2M2N4K16 algoint8x8x32avx2_m2n4k16;
AlgoInt8x8x32SSEM4N8K2 algoint8x8x32sse_m4n8k2;
AlgoInt8x8x16AVX2 algoint8x8x16avx2_m4n16k2;
AlgoInt8x8x16SSE algoint8x8x16sse_m4n8k2;
AlgoF32MK8_8x8 algof32mk8_8x8;

public:
@@ -51,6 +52,7 @@ public:
all_algos.emplace_back(&algoint8x8x16avx2_m4n16k2);
all_algos.emplace_back(&algoint8x8x32avx2_m2n4k16);
all_algos.emplace_back(&algoint8x8x32sse_m4n8k2);
all_algos.emplace_back(&algoint8x8x16sse_m4n8k2);
all_algos.emplace_back(&algof32mk8_8x8);
#if MEGDNN_X86_WITH_MKL_DNN
all_algos.emplace_back(&algoint8x8x32mkldnn);


+ 1
- 0
dnn/src/x86/matrix_mul/opr_impl.h View File

@@ -56,6 +56,7 @@ protected:
class AlgoInt8x8x32AVX2M4N16K2;
class AlgoInt8x8x32SSEM4N8K2;
class AlgoInt8x8x16AVX2;
class AlgoInt8x8x16SSE;
class AlgoPack;
class AlgoF32MK8_8x8;
};


+ 8
- 1
dnn/test/x86/conv_bias.cpp View File

@@ -835,6 +835,7 @@ TEST_F(X86_MULTI_THREADS, CONV_BIAS_IM2COLMATMUL_INT8X8X) {
}
if (::megdnn::x86::is_supported(::megdnn::x86::SIMDType::SSE4_2)) {
cb("IM2COLMATMUL:X86_INT8X8X32_SSE_4X8X2");
cb2("IM2COLMATMUL:X86_INT8X8X16_SSE");
}

#undef cb
@@ -1002,7 +1003,7 @@ TEST_F(X86_MULTI_THREADS, CONV_BIAS_CONV1X1_S1_FP32_BLAS) {
}
#endif

TEST_F(X86_MULTI_THREADS, CONV_BIAS_CONV1X1_S1_INT8X8X32) {
TEST_F(X86_MULTI_THREADS, CONV_BIAS_CONV1X1_S1_INT8X8X) {
using namespace conv_bias;
UniformIntRNG rng{-50, 50};
float epsilon = 0.001;
@@ -1028,10 +1029,16 @@ TEST_F(X86_MULTI_THREADS, CONV_BIAS_CONV1X1_S1_INT8X8X32) {
checker_conv_bias(args, handle(), &rng, epsilon, dtype::Int8{},
dtype::Int8{}, dtype::Int32{}, dtype::Int32{},
"CONV1x1:X86_INT8X8X32_AVX2_2X4X16:24");
checker_conv_bias(args, handle(), &rng, epsilon, dtype::Int8{},
dtype::Int8{}, dtype::Int16{}, dtype::Int16{},
"CONV1x1:X86_INT8X8X16_AVX2");
}
checker_conv_bias(args, handle(), &rng, epsilon, dtype::Int8{},
dtype::Int8{}, dtype::Int32{}, dtype::Int32{},
"CONV1x1:X86_INT8X8X32_SSE_4X8X2:48");
checker_conv_bias(args, handle(), &rng, epsilon, dtype::Int8{},
dtype::Int8{}, dtype::Int16{}, dtype::Int16{},
"CONV1x1:X86_INT8X8X16_SSE");
}
/************************* End Conv1x1 PackA ************************/



+ 1
- 0
dnn/test/x86/convolution.cpp View File

@@ -403,6 +403,7 @@ TEST_F(X86, BENCHMARK_CONVOLUTION_I8x8x16) {
benchmark.set_dtype(0, dtype::Int8())
.set_dtype(1, dtype::Int8())
.set_dtype(2, dtype::Int16());
benchmark.set_before_exec_callback(AlgoChecker<Convolution>(".*"));
benchmark.set_display(false);
benchmark.set_times(RUN);



+ 21
- 0
dnn/test/x86/matrix_mul.cpp View File

@@ -52,6 +52,10 @@ TEST_F(X86, MATRIX_MUL_AVX2_8X8X16) {
matrix_mul::check_matrix_mul(dtype::Int8{}, dtype::Int8{}, dtype::Int16{},
handle(), "X86_INT8X8X16_AVX2");
}
TEST_F(X86, MATRIX_MUL_SSE_8X8X16) {
matrix_mul::check_matrix_mul(dtype::Int8{}, dtype::Int8{}, dtype::Int16{},
handle(), "X86_INT8X8X16_SSE");
}
TEST_F(X86, MATRIX_MUL_SSE_8X8X32) {
matrix_mul::check_matrix_mul(dtype::Int8{}, dtype::Int8{}, dtype::Int32{},
handle(), "X86_INT8X8X32_SSE_4X8X2");
@@ -132,6 +136,17 @@ TEST_F(X86, BENCHMARK_MATRIX_MUL_8X8X32) {
benchmarker_avx2_4x16x2_8816.set_before_exec_callback(
AlgoChecker<MatrixMul>("X86_INT8X8X16_AVX2"));

Benchmarker<MatrixMul> benchmarker_sse_4x8x2_8816(handle());
benchmarker_sse_4x8x2_8816.set_display(false)
.set_times(RUNS)
.set_dtype(0, dtype::Int8{})
.set_dtype(1, dtype::Int8{})
.set_dtype(2, dtype::Int16{})
.set_rng(0, rng.get())
.set_rng(1, rng.get());
benchmarker_sse_4x8x2_8816.set_before_exec_callback(
AlgoChecker<MatrixMul>("X86_INT8X8X16_SSE"));

Benchmarker<MatrixMul> benchmarker_avx2_2x4x16(handle());
benchmarker_avx2_2x4x16.set_display(false)
.set_times(RUNS)
@@ -212,9 +227,15 @@ TEST_F(X86, BENCHMARK_MATRIX_MUL_8X8X32) {
std::cout << "sse: " << sse_used << " ms, "
<< computations / sse_used << " Gflops, "
<< "speed_up " << float_used / sse_used << ", ";
auto sse_used_8816 =
benchmarker_sse_4x8x2_8816.exec({{M, K}, {K, N}, {}}) /
RUNS;
std::cout << "sse_8816: " << sse_used_8816 << " ms, "
<< computations / sse_used_8816 << " Gflops, ";
}
std::cout << std::endl;
};
run(256, 256, 256);

for (size_t M : {8, 64, 112, 256, 512}) {
for (size_t K : {8, 16, 32, 64, 112, 256, 512}) {


Write Preview
Loading…
Cancel
Save