|
|
|
@@ -12,8 +12,21 @@ |
|
|
|
// CONDITIONS OF ANY KIND, either express or implied. See the License for the |
|
|
|
// specific language governing permissions and limitations under the License. |
|
|
|
|
|
|
|
#if NCNN_RUNTIME_CPU && NCNN_MMI && !__mips_msa && !__mips_loongson_mmi |
|
|
|
void im2col_sgemm_int8_loongson_mmi(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Option& opt); |
|
|
|
void convolution_im2col_sgemm_transform_kernel_int8_loongson_mmi(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h); |
|
|
|
#endif |
|
|
|
|
|
|
|
static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Option& opt) |
|
|
|
{ |
|
|
|
#if NCNN_RUNTIME_CPU && NCNN_MMI && !__mips_msa && !__mips_loongson_mmi |
|
|
|
if (ncnn::cpu_support_loongson_mmi()) |
|
|
|
{ |
|
|
|
im2col_sgemm_int8_loongson_mmi(bottom_im2col, top_blob, kernel, opt); |
|
|
|
return; |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
// Mat bottom_im2col(size, maxk, inch, 8u, 8, opt.workspace_allocator); |
|
|
|
|
|
|
|
const int size = bottom_im2col.w; |
|
|
|
@@ -24,7 +37,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
|
|
|
|
// permute |
|
|
|
Mat tmp; |
|
|
|
#if __mips_msa |
|
|
|
#if __mips_msa || __mips_loongson_mmi |
|
|
|
if (inch >= 4) |
|
|
|
{ |
|
|
|
if (size >= 2) |
|
|
|
@@ -33,7 +46,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
tmp.create(maxk, inch / 4 + inch % 4, size, 4u, 4, opt.workspace_allocator); |
|
|
|
} |
|
|
|
else |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
{ |
|
|
|
if (size >= 2) |
|
|
|
tmp.create(2 * maxk, inch, size / 2 + size % 2, 1u, 1, opt.workspace_allocator); |
|
|
|
@@ -52,7 +65,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
signed char* tmpptr = tmp.channel(i / 2); |
|
|
|
|
|
|
|
int q = 0; |
|
|
|
#if __mips_msa |
|
|
|
#if __mips_msa || __mips_loongson_mmi |
|
|
|
for (; q + 3 < inch; q += 4) |
|
|
|
{ |
|
|
|
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; |
|
|
|
@@ -62,6 +75,16 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
|
|
|
|
for (int k = 0; k < maxk; k++) |
|
|
|
{ |
|
|
|
#if __mips_loongson_mmi |
|
|
|
tmpptr[0] = img0[0]; |
|
|
|
tmpptr[1] = img1[0]; |
|
|
|
tmpptr[2] = img0[1]; |
|
|
|
tmpptr[3] = img1[1]; |
|
|
|
tmpptr[4] = img2[0]; |
|
|
|
tmpptr[5] = img3[0]; |
|
|
|
tmpptr[6] = img2[1]; |
|
|
|
tmpptr[7] = img3[1]; |
|
|
|
#else // __mips_loongson_mmi |
|
|
|
tmpptr[0] = img0[0]; |
|
|
|
tmpptr[1] = img1[0]; |
|
|
|
tmpptr[2] = img2[0]; |
|
|
|
@@ -70,6 +93,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
tmpptr[5] = img1[1]; |
|
|
|
tmpptr[6] = img2[1]; |
|
|
|
tmpptr[7] = img3[1]; |
|
|
|
#endif // __mips_loongson_mmi |
|
|
|
tmpptr += 8; |
|
|
|
|
|
|
|
img0 += size; |
|
|
|
@@ -78,7 +102,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
img3 += size; |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
for (; q < inch; q++) |
|
|
|
{ |
|
|
|
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; |
|
|
|
@@ -103,7 +127,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
signed char* tmpptr = tmp.channel(i / 2 + i % 2); |
|
|
|
|
|
|
|
int q = 0; |
|
|
|
#if __mips_msa |
|
|
|
#if __mips_msa || __mips_loongson_mmi |
|
|
|
for (; q + 3 < inch; q += 4) |
|
|
|
{ |
|
|
|
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; |
|
|
|
@@ -125,7 +149,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
img3 += size; |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
for (; q < inch; q++) |
|
|
|
{ |
|
|
|
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; |
|
|
|
@@ -410,13 +434,111 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
const signed char* tmpptr = tmp.channel(i / 2); |
|
|
|
const signed char* kptr = kernel.channel(p / 2); |
|
|
|
|
|
|
|
int nn1 = inch * maxk; |
|
|
|
|
|
|
|
int sum00 = 0; |
|
|
|
int sum01 = 0; |
|
|
|
int sum10 = 0; |
|
|
|
int sum11 = 0; |
|
|
|
|
|
|
|
#if __mips_loongson_mmi |
|
|
|
int nn4 = (inch / 4) * maxk; |
|
|
|
int nn1 = (inch % 4) * maxk; |
|
|
|
|
|
|
|
if (nn4 > 0) |
|
|
|
{ |
|
|
|
int32x2_t _sum0 = __mmi_pzerow_s(); |
|
|
|
int32x2_t _sum1 = __mmi_pzerow_s(); |
|
|
|
|
|
|
|
double temp0; |
|
|
|
double temp1; |
|
|
|
double temp2; |
|
|
|
double temp3; |
|
|
|
double temp4; |
|
|
|
double temp5; |
|
|
|
double temp6; |
|
|
|
double temp7; |
|
|
|
|
|
|
|
uint64_t flag_0x44 = 0x44; |
|
|
|
uint64_t flag_0xee = 0xee; |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn4; j++) |
|
|
|
{ |
|
|
|
asm volatile( |
|
|
|
"ld $0, 32(%0) \n" // __builtin_prefetch(tmpptr + 32); |
|
|
|
"ld $0, 32(%1) \n" // __builtin_prefetch(kptr + 32); |
|
|
|
|
|
|
|
"ldc1 %4, 0(%0) \n" // int8x8_t _v = __mmi_pldb_s(tmpptr); |
|
|
|
"ldc1 %6, 0(%1) \n" // int8x8_t _k = __mmi_pldb_s(kptr); |
|
|
|
|
|
|
|
"mtc1 $0, %8 \n" // int8x8_t _zero = __mmi_pzerob_s(); |
|
|
|
"pcmpgtb %5, %8, %4 \n" // int8x8_t _extv = __mmi_pcmpgtb_s(_zero, _v); |
|
|
|
"pcmpgtb %7, %8, %6 \n" // int8x8_t _extk = __mmi_pcmpgtb_s(_zero, _k); |
|
|
|
|
|
|
|
"punpcklbh %8, %4, %5 \n" // int16x4_t _v0 = (int16x4_t)__mmi_punpcklbh_s(_v, _extv); |
|
|
|
"punpckhbh %9, %4, %5 \n" // int16x4_t _v1 = (int16x4_t)__mmi_punpckhbh_s(_v, _extv); |
|
|
|
"punpcklbh %10, %6, %7 \n" // int16x4_t _k0 = (int16x4_t)__mmi_punpcklbh_s(_k, _extk); |
|
|
|
"punpckhbh %11, %6, %7 \n" // int16x4_t _k1 = (int16x4_t)__mmi_punpckhbh_s(_k, _extk); |
|
|
|
|
|
|
|
"pshufh %4, %10, %24 \n" // int16x4_t _k0202 = __mmi_pshufh_s(_k0, 0x44); |
|
|
|
"pshufh %5, %10, %25 \n" // int16x4_t _k1313 = __mmi_pshufh_s(_k0, 0xee); |
|
|
|
"pshufh %6, %11, %24 \n" // int16x4_t _k4646 = __mmi_pshufh_s(_k1, 0x44); |
|
|
|
"pshufh %7, %11, %25 \n" // int16x4_t _k5757 = __mmi_pshufh_s(_k1, 0xee); |
|
|
|
|
|
|
|
"pmaddhw %4, %8, %4 \n" // int32x2_t _s0x = __mmi_pmaddhw(_v0, _k0202); |
|
|
|
"pmaddhw %5, %8, %5 \n" // int32x2_t _s1x = __mmi_pmaddhw(_v0, _k1313); |
|
|
|
"pmaddhw %6, %9, %6 \n" // int32x2_t _s0y = __mmi_pmaddhw(_v1, _k4646); |
|
|
|
"pmaddhw %7, %9, %7 \n" // int32x2_t _s1y = __mmi_pmaddhw(_v1, _k5757); |
|
|
|
|
|
|
|
"paddw %2, %2, %4 \n" // _sum0 = __mmi_paddw_s(_sum0, _s0x); |
|
|
|
"paddw %3, %3, %5 \n" // _sum1 = __mmi_paddw_s(_sum1, _s1x); |
|
|
|
"paddw %2, %2, %6 \n" // _sum0 = __mmi_paddw_s(_sum0, _s0y); |
|
|
|
"paddw %3, %3, %7 \n" // _sum1 = __mmi_paddw_s(_sum1, _s1y); |
|
|
|
|
|
|
|
: "=r"(tmpptr), // %0 |
|
|
|
"=r"(kptr), // %1 |
|
|
|
"=f"(_sum0), // %2 |
|
|
|
"=f"(_sum1), // %3 |
|
|
|
"=f"(temp0), // %4 |
|
|
|
"=f"(temp1), // %5 |
|
|
|
"=f"(temp2), // %6 |
|
|
|
"=f"(temp3), // %7 |
|
|
|
"=f"(temp4), // %8 |
|
|
|
"=f"(temp5), // %9 |
|
|
|
"=f"(temp6), // %10 |
|
|
|
"=f"(temp7) // %11 |
|
|
|
: "0"(tmpptr), |
|
|
|
"1"(kptr), |
|
|
|
"2"(_sum0), |
|
|
|
"3"(_sum1), |
|
|
|
"4"(temp0), |
|
|
|
"5"(temp1), |
|
|
|
"6"(temp2), |
|
|
|
"7"(temp3), |
|
|
|
"8"(temp4), |
|
|
|
"9"(temp5), |
|
|
|
"10"(temp6), |
|
|
|
"11"(temp7), |
|
|
|
"f"(flag_0x44), // %24 |
|
|
|
"f"(flag_0xee) // %25 |
|
|
|
: "memory"); |
|
|
|
|
|
|
|
tmpptr += 8; |
|
|
|
kptr += 8; |
|
|
|
} |
|
|
|
|
|
|
|
int sum[4]; |
|
|
|
__mmi_pstw_s(sum, _sum0); |
|
|
|
__mmi_pstw_s(sum + 2, _sum1); |
|
|
|
|
|
|
|
sum00 = sum[0]; |
|
|
|
sum01 = sum[1]; |
|
|
|
sum10 = sum[2]; |
|
|
|
sum11 = sum[3]; |
|
|
|
} |
|
|
|
#else // __mips_loongson_mmi |
|
|
|
int nn1 = inch * maxk; |
|
|
|
#endif // __mips_loongson_mmi |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn1; j++) |
|
|
|
{ |
|
|
|
@@ -446,11 +568,90 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
const signed char* tmpptr = tmp.channel(i / 2 + i % 2); |
|
|
|
const signed char* kptr = kernel.channel(p / 2); |
|
|
|
|
|
|
|
int nn1 = inch * maxk; |
|
|
|
|
|
|
|
int sum00 = 0; |
|
|
|
int sum10 = 0; |
|
|
|
|
|
|
|
#if __mips_loongson_mmi |
|
|
|
int nn4 = (inch / 4) * maxk; |
|
|
|
int nn1 = (inch % 4) * maxk; |
|
|
|
|
|
|
|
if (nn4 > 0) |
|
|
|
{ |
|
|
|
int32x2_t _sum01 = __mmi_pzerow_s(); |
|
|
|
|
|
|
|
double temp0; |
|
|
|
double temp1; |
|
|
|
double temp2; |
|
|
|
double temp3; |
|
|
|
double temp4; |
|
|
|
double temp5; |
|
|
|
|
|
|
|
uint64_t flag_0x44 = 0x44; |
|
|
|
uint64_t flag_0xee = 0xee; |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn4; j++) |
|
|
|
{ |
|
|
|
asm volatile( |
|
|
|
"ld $0, 16(%0) \n" // __builtin_prefetch(tmpptr + 16); |
|
|
|
"ld $0, 32(%1) \n" // __builtin_prefetch(kptr + 32); |
|
|
|
|
|
|
|
"ldc1 %3, 0(%0) \n" // int8x8_t _v = __mmi_pldb_s(tmpptr); |
|
|
|
"ldc1 %5, 0(%1) \n" // int8x8_t _k = __mmi_pldb_s(kptr); |
|
|
|
|
|
|
|
"mtc1 $0, %7 \n" // int8x8_t _zero = __mmi_pzerob_s(); |
|
|
|
"pcmpgtb %4, %7, %3 \n" // int8x8_t _extv = __mmi_pcmpgtb_s(_zero, _v); |
|
|
|
"pcmpgtb %6, %7, %5 \n" // int8x8_t _extk = __mmi_pcmpgtb_s(_zero, _k); |
|
|
|
|
|
|
|
"punpcklbh %4, %3, %4 \n" // int16x4_t _v0 = (int16x4_t)__mmi_punpcklbh_s(_v, _extv); |
|
|
|
"punpcklbh %7, %5, %6 \n" // int16x4_t _k0 = (int16x4_t)__mmi_punpcklbh_s(_k, _extk); |
|
|
|
"punpckhbh %8, %5, %6 \n" // int16x4_t _k1 = (int16x4_t)__mmi_punpckhbh_s(_k, _extk); |
|
|
|
|
|
|
|
"pshufh %3, %4, %18 \n" // int16x4_t _v0101 = __mmi_pshufh_s(_v0, 0x44); |
|
|
|
"pshufh %4, %4, %19 \n" // int16x4_t _v2323 = __mmi_pshufh_s(_v0, 0xee); |
|
|
|
|
|
|
|
"pmaddhw %3, %3, %7 \n" // int32x2_t _s01x = __mmi_pmaddhw(_v0101, _k0); |
|
|
|
"pmaddhw %4, %4, %8 \n" // int32x2_t _s01y = __mmi_pmaddhw(_v2323, _k1); |
|
|
|
|
|
|
|
"paddw %2, %2, %3 \n" // _sum01 = __mmi_paddw_s(_sum01, _s01x); |
|
|
|
"paddw %2, %2, %4 \n" // _sum01 = __mmi_paddw_s(_sum01, _s01y); |
|
|
|
|
|
|
|
: "=r"(tmpptr), // %0 |
|
|
|
"=r"(kptr), // %1 |
|
|
|
"=f"(_sum01), // %2 |
|
|
|
"=f"(temp0), // %3 |
|
|
|
"=f"(temp1), // %4 |
|
|
|
"=f"(temp2), // %5 |
|
|
|
"=f"(temp3), // %6 |
|
|
|
"=f"(temp4), // %7 |
|
|
|
"=f"(temp5) // %8 |
|
|
|
: "0"(tmpptr), |
|
|
|
"1"(kptr), |
|
|
|
"2"(_sum01), |
|
|
|
"3"(temp0), |
|
|
|
"4"(temp1), |
|
|
|
"5"(temp2), |
|
|
|
"6"(temp3), |
|
|
|
"7"(temp4), |
|
|
|
"8"(temp5), |
|
|
|
"f"(flag_0x44), // %18 |
|
|
|
"f"(flag_0xee) // %19 |
|
|
|
: "memory"); |
|
|
|
|
|
|
|
tmpptr += 4; |
|
|
|
kptr += 8; |
|
|
|
} |
|
|
|
|
|
|
|
int sum[2]; |
|
|
|
__mmi_pstw_s(sum, _sum01); |
|
|
|
|
|
|
|
sum00 = sum[0]; |
|
|
|
sum10 = sum[1]; |
|
|
|
} |
|
|
|
#else // __mips_loongson_mmi |
|
|
|
int nn1 = inch * maxk; |
|
|
|
#endif // __mips_loongson_mmi |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn1; j++) |
|
|
|
{ |
|
|
|
@@ -526,9 +727,86 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
sum0 = _sum0[0] + _sum0[1] + _sum0[2] + _sum0[3]; |
|
|
|
sum1 = _sum1[0] + _sum1[1] + _sum1[2] + _sum1[3]; |
|
|
|
} |
|
|
|
#else |
|
|
|
#elif __mips_loongson_mmi |
|
|
|
int nn4 = (inch / 4) * maxk; |
|
|
|
int nn1 = (inch % 4) * maxk; |
|
|
|
|
|
|
|
if (nn4 > 0) |
|
|
|
{ |
|
|
|
int32x2_t _sum01 = __mmi_pzerow_s(); |
|
|
|
|
|
|
|
double temp0; |
|
|
|
double temp1; |
|
|
|
double temp2; |
|
|
|
double temp3; |
|
|
|
double temp4; |
|
|
|
double temp5; |
|
|
|
|
|
|
|
uint64_t flag_0x44 = 0x44; |
|
|
|
uint64_t flag_0xee = 0xee; |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn4; j++) |
|
|
|
{ |
|
|
|
asm volatile( |
|
|
|
"ld $0, 32(%0) \n" // __builtin_prefetch(tmpptr + 32); |
|
|
|
"ld $0, 16(%1) \n" // __builtin_prefetch(kptr + 16); |
|
|
|
|
|
|
|
"ldc1 %3, 0(%0) \n" // int8x8_t _v = __mmi_pldb_s(tmpptr); |
|
|
|
"ldc1 %5, 0(%1) \n" // int8x8_t _k = __mmi_pldb_s(kptr); |
|
|
|
|
|
|
|
"mtc1 $0, %6 \n" // int8x8_t _zero = __mmi_pzerob_s(); |
|
|
|
"pcmpgtb %4, %6, %3 \n" // int8x8_t _extv = __mmi_pcmpgtb_s(_zero, _v); |
|
|
|
"pcmpgtb %6, %6, %5 \n" // int8x8_t _extk = __mmi_pcmpgtb_s(_zero, _k); |
|
|
|
|
|
|
|
"punpcklbh %7, %3, %4 \n" // int16x4_t _v0 = (int16x4_t)__mmi_punpcklbh_s(_v, _extv); |
|
|
|
"punpckhbh %8, %3, %4 \n" // int16x4_t _v1 = (int16x4_t)__mmi_punpckhbh_s(_v, _extv); |
|
|
|
"punpcklbh %5, %5, %6 \n" // int16x4_t _k0 = (int16x4_t)__mmi_punpcklbh_s(_k, _extk); |
|
|
|
|
|
|
|
"pshufh %3, %5, %18 \n" // int16x4_t _k0202 = __mmi_pshufh_s(_k0, 0x44); |
|
|
|
"pshufh %4, %5, %19 \n" // int16x4_t _k1313 = __mmi_pshufh_s(_k0, 0xee); |
|
|
|
|
|
|
|
"pmaddhw %3, %7, %3 \n" // int32x2_t _s01x = __mmi_pmaddhw(_v0, _k0101); |
|
|
|
"pmaddhw %4, %8, %4 \n" // int32x2_t _s01y = __mmi_pmaddhw(_v1, _k2323); |
|
|
|
|
|
|
|
"paddw %2, %2, %3 \n" // _sum01 = __mmi_paddw_s(_sum01, _s01x); |
|
|
|
"paddw %2, %2, %4 \n" // _sum01 = __mmi_paddw_s(_sum01, _s01y); |
|
|
|
|
|
|
|
: "=r"(tmpptr), // %0 |
|
|
|
"=r"(kptr), // %1 |
|
|
|
"=f"(_sum01), // %2 |
|
|
|
"=f"(temp0), // %3 |
|
|
|
"=f"(temp1), // %4 |
|
|
|
"=f"(temp2), // %5 |
|
|
|
"=f"(temp3), // %6 |
|
|
|
"=f"(temp4), // %7 |
|
|
|
"=f"(temp5) // %8 |
|
|
|
: "0"(tmpptr), |
|
|
|
"1"(kptr), |
|
|
|
"2"(_sum01), |
|
|
|
"3"(temp0), |
|
|
|
"4"(temp1), |
|
|
|
"5"(temp2), |
|
|
|
"6"(temp3), |
|
|
|
"7"(temp4), |
|
|
|
"8"(temp5), |
|
|
|
"f"(flag_0x44), // %18 |
|
|
|
"f"(flag_0xee) // %19 |
|
|
|
: "memory"); |
|
|
|
|
|
|
|
tmpptr += 8; |
|
|
|
kptr += 4; |
|
|
|
} |
|
|
|
|
|
|
|
int sum[2]; |
|
|
|
__mmi_pstw_s(sum, _sum01); |
|
|
|
|
|
|
|
sum0 = sum[0]; |
|
|
|
sum1 = sum[1]; |
|
|
|
} |
|
|
|
#else // __mips_loongson_mmi |
|
|
|
int nn1 = inch * maxk; |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn1; j++) |
|
|
|
@@ -587,7 +865,65 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
|
|
|
|
sum = _sum[0] + _sum[1] + _sum[2] + _sum[3]; |
|
|
|
} |
|
|
|
#else |
|
|
|
#elif __mips_loongson_mmi |
|
|
|
int nn4 = (inch / 4) * maxk; |
|
|
|
int nn1 = (inch % 4) * maxk; |
|
|
|
|
|
|
|
if (nn4 > 0) |
|
|
|
{ |
|
|
|
int32x2_t _sum0 = __mmi_pzerow_s(); |
|
|
|
|
|
|
|
double temp0; |
|
|
|
double temp1; |
|
|
|
double temp2; |
|
|
|
double temp3; |
|
|
|
|
|
|
|
int j = 0; |
|
|
|
for (; j < nn4; j++) |
|
|
|
{ |
|
|
|
asm volatile( |
|
|
|
"ld $0, 16(%0) \n" // __builtin_prefetch(tmpptr + 16); |
|
|
|
"ld $0, 16(%1) \n" // __builtin_prefetch(kptr + 16); |
|
|
|
|
|
|
|
"ldc1 %3, 0(%0) \n" // int8x8_t _v = __mmi_pldb_s(tmpptr); |
|
|
|
"ldc1 %5, 0(%1) \n" // int8x8_t _k = __mmi_pldb_s(kptr); |
|
|
|
|
|
|
|
"mtc1 $0, %6 \n" // int8x8_t _zero = __mmi_pzerob_s(); |
|
|
|
"pcmpgtb %4, %6, %3 \n" // int8x8_t _extv = __mmi_pcmpgtb_s(_zero, _v); |
|
|
|
"pcmpgtb %6, %6, %5 \n" // int8x8_t _extk = __mmi_pcmpgtb_s(_zero, _k); |
|
|
|
|
|
|
|
"punpcklbh %3, %3, %4 \n" // int16x4_t _v0 = (int16x4_t)__mmi_punpcklbh_s(_v, _extv); |
|
|
|
"punpcklbh %5, %5, %6 \n" // int16x4_t _k0 = (int16x4_t)__mmi_punpcklbh_s(_k, _extk); |
|
|
|
|
|
|
|
"pmaddhw %3, %3, %5 \n" // int32x2_t _s0x = __mmi_pmaddhw(_v0, _k0); |
|
|
|
"paddw %2, %2, %3 \n" // _sum0 = __mmi_paddw_s(_sum0, _s0x); |
|
|
|
|
|
|
|
: "=r"(tmpptr), // %0 |
|
|
|
"=r"(kptr), // %1 |
|
|
|
"=f"(_sum0), // %2 |
|
|
|
"=f"(temp0), // %3 |
|
|
|
"=f"(temp1), // %4 |
|
|
|
"=f"(temp2), // %5 |
|
|
|
"=f"(temp3) // %6 |
|
|
|
: "0"(tmpptr), |
|
|
|
"1"(kptr), |
|
|
|
"2"(_sum0), |
|
|
|
"3"(temp0), |
|
|
|
"4"(temp1), |
|
|
|
"5"(temp2), |
|
|
|
"6"(temp3) |
|
|
|
: "memory"); |
|
|
|
|
|
|
|
tmpptr += 4; |
|
|
|
kptr += 4; |
|
|
|
} |
|
|
|
|
|
|
|
int tmp[2]; |
|
|
|
__mmi_pstw_s(tmp, _sum0); |
|
|
|
|
|
|
|
sum = tmp[0] + tmp[1]; |
|
|
|
} |
|
|
|
#else // __mips_loongson_mmi |
|
|
|
int nn1 = inch * maxk; |
|
|
|
#endif // __mips_msa |
|
|
|
|
|
|
|
@@ -611,6 +947,14 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const |
|
|
|
|
|
|
|
static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) |
|
|
|
{ |
|
|
|
#if NCNN_RUNTIME_CPU && NCNN_MMI && !__mips_msa && !__mips_loongson_mmi |
|
|
|
if (ncnn::cpu_support_loongson_mmi()) |
|
|
|
{ |
|
|
|
convolution_im2col_sgemm_transform_kernel_int8_loongson_mmi(_kernel, kernel_tm, inch, outch, kernel_w, kernel_h); |
|
|
|
return; |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
const int maxk = kernel_w * kernel_h; |
|
|
|
|
|
|
|
// interleave |
|
|
|
@@ -628,16 +972,23 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne |
|
|
|
#else |
|
|
|
if (outch >= 2) |
|
|
|
{ |
|
|
|
kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2, (size_t)1u); |
|
|
|
#if __mips_loongson_mmi |
|
|
|
if (inch >= 4) |
|
|
|
kernel_tm.create(8 * maxk, inch / 4 + inch % 4, outch / 2 + outch % 2, (size_t)1u); |
|
|
|
else |
|
|
|
#endif // __mips_loongson_mmi |
|
|
|
{ |
|
|
|
kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2, (size_t)1u); |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // __mips_msa |
|
|
|
else |
|
|
|
{ |
|
|
|
#if __mips_msa |
|
|
|
#if __mips_msa || __mips_loongson_mmi |
|
|
|
if (inch >= 4) |
|
|
|
kernel_tm.create(4 * maxk, inch / 4 + inch % 4, outch, (size_t)1u); |
|
|
|
else |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
{ |
|
|
|
kernel_tm.create(1 * maxk, inch, outch, (size_t)1u); |
|
|
|
} |
|
|
|
@@ -678,12 +1029,39 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
#else // __mips_msa |
|
|
|
#else // __mips_msa |
|
|
|
for (; q + 1 < outch; q += 2) |
|
|
|
{ |
|
|
|
signed char* g00 = kernel_tm.channel(q / 2); |
|
|
|
|
|
|
|
int p = 0; |
|
|
|
#if __mips_loongson_mmi |
|
|
|
for (; p + 3 < inch; p += 4) |
|
|
|
{ |
|
|
|
for (int k = 0; k < maxk; k++) |
|
|
|
{ |
|
|
|
const signed char* k00 = kernel.channel(q).row<const signed char>(p); |
|
|
|
const signed char* k01 = kernel.channel(q).row<const signed char>(p + 1); |
|
|
|
const signed char* k02 = kernel.channel(q).row<const signed char>(p + 2); |
|
|
|
const signed char* k03 = kernel.channel(q).row<const signed char>(p + 3); |
|
|
|
const signed char* k10 = kernel.channel(q + 1).row<const signed char>(p); |
|
|
|
const signed char* k11 = kernel.channel(q + 1).row<const signed char>(p + 1); |
|
|
|
const signed char* k12 = kernel.channel(q + 1).row<const signed char>(p + 2); |
|
|
|
const signed char* k13 = kernel.channel(q + 1).row<const signed char>(p + 3); |
|
|
|
|
|
|
|
g00[0] = k00[k]; |
|
|
|
g00[1] = k01[k]; |
|
|
|
g00[2] = k10[k]; |
|
|
|
g00[3] = k11[k]; |
|
|
|
g00[4] = k02[k]; |
|
|
|
g00[5] = k03[k]; |
|
|
|
g00[6] = k12[k]; |
|
|
|
g00[7] = k13[k]; |
|
|
|
|
|
|
|
g00 += 8; |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // __mips_loongson_mmi |
|
|
|
for (; p < inch; p++) |
|
|
|
{ |
|
|
|
for (int k = 0; k < maxk; k++) |
|
|
|
@@ -707,7 +1085,7 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne |
|
|
|
#endif |
|
|
|
|
|
|
|
int p = 0; |
|
|
|
#if __mips_msa |
|
|
|
#if __mips_msa || __mips_loongson_mmi |
|
|
|
for (; p + 3 < inch; p += 4) |
|
|
|
{ |
|
|
|
for (int k = 0; k < maxk; k++) |
|
|
|
@@ -720,7 +1098,7 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
#endif // __mips_msa |
|
|
|
#endif // __mips_msa || __mips_loongson_mmi |
|
|
|
for (; p < inch; p++) |
|
|
|
{ |
|
|
|
for (int k = 0; k < maxk; k++) |
|
|
|
|