nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

mips convolution winograd dot function and strategy (#3925)

tags/20220701
nihui GitHub 4 years ago
parent
20a14bf5ae
commit
e7ca89853e
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1240 additions and 1815 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +2
    -960
      src/layer/mips/convolution_3x3.h
  2. +112
    -843
      src/layer/mips/convolution_3x3_pack4.h
  3. +13
    -12
      src/layer/mips/convolution_mips.cpp
  4. +495
    -0
      src/layer/mips/convolution_winograd_dot.h
  5. +448
    -0
      src/layer/mips/convolution_winograd_dot_pack4.h
  6. +170
    -0
      src/layer/mips/convolution_winograd_transform_pack4.h

+ 2
- 960
src/layer/mips/convolution_3x3.h View File

@@ -188,486 +188,7 @@ static void conv3x3s1_winograd23_msa(const Mat& bottom_blob, Mat& top_blob, cons

// BEGIN dot
Mat top_blob_tm;
{
int w_tm = outw / 2 * 4;
int h_tm = outh / 2 * 4;

const int tiles = h_tm / 4 * w_tm / 4;

// permute
Mat bottom_blob_tm2;
if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 16, 4u, opt.workspace_allocator);
else
bottom_blob_tm2.create(1 * inch, tiles, 16, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(opt.num_threads)
for (int r = 0; r < 16; r++)
{
Mat tm2 = bottom_blob_tm2.channel(r);

// tile
int i = 0;
for (; i + 3 < tiles; i += 4)
{
float* tmpptr = tm2.row(i / 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
#if __mips_msa
__msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0);
#else
tmpptr[0] = r0[0];
tmpptr[1] = r0[1];
tmpptr[2] = r0[2];
tmpptr[3] = r0[3];
#endif

r0 += bottom_blob_tm.cstep;
tmpptr += 4;
}
}
for (; i < tiles; i++)
{
float* tmpptr = tm2.row(i / 4 + i % 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
tmpptr[0] = r0[0];

r0 += bottom_blob_tm.cstep;
tmpptr += 1;
}
}
}

bottom_blob_tm = Mat();
// permute end

top_blob_tm.create(tiles, 16, outch, 4u, opt.workspace_allocator);

#if __mips_msa
int nn_outch = outch >> 3;
int remain_outch_start = nn_outch << 3;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 8;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);
float* output4_tm = top_blob_tm.channel(p + 4);
float* output5_tm = top_blob_tm.channel(p + 5);
float* output6_tm = top_blob_tm.channel(p + 6);
float* output7_tm = top_blob_tm.channel(p + 7);

const Mat kernel0_tm = kernel_tm.channel(p / 8);

for (int r = 0; r < 16; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);
v4f32 _sum4 = (v4f32)__msa_fill_w(0);
v4f32 _sum5 = (v4f32)__msa_fill_w(0);
v4f32 _sum6 = (v4f32)__msa_fill_w(0);
v4f32 _sum7 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 32);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
v4i32 _w4567 = __msa_ld_w(k0 + 4, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));
_sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0));
_sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1));
_sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2));
_sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3));

r0 += 4;
k0 += 8;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);
__msa_st_w((v4i32)_sum4, output4_tm, 0);
__msa_st_w((v4i32)_sum5, output5_tm, 0);
__msa_st_w((v4i32)_sum6, output6_tm, 0);
__msa_st_w((v4i32)_sum7, output7_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
output4_tm += 4;
output5_tm += 4;
output6_tm += 4;
output7_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;
float sum4 = 0.f;
float sum5 = 0.f;
float sum6 = 0.f;
float sum7 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];
sum4 += r0[0] * k0[4];
sum5 += r0[0] * k0[5];
sum6 += r0[0] * k0[6];
sum7 += r0[0] * k0[7];

r0 += 1;
k0 += 8;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;
output4_tm[0] = sum4;
output5_tm[0] = sum5;
output6_tm[0] = sum6;
output7_tm[0] = sum7;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
output4_tm++;
output5_tm++;
output6_tm++;
output7_tm++;
}
}
}

nn_outch = (outch - remain_outch_start) >> 2;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = remain_outch_start + pp * 4;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);

const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4);

for (int r = 0; r < 16; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 16);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));

r0 += 4;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];

r0 += 1;
k0 += 4;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
}
}
}

remain_outch_start += nn_outch << 2;
#else
int nn_outch = outch >> 1;
int remain_outch_start = nn_outch << 1;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 2;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);

const Mat kernel0_tm = kernel_tm.channel(p / 2);

for (int r = 0; r < 16; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum01 = 0.f;
float sum02 = 0.f;
float sum03 = 0.f;
float sum10 = 0.f;
float sum11 = 0.f;
float sum12 = 0.f;
float sum13 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 8);
float w0 = k0[0];
float w1 = k0[1];
sum00 += r0[0] * w0;
sum01 += r0[1] * w0;
sum02 += r0[2] * w0;
sum03 += r0[3] * w0;
sum10 += r0[0] * w1;
sum11 += r0[1] * w1;
sum12 += r0[2] * w1;
sum13 += r0[3] * w1;

r0 += 4;
k0 += 2;
}

output0_tm[0] = sum00;
output0_tm[1] = sum01;
output0_tm[2] = sum02;
output0_tm[3] = sum03;
output1_tm[0] = sum10;
output1_tm[1] = sum11;
output1_tm[2] = sum12;
output1_tm[3] = sum13;

output0_tm += 4;
output1_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum10 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 4);
__builtin_prefetch(k0 + 8);
float val0 = r0[0];
sum00 += val0 * k0[0];
sum10 += val0 * k0[1];

r0 += 1;
k0 += 2;
}

output0_tm[0] = sum00;
output1_tm[0] = sum10;
output0_tm++;
output1_tm++;
}
}
}
#endif

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_outch_start; p < outch; p++)
{
float* output0_tm = top_blob_tm.channel(p);

#if __mips_msa
const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2);
#endif

for (int r = 0; r < 16; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

int j = 0;
#if __mips_msa
v4f32 _sum0 = (v4f32)__msa_fill_w(0);

for (; j < nn; j++)
{
_sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0));
r0 += 4;
k0++;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
output0_tm += 4;
#else // __mips_msa
float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 4);
float w0 = k0[0];
sum0 += r0[0] * w0;
sum1 += r0[1] * w0;
sum2 += r0[2] * w0;
sum3 += r0[3] * w0;

r0 += 4;
k0++;
}

output0_tm[0] = sum0;
output0_tm[1] = sum1;
output0_tm[2] = sum2;
output0_tm[3] = sum3;
output0_tm += 4;
#endif // __mips_msa
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum = 0.f;

for (int j = 0; j < nn; j++)
{
float w0 = k0[0];
float val0 = r0[0];
sum += val0 * w0;

r0 += 1;
k0 += 1;
}

output0_tm[0] = sum;
output0_tm += 1;
}
}
}
}
bottom_blob_tm = Mat();
convolution_winograd_dot_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt);
// END dot

// BEGIN transform output
@@ -868,486 +389,7 @@ static void conv3x3s1_winograd43_msa(const Mat& bottom_blob, Mat& top_blob, cons

// BEGIN dot
Mat top_blob_tm;
{
int w_tm = outw / 4 * 6;
int h_tm = outh / 4 * 6;

const int tiles = h_tm / 6 * w_tm / 6;

// permute
Mat bottom_blob_tm2;
if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 36, 4u, opt.workspace_allocator);
else
bottom_blob_tm2.create(1 * inch, tiles, 36, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(opt.num_threads)
for (int r = 0; r < 36; r++)
{
Mat tm2 = bottom_blob_tm2.channel(r);

// tile
int i = 0;
for (; i + 3 < tiles; i += 4)
{
float* tmpptr = tm2.row(i / 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
#if __mips_msa
__msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0);
#else
tmpptr[0] = r0[0];
tmpptr[1] = r0[1];
tmpptr[2] = r0[2];
tmpptr[3] = r0[3];
#endif

r0 += bottom_blob_tm.cstep;
tmpptr += 4;
}
}
for (; i < tiles; i++)
{
float* tmpptr = tm2.row(i / 4 + i % 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
tmpptr[0] = r0[0];

r0 += bottom_blob_tm.cstep;
tmpptr += 1;
}
}
}

bottom_blob_tm = Mat();
// permute end

top_blob_tm.create(tiles, 36, outch, 4u, opt.workspace_allocator);

#if __mips_msa
int nn_outch = outch >> 3;
int remain_outch_start = nn_outch << 3;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 8;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);
float* output4_tm = top_blob_tm.channel(p + 4);
float* output5_tm = top_blob_tm.channel(p + 5);
float* output6_tm = top_blob_tm.channel(p + 6);
float* output7_tm = top_blob_tm.channel(p + 7);

const Mat kernel0_tm = kernel_tm.channel(p / 8);

for (int r = 0; r < 36; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);
v4f32 _sum4 = (v4f32)__msa_fill_w(0);
v4f32 _sum5 = (v4f32)__msa_fill_w(0);
v4f32 _sum6 = (v4f32)__msa_fill_w(0);
v4f32 _sum7 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 32);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
v4i32 _w4567 = __msa_ld_w(k0 + 4, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));
_sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0));
_sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1));
_sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2));
_sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3));

r0 += 4;
k0 += 8;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);
__msa_st_w((v4i32)_sum4, output4_tm, 0);
__msa_st_w((v4i32)_sum5, output5_tm, 0);
__msa_st_w((v4i32)_sum6, output6_tm, 0);
__msa_st_w((v4i32)_sum7, output7_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
output4_tm += 4;
output5_tm += 4;
output6_tm += 4;
output7_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;
float sum4 = 0.f;
float sum5 = 0.f;
float sum6 = 0.f;
float sum7 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];
sum4 += r0[0] * k0[4];
sum5 += r0[0] * k0[5];
sum6 += r0[0] * k0[6];
sum7 += r0[0] * k0[7];

r0 += 1;
k0 += 8;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;
output4_tm[0] = sum4;
output5_tm[0] = sum5;
output6_tm[0] = sum6;
output7_tm[0] = sum7;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
output4_tm++;
output5_tm++;
output6_tm++;
output7_tm++;
}
}
}

nn_outch = (outch - remain_outch_start) >> 2;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = remain_outch_start + pp * 4;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);

const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4);

for (int r = 0; r < 36; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 16);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));

r0 += 4;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];

r0 += 1;
k0 += 4;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
}
}
}

remain_outch_start += nn_outch << 2;
#else
int nn_outch = outch >> 1;
int remain_outch_start = nn_outch << 1;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 2;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);

const Mat kernel0_tm = kernel_tm.channel(p / 2);

for (int r = 0; r < 36; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum01 = 0.f;
float sum02 = 0.f;
float sum03 = 0.f;
float sum10 = 0.f;
float sum11 = 0.f;
float sum12 = 0.f;
float sum13 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 8);
float w0 = k0[0];
float w1 = k0[1];
sum00 += r0[0] * w0;
sum01 += r0[1] * w0;
sum02 += r0[2] * w0;
sum03 += r0[3] * w0;
sum10 += r0[0] * w1;
sum11 += r0[1] * w1;
sum12 += r0[2] * w1;
sum13 += r0[3] * w1;

r0 += 4;
k0 += 2;
}

output0_tm[0] = sum00;
output0_tm[1] = sum01;
output0_tm[2] = sum02;
output0_tm[3] = sum03;
output1_tm[0] = sum10;
output1_tm[1] = sum11;
output1_tm[2] = sum12;
output1_tm[3] = sum13;

output0_tm += 4;
output1_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum10 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 4);
__builtin_prefetch(k0 + 8);
float val0 = r0[0];
sum00 += val0 * k0[0];
sum10 += val0 * k0[1];

r0 += 1;
k0 += 2;
}

output0_tm[0] = sum00;
output1_tm[0] = sum10;
output0_tm++;
output1_tm++;
}
}
}
#endif

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_outch_start; p < outch; p++)
{
float* output0_tm = top_blob_tm.channel(p);

#if __mips_msa
const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2);
#endif

for (int r = 0; r < 36; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

int j = 0;
#if __mips_msa
v4f32 _sum0 = (v4f32)__msa_fill_w(0);

for (; j < nn; j++)
{
_sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0));
r0 += 4;
k0++;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
output0_tm += 4;
#else // __mips_msa
float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 4);
float w0 = k0[0];
sum0 += r0[0] * w0;
sum1 += r0[1] * w0;
sum2 += r0[2] * w0;
sum3 += r0[3] * w0;

r0 += 4;
k0++;
}

output0_tm[0] = sum0;
output0_tm[1] = sum1;
output0_tm[2] = sum2;
output0_tm[3] = sum3;
output0_tm += 4;
#endif // __mips_msa
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum = 0.f;

for (int j = 0; j < nn; j++)
{
float w0 = k0[0];
float val0 = r0[0];
sum += val0 * w0;

r0 += 1;
k0 += 1;
}

output0_tm[0] = sum;
output0_tm += 1;
}
}
}
}
bottom_blob_tm = Mat();
convolution_winograd_dot_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt);
// END dot

// BEGIN transform output


+ 112
- 843
src/layer/mips/convolution_3x3_pack4.h
File diff suppressed because it is too large
View File


+ 13
- 12
src/layer/mips/convolution_mips.cpp View File

@@ -31,6 +31,7 @@ namespace ncnn {

#include "convolution_sgemm.h"
#include "convolution_winograd_transform.h"
#include "convolution_winograd_dot.h"
#include "convolution_1x1.h"
#include "convolution_3x3.h"

@@ -48,6 +49,7 @@ namespace ncnn {
#include "convolution_sgemm_pack4.h"
#include "convolution_sgemm_pack4to1.h"
#include "convolution_winograd_transform_pack4.h"
#include "convolution_winograd_dot_pack4.h"
#include "convolution_1x1_pack4.h"
#include "convolution_1x1_pack4to1.h"
#include "convolution_3x3_pack4.h"
@@ -145,12 +147,14 @@ int Convolution_mips::create_pipeline(const Option& opt)
// pack4
if (elempack == 4 && out_elempack == 4)
{
if (opt.use_winograd_convolution && (opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16)
if (opt.use_winograd_convolution && (opt.use_winograd23_convolution || opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
if (opt.use_winograd63_convolution)
if ((opt.use_winograd63_convolution && num_input >= 8 && num_output >= 8 && num_input <= 64 && num_output <= 64) || (!opt.use_winograd43_convolution && !opt.use_winograd23_convolution))
conv3x3s1_winograd63_transform_kernel_pack4_msa(weight_data, weight_winograd63_data, num_input, num_output, opt);
if (opt.use_winograd43_convolution)
else if ((opt.use_winograd43_convolution && num_input >= 8 && num_output >= 8) || (!opt.use_winograd63_convolution && !opt.use_winograd23_convolution))
conv3x3s1_winograd43_transform_kernel_pack4_msa(weight_data, weight_winograd43_data, num_input, num_output, opt);
else // if (opt.use_winograd23_convolution)
conv3x3s1_winograd23_transform_kernel_pack4_msa(weight_data, weight_winograd23_data, num_input, num_output, opt);
}
else
{
@@ -340,17 +344,14 @@ int Convolution_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
activation->forward_inplace(top_blob, opt);
}
}
else if (opt.use_winograd_convolution && (opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16)
else if (opt.use_winograd_convolution && (opt.use_winograd23_convolution || opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
// we need more proper conditions
if ((opt.use_winograd43_convolution && (w <= 10 || (w >= 15 && w <= 18) || w == 21 || w == 22) && (h <= 10 || (h >= 15 && h <= 18) || h == 21 || h == 22)) || !opt.use_winograd63_convolution)
{
conv3x3s1_winograd43_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt);
}
else
{
if ((opt.use_winograd63_convolution && num_input >= 8 && num_output >= 8 && num_input <= 64 && num_output <= 64) || (!opt.use_winograd43_convolution && !opt.use_winograd23_convolution))
conv3x3s1_winograd63_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd63_data, bias_data, opt);
}
else if ((opt.use_winograd43_convolution && num_input >= 8 && num_output >= 8) || (!opt.use_winograd63_convolution && !opt.use_winograd23_convolution))
conv3x3s1_winograd43_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt);
else // if (opt.use_winograd23_convolution)
conv3x3s1_winograd23_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd23_data, bias_data, opt);

if (activation)
{


+ 495
- 0
src/layer/mips/convolution_winograd_dot.h View File

@@ -0,0 +1,495 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void convolution_winograd_dot_msa(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt)
{
// Mat bottom_blob_tm(tiles, 16/36/64, inch, 4u, opt.workspace_allocator);

const int tiles = bottom_blob_tm.w;
const int batch = bottom_blob_tm.h;
const int inch = bottom_blob_tm.c;

// permute
Mat bottom_blob_tm2;
if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 4u, opt.workspace_allocator);
else
bottom_blob_tm2.create(1 * inch, tiles, batch, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(opt.num_threads)
for (int r = 0; r < batch; r++)
{
Mat tm2 = bottom_blob_tm2.channel(r);

// tile
int i = 0;
for (; i + 3 < tiles; i += 4)
{
float* tmpptr = tm2.row(i / 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
#if __mips_msa
__msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0);
#else
tmpptr[0] = r0[0];
tmpptr[1] = r0[1];
tmpptr[2] = r0[2];
tmpptr[3] = r0[3];
#endif

r0 += bottom_blob_tm.cstep;
tmpptr += 4;
}
}
for (; i < tiles; i++)
{
float* tmpptr = tm2.row(i / 4 + i % 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i);

for (int q = 0; q < inch; q++)
{
tmpptr[0] = r0[0];

r0 += bottom_blob_tm.cstep;
tmpptr += 1;
}
}
}

bottom_blob_tm = Mat();
// permute end

top_blob_tm.create(tiles, batch, outch, 4u, opt.workspace_allocator);

#if __mips_msa
int nn_outch = outch >> 3;
int remain_outch_start = nn_outch << 3;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 8;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);
float* output4_tm = top_blob_tm.channel(p + 4);
float* output5_tm = top_blob_tm.channel(p + 5);
float* output6_tm = top_blob_tm.channel(p + 6);
float* output7_tm = top_blob_tm.channel(p + 7);

const Mat kernel0_tm = kernel_tm.channel(p / 8);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);
v4f32 _sum4 = (v4f32)__msa_fill_w(0);
v4f32 _sum5 = (v4f32)__msa_fill_w(0);
v4f32 _sum6 = (v4f32)__msa_fill_w(0);
v4f32 _sum7 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 32);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
v4i32 _w4567 = __msa_ld_w(k0 + 4, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));
_sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0));
_sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1));
_sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2));
_sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3));

r0 += 4;
k0 += 8;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);
__msa_st_w((v4i32)_sum4, output4_tm, 0);
__msa_st_w((v4i32)_sum5, output5_tm, 0);
__msa_st_w((v4i32)_sum6, output6_tm, 0);
__msa_st_w((v4i32)_sum7, output7_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
output4_tm += 4;
output5_tm += 4;
output6_tm += 4;
output7_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;
float sum4 = 0.f;
float sum5 = 0.f;
float sum6 = 0.f;
float sum7 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];
sum4 += r0[0] * k0[4];
sum5 += r0[0] * k0[5];
sum6 += r0[0] * k0[6];
sum7 += r0[0] * k0[7];

r0 += 1;
k0 += 8;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;
output4_tm[0] = sum4;
output5_tm[0] = sum5;
output6_tm[0] = sum6;
output7_tm[0] = sum7;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
output4_tm++;
output5_tm++;
output6_tm++;
output7_tm++;
}
}
}

nn_outch = (outch - remain_outch_start) >> 2;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = remain_outch_start + pp * 4;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);
float* output2_tm = top_blob_tm.channel(p + 2);
float* output3_tm = top_blob_tm.channel(p + 3);

const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);

int j = 0;
for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 16);
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
v4i32 _w0123 = __msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0));
_sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1));
_sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2));
_sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3));

r0 += 4;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output1_tm, 0);
__msa_st_w((v4i32)_sum2, output2_tm, 0);
__msa_st_w((v4i32)_sum3, output3_tm, 0);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

int j = 0;
for (; j < nn; j++)
{
sum0 += r0[0] * k0[0];
sum1 += r0[0] * k0[1];
sum2 += r0[0] * k0[2];
sum3 += r0[0] * k0[3];

r0 += 1;
k0 += 4;
}

output0_tm[0] = sum0;
output1_tm[0] = sum1;
output2_tm[0] = sum2;
output3_tm[0] = sum3;

output0_tm++;
output1_tm++;
output2_tm++;
output3_tm++;
}
}
}

remain_outch_start += nn_outch << 2;
#else
int nn_outch = outch >> 1;
int remain_outch_start = nn_outch << 1;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 2;

float* output0_tm = top_blob_tm.channel(p);
float* output1_tm = top_blob_tm.channel(p + 1);

const Mat kernel0_tm = kernel_tm.channel(p / 2);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum01 = 0.f;
float sum02 = 0.f;
float sum03 = 0.f;
float sum10 = 0.f;
float sum11 = 0.f;
float sum12 = 0.f;
float sum13 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 8);
float w0 = k0[0];
float w1 = k0[1];
sum00 += r0[0] * w0;
sum01 += r0[1] * w0;
sum02 += r0[2] * w0;
sum03 += r0[3] * w0;
sum10 += r0[0] * w1;
sum11 += r0[1] * w1;
sum12 += r0[2] * w1;
sum13 += r0[3] * w1;

r0 += 4;
k0 += 2;
}

output0_tm[0] = sum00;
output0_tm[1] = sum01;
output0_tm[2] = sum02;
output0_tm[3] = sum03;
output1_tm[0] = sum10;
output1_tm[1] = sum11;
output1_tm[2] = sum12;
output1_tm[3] = sum13;

output0_tm += 4;
output1_tm += 4;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum00 = 0.f;
float sum10 = 0.f;

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 4);
__builtin_prefetch(k0 + 8);
float val0 = r0[0];
sum00 += val0 * k0[0];
sum10 += val0 * k0[1];

r0 += 1;
k0 += 2;
}

output0_tm[0] = sum00;
output1_tm[0] = sum10;
output0_tm++;
output1_tm++;
}
}
}
#endif

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_outch_start; p < outch; p++)
{
float* output0_tm = top_blob_tm.channel(p);

#if __mips_msa
const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2);
#endif

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

int j = 0;
#if __mips_msa
v4f32 _sum0 = (v4f32)__msa_fill_w(0);

for (; j < nn; j++)
{
_sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0));
r0 += 4;
k0++;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
output0_tm += 4;
#else // __mips_msa
float sum0 = 0.f;
float sum1 = 0.f;
float sum2 = 0.f;
float sum3 = 0.f;

for (; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 4);
float w0 = k0[0];
sum0 += r0[0] * w0;
sum1 += r0[1] * w0;
sum2 += r0[2] * w0;
sum3 += r0[3] * w0;

r0 += 4;
k0++;
}

output0_tm[0] = sum0;
output0_tm[1] = sum1;
output0_tm[2] = sum2;
output0_tm[3] = sum3;
output0_tm += 4;
#endif // __mips_msa
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 4 + i % 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch; // inch always > 0

float sum = 0.f;

for (int j = 0; j < nn; j++)
{
float w0 = k0[0];
float val0 = r0[0];
sum += val0 * w0;

r0 += 1;
k0 += 1;
}

output0_tm[0] = sum;
output0_tm += 1;
}
}
}
}

+ 448
- 0
src/layer/mips/convolution_winograd_dot_pack4.h View File

@@ -0,0 +1,448 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void convolution_winograd_dot_pack4_msa(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt)
{
// Mat bottom_blob_tm(tiles, 16/36/64, inch, 16u, 4, opt.workspace_allocator);

const int tiles = bottom_blob_tm.w;
const int batch = bottom_blob_tm.h;
const int inch = bottom_blob_tm.c;

// permute
Mat bottom_blob_tm2;
if (tiles >= 12)
bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, batch, 16u, 4, opt.workspace_allocator);
else if (tiles >= 8)
bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator);
else if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator);
else if (tiles >= 2)
bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator);
else // if (tiles >= 1)
bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 4, opt.workspace_allocator);

#pragma omp parallel for num_threads(opt.num_threads)
for (int r = 0; r < batch; r++)
{
Mat tm2 = bottom_blob_tm2.channel(r);

// tile
int i = 0;
for (; i + 11 < tiles; i += 12)
{
float* tmpptr = tm2.row(i / 12);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 4;

for (int q = 0; q < inch; q++)
{
// transpose 4x8
v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0);
v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0);
v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0);
v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0);
v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0);
v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0);
v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0);
v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0);
v4f32 _r8 = (v4f32)__msa_ld_w(r0 + 4 * 8, 0);
v4f32 _r9 = (v4f32)__msa_ld_w(r0 + 4 * 9, 0);
v4f32 _ra = (v4f32)__msa_ld_w(r0 + 4 * 10, 0);
v4f32 _rb = (v4f32)__msa_ld_w(r0 + 4 * 11, 0);

v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2);
v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2);
v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4);
v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4);
v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6);
v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6);
v4i32 _r89r = __msa_ilvr_w((v4i32)_r9, (v4i32)_r8);
v4i32 _r89l = __msa_ilvl_w((v4i32)_r9, (v4i32)_r8);
v4i32 _rabr = __msa_ilvr_w((v4i32)_rb, (v4i32)_ra);
v4i32 _rabl = __msa_ilvl_w((v4i32)_rb, (v4i32)_ra);
v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l);
v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l);
v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r);
v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r);
v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l);
v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l);
v2i64 _r89ab_0 = __msa_ilvr_d((v2i64)_rabr, (v2i64)_r89r);
v2i64 _r89ab_1 = __msa_ilvl_d((v2i64)_rabr, (v2i64)_r89r);
v2i64 _r89ab_2 = __msa_ilvr_d((v2i64)_rabl, (v2i64)_r89l);
v2i64 _r89ab_3 = __msa_ilvl_d((v2i64)_rabl, (v2i64)_r89l);

__msa_st_w((v4i32)_r0123_0, tmpptr, 0);
__msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0);
__msa_st_w((v4i32)_r89ab_0, tmpptr + 4 * 2, 0);
__msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 3, 0);
__msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 4, 0);
__msa_st_w((v4i32)_r89ab_1, tmpptr + 4 * 5, 0);
__msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 6, 0);
__msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 7, 0);
__msa_st_w((v4i32)_r89ab_2, tmpptr + 4 * 8, 0);
__msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 9, 0);
__msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 10, 0);
__msa_st_w((v4i32)_r89ab_3, tmpptr + 4 * 11, 0);

r0 += bottom_blob_tm.cstep * 4;
tmpptr += 48;
}
}
for (; i + 7 < tiles; i += 8)
{
float* tmpptr = tm2.row(i / 12 + (i % 12) / 8);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 4;

for (int q = 0; q < inch; q++)
{
// transpose 4x8
v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0);
v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0);
v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0);
v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0);
v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0);
v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0);
v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0);
v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0);

v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2);
v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2);
v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4);
v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4);
v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6);
v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6);
v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l);
v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l);
v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r);
v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r);
v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l);
v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l);

__msa_st_w((v4i32)_r0123_0, tmpptr, 0);
__msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0);
__msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 2, 0);
__msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 3, 0);
__msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 4, 0);
__msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 5, 0);
__msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 6, 0);
__msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 7, 0);

r0 += bottom_blob_tm.cstep * 4;
tmpptr += 32;
}
}
for (; i + 3 < tiles; i += 4)
{
float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 4;

for (int q = 0; q < inch; q++)
{
// transpose 4x4
v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0);
v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0);
v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0);
v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0);

v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2);
v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2);
v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r);
v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l);
v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l);

__msa_st_w((v4i32)_r0123_0, tmpptr, 0);
__msa_st_w((v4i32)_r0123_1, tmpptr + 4, 0);
__msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 2, 0);
__msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 3, 0);

r0 += bottom_blob_tm.cstep * 4;
tmpptr += 16;
}
}
for (; i + 1 < tiles; i += 2)
{
float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 4;

for (int q = 0; q < inch; q++)
{
// transpose 4x2
v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0);
v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0);

v4i32 _r01_0 = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0);
v4i32 _r01_1 = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0);

__msa_st_w((v4i32)_r01_0, tmpptr, 0);
__msa_st_w((v4i32)_r01_1, tmpptr + 4, 0);

r0 += bottom_blob_tm.cstep * 4;
tmpptr += 8;
}
}
for (; i < tiles; i++)
{
float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);

const float* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 4;

for (int q = 0; q < inch; q++)
{
v4f32 _val = (v4f32)__msa_ld_w(r0, 0);
__msa_st_w((v4i32)_val, tmpptr, 0);

r0 += bottom_blob_tm.cstep * 4;
tmpptr += 4;
}
}
}

bottom_blob_tm = Mat();
// permute end

top_blob_tm.create(tiles, batch, outch, 16u, 4, opt.workspace_allocator);

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
float* output0_tm = top_blob_tm.channel(p);

const Mat kernel0_tm = kernel_tm.channel(p);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
for (; i + 11 < tiles; i += 12)
{
const float* r0 = bb2.row(i / 12);
const float* k0 = kernel0_tm.row(r);

int nn = inch * 4; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);
v4f32 _sum4 = (v4f32)__msa_fill_w(0);
v4f32 _sum5 = (v4f32)__msa_fill_w(0);
v4f32 _sum6 = (v4f32)__msa_fill_w(0);
v4f32 _sum7 = (v4f32)__msa_fill_w(0);
v4f32 _sum8 = (v4f32)__msa_fill_w(0);
v4f32 _sum9 = (v4f32)__msa_fill_w(0);
v4f32 _suma = (v4f32)__msa_fill_w(0);
v4f32 _sumb = (v4f32)__msa_fill_w(0);

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 48);
__builtin_prefetch(k0 + 16);
v4i32 _val0123 = __msa_ld_w(r0, 0);
v4i32 _val4567 = __msa_ld_w(r0 + 4, 0);
v4i32 _val89ab = __msa_ld_w(r0 + 8, 0);
v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0);
_sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0);
_sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0);
_sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0);
_sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0);
_sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0);
_sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0);
_sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0);
_sum8 = __msa_fmadd_w(_sum8, (v4f32)__msa_splati_w(_val89ab, 0), _w0);
_sum9 = __msa_fmadd_w(_sum9, (v4f32)__msa_splati_w(_val89ab, 1), _w0);
_suma = __msa_fmadd_w(_suma, (v4f32)__msa_splati_w(_val89ab, 2), _w0);
_sumb = __msa_fmadd_w(_sumb, (v4f32)__msa_splati_w(_val89ab, 3), _w0);

r0 += 12;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output0_tm + 4, 0);
__msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0);
__msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0);
__msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0);
__msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0);
__msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0);
__msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0);
__msa_st_w((v4i32)_sum8, output0_tm + 4 * 8, 0);
__msa_st_w((v4i32)_sum9, output0_tm + 4 * 9, 0);
__msa_st_w((v4i32)_suma, output0_tm + 4 * 10, 0);
__msa_st_w((v4i32)_sumb, output0_tm + 4 * 11, 0);

output0_tm += 4 * 12;
}
for (; i + 7 < tiles; i += 8)
{
const float* r0 = bb2.row(i / 12 + (i % 12) / 8);
const float* k0 = kernel0_tm.row(r);

int nn = inch * 4; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);
v4f32 _sum4 = (v4f32)__msa_fill_w(0);
v4f32 _sum5 = (v4f32)__msa_fill_w(0);
v4f32 _sum6 = (v4f32)__msa_fill_w(0);
v4f32 _sum7 = (v4f32)__msa_fill_w(0);

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 32);
__builtin_prefetch(k0 + 16);
v4i32 _val0123 = __msa_ld_w(r0, 0);
v4i32 _val4567 = __msa_ld_w(r0 + 4, 0);
v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0);
_sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0);
_sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0);
_sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0);
_sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0);
_sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0);
_sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0);
_sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0);

r0 += 8;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output0_tm + 4, 0);
__msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0);
__msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0);
__msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0);
__msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0);
__msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0);
__msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0);

output0_tm += 4 * 8;
}
for (; i + 3 < tiles; i += 4)
{
const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4);
const float* k0 = kernel0_tm.row(r);

int nn = inch * 4; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);
v4f32 _sum2 = (v4f32)__msa_fill_w(0);
v4f32 _sum3 = (v4f32)__msa_fill_w(0);

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 16);
__builtin_prefetch(k0 + 16);
v4i32 _val0123 = __msa_ld_w(r0, 0);
v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0);
_sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0);
_sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0);
_sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0);

r0 += 4;
k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output0_tm + 4, 0);
__msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0);
__msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0);

output0_tm += 4 * 4;
}
for (; i + 1 < tiles; i += 2)
{
const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2);
const float* k0 = kernel0_tm.row(r);

int nn = inch * 4; // inch always > 0

v4f32 _sum0 = (v4f32)__msa_fill_w(0);
v4f32 _sum1 = (v4f32)__msa_fill_w(0);

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 8);
__builtin_prefetch(k0 + 16);
v4f32 _val0 = __msa_fill_w_f32(*r0++);
v4f32 _val1 = __msa_fill_w_f32(*r0++);
v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0);
_sum0 = __msa_fmadd_w(_sum0, _val0, _w0);
_sum1 = __msa_fmadd_w(_sum1, _val1, _w0);

k0 += 4;
}

__msa_st_w((v4i32)_sum0, output0_tm, 0);
__msa_st_w((v4i32)_sum1, output0_tm + 4, 0);

output0_tm += 4 * 2;
}
for (; i < tiles; i++)
{
const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2);
const float* k0 = kernel0_tm.row(r);

int nn = inch * 4; // inch always > 0

v4f32 _sum = (v4f32)__msa_fill_w(0);

for (int j = 0; j < nn; j++)
{
__builtin_prefetch(r0 + 4);
__builtin_prefetch(k0 + 16);
v4f32 _val0 = __msa_fill_w_f32(*r0++);
v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0);
_sum = __msa_fmadd_w(_sum, _val0, _w0);

k0 += 4;
}

__msa_st_w((v4i32)_sum, output0_tm, 0);

output0_tm += 4;
}
}
}
}

+ 170
- 0
src/layer/mips/convolution_winograd_transform_pack4.h View File

@@ -558,3 +558,173 @@ static void conv3x3s1_winograd43_transform_output_pack4_msa(const Mat& top_blob_
}
}
}

static void conv3x3s1_winograd23_transform_input_pack4_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
const int w = bottom_blob.w;
const int h = bottom_blob.h;
const int inch = bottom_blob.c;

const int w_tiles = (w - 2) / 2;
const int h_tiles = (h - 2) / 2;
const int tiles = w_tiles * h_tiles;

// const float itm[4][4] = {
// {1.0f, 0.0f, -1.0f, 0.0f},
// {0.0f, 1.0f, 1.00f, 0.0f},
// {0.0f, -1.0f, 1.00f, 0.0f},
// {0.0f, -1.0f, 0.00f, 1.0f}
// };

// 0 = r00 - r02
// 1 = r01 + r02
// 2 = r02 - r01
// 3 = r03 - r01

#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < inch; q++)
{
const Mat img0 = bottom_blob.channel(q);
Mat img0_tm = bottom_blob_tm.channel(q);

float tmp[4][4][4];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* r0 = img0.row(i * 2) + (j * 2) * 4;

for (int m = 0; m < 4; m++)
{
v4f32 _r00 = (v4f32)__msa_ld_w(r0, 0);
v4f32 _r01 = (v4f32)__msa_ld_w(r0 + 4, 0);
v4f32 _r02 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0);
v4f32 _r03 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0);

v4f32 _tmp0m = __msa_fsub_w(_r00, _r02);
v4f32 _tmp1m = __msa_fadd_w(_r01, _r02);
v4f32 _tmp2m = __msa_fsub_w(_r02, _r01);
v4f32 _tmp3m = __msa_fsub_w(_r03, _r01);

__msa_st_w((v4i32)_tmp0m, tmp[0][m], 0);
__msa_st_w((v4i32)_tmp1m, tmp[1][m], 0);
__msa_st_w((v4i32)_tmp2m, tmp[2][m], 0);
__msa_st_w((v4i32)_tmp3m, tmp[3][m], 0);

r0 += w * 4;
}

float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j) * 4;
float* r0_tm_1 = r0_tm_0 + tiles * 4;
float* r0_tm_2 = r0_tm_0 + tiles * 4 * 2;
float* r0_tm_3 = r0_tm_0 + tiles * 4 * 3;

for (int m = 0; m < 4; m++)
{
v4f32 _tmp00 = (v4f32)__msa_ld_w(tmp[m][0], 0);
v4f32 _tmp01 = (v4f32)__msa_ld_w(tmp[m][1], 0);
v4f32 _tmp02 = (v4f32)__msa_ld_w(tmp[m][2], 0);
v4f32 _tmp03 = (v4f32)__msa_ld_w(tmp[m][3], 0);

v4f32 _r0tm0 = __msa_fsub_w(_tmp00, _tmp02);
v4f32 _r0tm1 = __msa_fadd_w(_tmp01, _tmp02);
v4f32 _r0tm2 = __msa_fsub_w(_tmp02, _tmp01);
v4f32 _r0tm3 = __msa_fsub_w(_tmp03, _tmp01);

__msa_st_w((v4i32)_r0tm0, r0_tm_0, 0);
__msa_st_w((v4i32)_r0tm1, r0_tm_1, 0);
__msa_st_w((v4i32)_r0tm2, r0_tm_2, 0);
__msa_st_w((v4i32)_r0tm3, r0_tm_3, 0);

r0_tm_0 += tiles * 4 * 4;
r0_tm_1 += tiles * 4 * 4;
r0_tm_2 += tiles * 4 * 4;
r0_tm_3 += tiles * 4 * 4;
}
}
}
}
}

static void conv3x3s1_winograd23_transform_output_pack4_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt)
{
const int outw = top_blob.w;
const int outh = top_blob.h;
const int outch = top_blob.c;

const int w_tiles = outw / 2;
const int h_tiles = outh / 2;
const int tiles = w_tiles * h_tiles;

const float* biasptr = bias;

// const float otm[2][4] = {
// {1.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 1.0f}
// };

// 0 = r00 + r01 + r02
// 1 = r01 - r02 + r03

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
const Mat out0_tm = top_blob_tm.channel(p);
Mat out0 = top_blob.channel(p);

v4f32 _bias0 = biasptr ? (v4f32)__msa_ld_w(biasptr + p * 4, 0) : (v4f32)__msa_fill_w(0);

float tmp[2][4][4];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j) * 4;
const float* output0_tm_1 = output0_tm_0 + tiles * 4;
const float* output0_tm_2 = output0_tm_0 + tiles * 4 * 2;
const float* output0_tm_3 = output0_tm_0 + tiles * 4 * 3;

float* output0 = out0.row<float>(i * 2) + (j * 2) * 4;

for (int m = 0; m < 4; m++)
{
v4f32 _out0tm0 = (v4f32)__msa_ld_w(output0_tm_0, 0);
v4f32 _out0tm1 = (v4f32)__msa_ld_w(output0_tm_1, 0);
v4f32 _out0tm2 = (v4f32)__msa_ld_w(output0_tm_2, 0);
v4f32 _out0tm3 = (v4f32)__msa_ld_w(output0_tm_3, 0);

v4f32 _tmp0m = __msa_fadd_w(__msa_fadd_w(_out0tm0, _out0tm1), _out0tm2);
v4f32 _tmp1m = __msa_fadd_w(__msa_fsub_w(_out0tm1, _out0tm2), _out0tm3);

__msa_st_w((v4i32)_tmp0m, tmp[0][m], 0);
__msa_st_w((v4i32)_tmp1m, tmp[1][m], 0);

output0_tm_0 += tiles * 4 * 4;
output0_tm_1 += tiles * 4 * 4;
output0_tm_2 += tiles * 4 * 4;
output0_tm_3 += tiles * 4 * 4;
}

for (int m = 0; m < 2; m++)
{
v4f32 _tmp00 = (v4f32)__msa_ld_w(tmp[m][0], 0);
v4f32 _tmp01 = (v4f32)__msa_ld_w(tmp[m][1], 0);
v4f32 _tmp02 = (v4f32)__msa_ld_w(tmp[m][2], 0);
v4f32 _tmp03 = (v4f32)__msa_ld_w(tmp[m][3], 0);

v4f32 _out00 = __msa_fadd_w(_bias0, __msa_fadd_w(__msa_fadd_w(_tmp00, _tmp01), _tmp02));
v4f32 _out01 = __msa_fadd_w(_bias0, __msa_fadd_w(__msa_fsub_w(_tmp01, _tmp02), _tmp03));

__msa_st_w((v4i32)_out00, output0, 0);
__msa_st_w((v4i32)_out01, output0 + 4, 0);

output0 += outw * 4;
}
}
}
}
}

Write Preview
Loading…
Cancel
Save