diff --git a/src/layer/mips/convolution_3x3.h b/src/layer/mips/convolution_3x3.h index 5e4cc4be2..4b21d20f9 100644 --- a/src/layer/mips/convolution_3x3.h +++ b/src/layer/mips/convolution_3x3.h @@ -188,486 +188,7 @@ static void conv3x3s1_winograd23_msa(const Mat& bottom_blob, Mat& top_blob, cons // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 2 * 4; - int h_tm = outh / 2 * 4; - - const int tiles = h_tm / 4 * w_tm / 4; - - // permute - Mat bottom_blob_tm2; - if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 16, 4u, opt.workspace_allocator); - else - bottom_blob_tm2.create(1 * inch, tiles, 16, 4u, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 16; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; - for (; i + 3 < tiles; i += 4) - { - float* tmpptr = tm2.row(i / 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i); - - for (int q = 0; q < inch; q++) - { -#if __mips_msa - __msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0); -#else - tmpptr[0] = r0[0]; - tmpptr[1] = r0[1]; - tmpptr[2] = r0[2]; - tmpptr[3] = r0[3]; -#endif - - r0 += bottom_blob_tm.cstep; - tmpptr += 4; - } - } - for (; i < tiles; i++) - { - float* tmpptr = tm2.row(i / 4 + i % 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i); - - for (int q = 0; q < inch; q++) - { - tmpptr[0] = r0[0]; - - r0 += bottom_blob_tm.cstep; - tmpptr += 1; - } - } - } - - bottom_blob_tm = Mat(); - // permute end - - top_blob_tm.create(tiles, 16, outch, 4u, opt.workspace_allocator); - -#if __mips_msa - int nn_outch = outch >> 3; - int remain_outch_start = nn_outch << 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - float* output2_tm = top_blob_tm.channel(p + 2); - float* output3_tm = top_blob_tm.channel(p + 3); - float* output4_tm = top_blob_tm.channel(p + 4); - float* output5_tm = top_blob_tm.channel(p + 5); - float* output6_tm = top_blob_tm.channel(p + 6); - float* output7_tm = top_blob_tm.channel(p + 7); - - const Mat kernel0_tm = kernel_tm.channel(p / 8); - - for (int r = 0; r < 16; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - - int j = 0; - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 32); - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - v4i32 _w0123 = __msa_ld_w(k0, 0); - v4i32 _w4567 = __msa_ld_w(k0 + 4, 0); - _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); - _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); - _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); - _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); - _sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0)); - _sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1)); - _sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2)); - _sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3)); - - r0 += 4; - k0 += 8; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output1_tm, 0); - __msa_st_w((v4i32)_sum2, output2_tm, 0); - __msa_st_w((v4i32)_sum3, output3_tm, 0); - __msa_st_w((v4i32)_sum4, output4_tm, 0); - __msa_st_w((v4i32)_sum5, output5_tm, 0); - __msa_st_w((v4i32)_sum6, output6_tm, 0); - __msa_st_w((v4i32)_sum7, output7_tm, 0); - - output0_tm += 4; - output1_tm += 4; - output2_tm += 4; - output3_tm += 4; - output4_tm += 4; - output5_tm += 4; - output6_tm += 4; - output7_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - float sum4 = 0.f; - float sum5 = 0.f; - float sum6 = 0.f; - float sum7 = 0.f; - - int j = 0; - for (; j < nn; j++) - { - sum0 += r0[0] * k0[0]; - sum1 += r0[0] * k0[1]; - sum2 += r0[0] * k0[2]; - sum3 += r0[0] * k0[3]; - sum4 += r0[0] * k0[4]; - sum5 += r0[0] * k0[5]; - sum6 += r0[0] * k0[6]; - sum7 += r0[0] * k0[7]; - - r0 += 1; - k0 += 8; - } - - output0_tm[0] = sum0; - output1_tm[0] = sum1; - output2_tm[0] = sum2; - output3_tm[0] = sum3; - output4_tm[0] = sum4; - output5_tm[0] = sum5; - output6_tm[0] = sum6; - output7_tm[0] = sum7; - - output0_tm++; - output1_tm++; - output2_tm++; - output3_tm++; - output4_tm++; - output5_tm++; - output6_tm++; - output7_tm++; - } - } - } - - nn_outch = (outch - remain_outch_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - float* output2_tm = top_blob_tm.channel(p + 2); - float* output3_tm = top_blob_tm.channel(p + 3); - - const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); - - for (int r = 0; r < 16; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - - int j = 0; - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 16); - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - v4i32 _w0123 = __msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); - _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); - _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); - _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); - - r0 += 4; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output1_tm, 0); - __msa_st_w((v4i32)_sum2, output2_tm, 0); - __msa_st_w((v4i32)_sum3, output3_tm, 0); - - output0_tm += 4; - output1_tm += 4; - output2_tm += 4; - output3_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - - int j = 0; - for (; j < nn; j++) - { - sum0 += r0[0] * k0[0]; - sum1 += r0[0] * k0[1]; - sum2 += r0[0] * k0[2]; - sum3 += r0[0] * k0[3]; - - r0 += 1; - k0 += 4; - } - - output0_tm[0] = sum0; - output1_tm[0] = sum1; - output2_tm[0] = sum2; - output3_tm[0] = sum3; - - output0_tm++; - output1_tm++; - output2_tm++; - output3_tm++; - } - } - } - - remain_outch_start += nn_outch << 2; -#else - int nn_outch = outch >> 1; - int remain_outch_start = nn_outch << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 2; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - - const Mat kernel0_tm = kernel_tm.channel(p / 2); - - for (int r = 0; r < 16; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum00 = 0.f; - float sum01 = 0.f; - float sum02 = 0.f; - float sum03 = 0.f; - float sum10 = 0.f; - float sum11 = 0.f; - float sum12 = 0.f; - float sum13 = 0.f; - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 8); - float w0 = k0[0]; - float w1 = k0[1]; - sum00 += r0[0] * w0; - sum01 += r0[1] * w0; - sum02 += r0[2] * w0; - sum03 += r0[3] * w0; - sum10 += r0[0] * w1; - sum11 += r0[1] * w1; - sum12 += r0[2] * w1; - sum13 += r0[3] * w1; - - r0 += 4; - k0 += 2; - } - - output0_tm[0] = sum00; - output0_tm[1] = sum01; - output0_tm[2] = sum02; - output0_tm[3] = sum03; - output1_tm[0] = sum10; - output1_tm[1] = sum11; - output1_tm[2] = sum12; - output1_tm[3] = sum13; - - output0_tm += 4; - output1_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum00 = 0.f; - float sum10 = 0.f; - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 4); - __builtin_prefetch(k0 + 8); - float val0 = r0[0]; - sum00 += val0 * k0[0]; - sum10 += val0 * k0[1]; - - r0 += 1; - k0 += 2; - } - - output0_tm[0] = sum00; - output1_tm[0] = sum10; - output0_tm++; - output1_tm++; - } - } - } -#endif - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - float* output0_tm = top_blob_tm.channel(p); - -#if __mips_msa - const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); -#endif - - for (int r = 0; r < 16; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int j = 0; -#if __mips_msa - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - - for (; j < nn; j++) - { - _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0)); - r0 += 4; - k0++; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - output0_tm += 4; -#else // __mips_msa - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 4); - float w0 = k0[0]; - sum0 += r0[0] * w0; - sum1 += r0[1] * w0; - sum2 += r0[2] * w0; - sum3 += r0[3] * w0; - - r0 += 4; - k0++; - } - - output0_tm[0] = sum0; - output0_tm[1] = sum1; - output0_tm[2] = sum2; - output0_tm[3] = sum3; - output0_tm += 4; -#endif // __mips_msa - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum = 0.f; - - for (int j = 0; j < nn; j++) - { - float w0 = k0[0]; - float val0 = r0[0]; - sum += val0 * w0; - - r0 += 1; - k0 += 1; - } - - output0_tm[0] = sum; - output0_tm += 1; - } - } - } - } - bottom_blob_tm = Mat(); + convolution_winograd_dot_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output @@ -868,486 +389,7 @@ static void conv3x3s1_winograd43_msa(const Mat& bottom_blob, Mat& top_blob, cons // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = h_tm / 6 * w_tm / 6; - - // permute - Mat bottom_blob_tm2; - if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 36, 4u, opt.workspace_allocator); - else - bottom_blob_tm2.create(1 * inch, tiles, 36, 4u, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 36; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; - for (; i + 3 < tiles; i += 4) - { - float* tmpptr = tm2.row(i / 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i); - - for (int q = 0; q < inch; q++) - { -#if __mips_msa - __msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0); -#else - tmpptr[0] = r0[0]; - tmpptr[1] = r0[1]; - tmpptr[2] = r0[2]; - tmpptr[3] = r0[3]; -#endif - - r0 += bottom_blob_tm.cstep; - tmpptr += 4; - } - } - for (; i < tiles; i++) - { - float* tmpptr = tm2.row(i / 4 + i % 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i); - - for (int q = 0; q < inch; q++) - { - tmpptr[0] = r0[0]; - - r0 += bottom_blob_tm.cstep; - tmpptr += 1; - } - } - } - - bottom_blob_tm = Mat(); - // permute end - - top_blob_tm.create(tiles, 36, outch, 4u, opt.workspace_allocator); - -#if __mips_msa - int nn_outch = outch >> 3; - int remain_outch_start = nn_outch << 3; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 8; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - float* output2_tm = top_blob_tm.channel(p + 2); - float* output3_tm = top_blob_tm.channel(p + 3); - float* output4_tm = top_blob_tm.channel(p + 4); - float* output5_tm = top_blob_tm.channel(p + 5); - float* output6_tm = top_blob_tm.channel(p + 6); - float* output7_tm = top_blob_tm.channel(p + 7); - - const Mat kernel0_tm = kernel_tm.channel(p / 8); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - - int j = 0; - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 32); - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - v4i32 _w0123 = __msa_ld_w(k0, 0); - v4i32 _w4567 = __msa_ld_w(k0 + 4, 0); - _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); - _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); - _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); - _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); - _sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0)); - _sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1)); - _sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2)); - _sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3)); - - r0 += 4; - k0 += 8; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output1_tm, 0); - __msa_st_w((v4i32)_sum2, output2_tm, 0); - __msa_st_w((v4i32)_sum3, output3_tm, 0); - __msa_st_w((v4i32)_sum4, output4_tm, 0); - __msa_st_w((v4i32)_sum5, output5_tm, 0); - __msa_st_w((v4i32)_sum6, output6_tm, 0); - __msa_st_w((v4i32)_sum7, output7_tm, 0); - - output0_tm += 4; - output1_tm += 4; - output2_tm += 4; - output3_tm += 4; - output4_tm += 4; - output5_tm += 4; - output6_tm += 4; - output7_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - float sum4 = 0.f; - float sum5 = 0.f; - float sum6 = 0.f; - float sum7 = 0.f; - - int j = 0; - for (; j < nn; j++) - { - sum0 += r0[0] * k0[0]; - sum1 += r0[0] * k0[1]; - sum2 += r0[0] * k0[2]; - sum3 += r0[0] * k0[3]; - sum4 += r0[0] * k0[4]; - sum5 += r0[0] * k0[5]; - sum6 += r0[0] * k0[6]; - sum7 += r0[0] * k0[7]; - - r0 += 1; - k0 += 8; - } - - output0_tm[0] = sum0; - output1_tm[0] = sum1; - output2_tm[0] = sum2; - output3_tm[0] = sum3; - output4_tm[0] = sum4; - output5_tm[0] = sum5; - output6_tm[0] = sum6; - output7_tm[0] = sum7; - - output0_tm++; - output1_tm++; - output2_tm++; - output3_tm++; - output4_tm++; - output5_tm++; - output6_tm++; - output7_tm++; - } - } - } - - nn_outch = (outch - remain_outch_start) >> 2; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = remain_outch_start + pp * 4; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - float* output2_tm = top_blob_tm.channel(p + 2); - float* output3_tm = top_blob_tm.channel(p + 3); - - const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - - int j = 0; - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 16); - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - v4i32 _w0123 = __msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); - _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); - _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); - _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); - - r0 += 4; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output1_tm, 0); - __msa_st_w((v4i32)_sum2, output2_tm, 0); - __msa_st_w((v4i32)_sum3, output3_tm, 0); - - output0_tm += 4; - output1_tm += 4; - output2_tm += 4; - output3_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - - int j = 0; - for (; j < nn; j++) - { - sum0 += r0[0] * k0[0]; - sum1 += r0[0] * k0[1]; - sum2 += r0[0] * k0[2]; - sum3 += r0[0] * k0[3]; - - r0 += 1; - k0 += 4; - } - - output0_tm[0] = sum0; - output1_tm[0] = sum1; - output2_tm[0] = sum2; - output3_tm[0] = sum3; - - output0_tm++; - output1_tm++; - output2_tm++; - output3_tm++; - } - } - } - - remain_outch_start += nn_outch << 2; -#else - int nn_outch = outch >> 1; - int remain_outch_start = nn_outch << 1; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_outch; pp++) - { - int p = pp * 2; - - float* output0_tm = top_blob_tm.channel(p); - float* output1_tm = top_blob_tm.channel(p + 1); - - const Mat kernel0_tm = kernel_tm.channel(p / 2); - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum00 = 0.f; - float sum01 = 0.f; - float sum02 = 0.f; - float sum03 = 0.f; - float sum10 = 0.f; - float sum11 = 0.f; - float sum12 = 0.f; - float sum13 = 0.f; - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 8); - float w0 = k0[0]; - float w1 = k0[1]; - sum00 += r0[0] * w0; - sum01 += r0[1] * w0; - sum02 += r0[2] * w0; - sum03 += r0[3] * w0; - sum10 += r0[0] * w1; - sum11 += r0[1] * w1; - sum12 += r0[2] * w1; - sum13 += r0[3] * w1; - - r0 += 4; - k0 += 2; - } - - output0_tm[0] = sum00; - output0_tm[1] = sum01; - output0_tm[2] = sum02; - output0_tm[3] = sum03; - output1_tm[0] = sum10; - output1_tm[1] = sum11; - output1_tm[2] = sum12; - output1_tm[3] = sum13; - - output0_tm += 4; - output1_tm += 4; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum00 = 0.f; - float sum10 = 0.f; - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 4); - __builtin_prefetch(k0 + 8); - float val0 = r0[0]; - sum00 += val0 * k0[0]; - sum10 += val0 * k0[1]; - - r0 += 1; - k0 += 2; - } - - output0_tm[0] = sum00; - output1_tm[0] = sum10; - output0_tm++; - output1_tm++; - } - } - } -#endif - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) - { - float* output0_tm = top_blob_tm.channel(p); - -#if __mips_msa - const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); -#else - const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); -#endif - - for (int r = 0; r < 36; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - int j = 0; -#if __mips_msa - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - - for (; j < nn; j++) - { - _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0)); - r0 += 4; - k0++; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - output0_tm += 4; -#else // __mips_msa - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; - - for (; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 4); - float w0 = k0[0]; - sum0 += r0[0] * w0; - sum1 += r0[1] * w0; - sum2 += r0[2] * w0; - sum3 += r0[3] * w0; - - r0 += 4; - k0++; - } - - output0_tm[0] = sum0; - output0_tm[1] = sum1; - output0_tm[2] = sum2; - output0_tm[3] = sum3; - output0_tm += 4; -#endif // __mips_msa - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 4 + i % 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch; // inch always > 0 - - float sum = 0.f; - - for (int j = 0; j < nn; j++) - { - float w0 = k0[0]; - float val0 = r0[0]; - sum += val0 * w0; - - r0 += 1; - k0 += 1; - } - - output0_tm[0] = sum; - output0_tm += 1; - } - } - } - } - bottom_blob_tm = Mat(); + convolution_winograd_dot_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output diff --git a/src/layer/mips/convolution_3x3_pack4.h b/src/layer/mips/convolution_3x3_pack4.h index 8c0873458..2f257f19f 100644 --- a/src/layer/mips/convolution_3x3_pack4.h +++ b/src/layer/mips/convolution_3x3_pack4.h @@ -69,22 +69,22 @@ static void conv3x3s1_winograd63_transform_kernel_pack4_msa(const Mat& kernel, M // dst = pb-pa-inch/pa-64-outch/pb kernel_tm_pack4.create(inch / 4, 64, outch / 4, (size_t)4u * 4 * 4, 4 * 4); - for (int q = 0; q + (4 - 1) < outch; q += 4) + for (int q = 0; q + 3 < outch; q += 4) { Mat g0 = kernel_tm_pack4.channel(q / 4); for (int k = 0; k < 64; k++) { - float* g00 = g0.row(k); + float* g00 = g0.row(k); - for (int p = 0; p + (4 - 1) < inch; p += 4) + for (int p = 0; p + 3 < inch; p += 4) { for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { const float* k00 = kernel_tm.channel(q + j).row(p + i); - g00[0] = (float)k00[k]; + g00[0] = k00[k]; g00++; } } @@ -130,440 +130,7 @@ static void conv3x3s1_winograd63_pack4_msa(const Mat& bottom_blob, Mat& top_blob // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 6 * 8; - int h_tm = outh / 6 * 8; - - const int tiles = h_tm / 8 * w_tm / 8; - - // permute - // bottom_blob_tm.create(tiles, 64, inch, elemsize, elempack, opt.workspace_allocator); - Mat bottom_blob_tm2; - if (tiles >= 12) - bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, 64, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 8) - bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 2) - bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 64, 4u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 64, 4u * elempack, elempack, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 64; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; - for (; i + 11 < tiles; i += 12) - { - float* tmpptr = tm2.row(i / 12); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x8 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); - v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); - v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); - v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); - v4f32 _r8 = (v4f32)__msa_ld_w(r0 + 4 * 8, 0); - v4f32 _r9 = (v4f32)__msa_ld_w(r0 + 4 * 9, 0); - v4f32 _ra = (v4f32)__msa_ld_w(r0 + 4 * 10, 0); - v4f32 _rb = (v4f32)__msa_ld_w(r0 + 4 * 11, 0); - - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r89r = __msa_ilvr_w((v4i32)_r9, (v4i32)_r8); - v4i32 _r89l = __msa_ilvl_w((v4i32)_r9, (v4i32)_r8); - v4i32 _rabr = __msa_ilvr_w((v4i32)_rb, (v4i32)_ra); - v4i32 _rabl = __msa_ilvl_w((v4i32)_rb, (v4i32)_ra); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r89ab_0 = __msa_ilvr_d((v2i64)_rabr, (v2i64)_r89r); - v2i64 _r89ab_1 = __msa_ilvl_d((v2i64)_rabr, (v2i64)_r89r); - v2i64 _r89ab_2 = __msa_ilvr_d((v2i64)_rabl, (v2i64)_r89l); - v2i64 _r89ab_3 = __msa_ilvl_d((v2i64)_rabl, (v2i64)_r89l); - - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); - __msa_st_w((v4i32)_r89ab_0, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 3, 0); - __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 4, 0); - __msa_st_w((v4i32)_r89ab_1, tmpptr + 4 * 5, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 6, 0); - __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 7, 0); - __msa_st_w((v4i32)_r89ab_2, tmpptr + 4 * 8, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 9, 0); - __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 10, 0); - __msa_st_w((v4i32)_r89ab_3, tmpptr + 4 * 11, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 48; - } - } - for (; i + 7 < tiles; i += 8) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x8 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); - v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); - v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); - v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); - - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); - - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 3, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 4, 0); - __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 5, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 6, 0); - __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 7, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 32; - } - } - for (; i + 3 < tiles; i += 4) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x4 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 3, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 16; - } - } - for (; i + 1 < tiles; i += 2) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x2 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - - v4i32 _r01_0 = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01_1 = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - - __msa_st_w((v4i32)_r01_0, tmpptr, 0); - __msa_st_w((v4i32)_r01_1, tmpptr + 4, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 8; - } - } - for (; i < tiles; i++) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - __msa_st_w((v4i32)_val, tmpptr, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 4; - } - } - } - - bottom_blob_tm = Mat(); - // permute end - - top_blob_tm.create(tiles, 64, outch, 4u * elempack, elempack, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) - { - float* output0_tm = top_blob_tm.channel(p); - - const Mat kernel0_tm = kernel_tm.channel(p); - - for (int r = 0; r < 64; r++) - { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 11 < tiles; i += 12) - { - const float* r0 = bb2.row(i / 12); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - v4f32 _sum8 = (v4f32)__msa_fill_w(0); - v4f32 _sum9 = (v4f32)__msa_fill_w(0); - v4f32 _suma = (v4f32)__msa_fill_w(0); - v4f32 _sumb = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 48); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); - v4i32 _val89ab = __msa_ld_w(r0 + 8, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); - _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); - _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); - _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); - _sum8 = __msa_fmadd_w(_sum8, (v4f32)__msa_splati_w(_val89ab, 0), _w0); - _sum9 = __msa_fmadd_w(_sum9, (v4f32)__msa_splati_w(_val89ab, 1), _w0); - _suma = __msa_fmadd_w(_suma, (v4f32)__msa_splati_w(_val89ab, 2), _w0); - _sumb = __msa_fmadd_w(_sumb, (v4f32)__msa_splati_w(_val89ab, 3), _w0); - - r0 += 12; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); - __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); - __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); - __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); - __msa_st_w((v4i32)_sum8, output0_tm + 4 * 8, 0); - __msa_st_w((v4i32)_sum9, output0_tm + 4 * 9, 0); - __msa_st_w((v4i32)_suma, output0_tm + 4 * 10, 0); - __msa_st_w((v4i32)_sumb, output0_tm + 4 * 11, 0); - - output0_tm += 4 * 12; - } - for (; i + 7 < tiles; i += 8) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 32); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); - _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); - _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); - _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); - - r0 += 8; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); - __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); - __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); - __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); - - output0_tm += 4 * 8; - } - for (; i + 3 < tiles; i += 4) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - - r0 += 4; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - - output0_tm += 4 * 4; - } - for (; i + 1 < tiles; i += 2) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 8); - __builtin_prefetch(k0 + 16); - v4f32 _val0 = __msa_fill_w_f32(*r0++); - v4f32 _val1 = __msa_fill_w_f32(*r0++); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, _val0, _w0); - _sum1 = __msa_fmadd_w(_sum1, _val1, _w0); - - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - - output0_tm += 4 * 2; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 4); - __builtin_prefetch(k0 + 16); - v4f32 _val0 = __msa_fill_w_f32(*r0++); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum = __msa_fmadd_w(_sum, _val0, _w0); - - k0 += 4; - } - - __msa_st_w((v4i32)_sum, output0_tm, 0); - - output0_tm += 4; - } - } - } - } - bottom_blob_tm = Mat(); + convolution_winograd_dot_pack4_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output @@ -639,22 +206,22 @@ static void conv3x3s1_winograd43_transform_kernel_pack4_msa(const Mat& kernel, M // dst = pb-pa-inch/pa-36-outch/pb kernel_tm_pack4.create(inch / 4, 36, outch / 4, (size_t)4u * 4 * 4, 4 * 4); - for (int q = 0; q + (4 - 1) < outch; q += 4) + for (int q = 0; q + 3 < outch; q += 4) { Mat g0 = kernel_tm_pack4.channel(q / 4); for (int k = 0; k < 36; k++) { - float* g00 = g0.row(k); + float* g00 = g0.row(k); - for (int p = 0; p + (4 - 1) < inch; p += 4) + for (int p = 0; p + 3 < inch; p += 4) { for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { const float* k00 = kernel_tm.channel(q + j).row(p + i); - g00[0] = (float)k00[k]; + g00[0] = k00[k]; g00++; } } @@ -700,440 +267,142 @@ static void conv3x3s1_winograd43_pack4_msa(const Mat& bottom_blob, Mat& top_blob // BEGIN dot Mat top_blob_tm; - { - int w_tm = outw / 4 * 6; - int h_tm = outh / 4 * 6; - - const int tiles = h_tm / 6 * w_tm / 6; - - // permute - // bottom_blob_tm.create(tiles, 36, inch, elemsize, elempack, opt.workspace_allocator); - Mat bottom_blob_tm2; - if (tiles >= 12) - bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, 36, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 8) - bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, 36, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 4) - bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, 36, 4u * elempack, elempack, opt.workspace_allocator); - else if (tiles >= 2) - bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, 36, 4u * elempack, elempack, opt.workspace_allocator); - else // if (tiles >= 1) - bottom_blob_tm2.create(1 * inch, tiles, 36, 4u * elempack, elempack, opt.workspace_allocator); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int r = 0; r < 36; r++) - { - Mat tm2 = bottom_blob_tm2.channel(r); - - // tile - int i = 0; - for (; i + 11 < tiles; i += 12) - { - float* tmpptr = tm2.row(i / 12); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x8 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); - v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); - v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); - v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); - v4f32 _r8 = (v4f32)__msa_ld_w(r0 + 4 * 8, 0); - v4f32 _r9 = (v4f32)__msa_ld_w(r0 + 4 * 9, 0); - v4f32 _ra = (v4f32)__msa_ld_w(r0 + 4 * 10, 0); - v4f32 _rb = (v4f32)__msa_ld_w(r0 + 4 * 11, 0); - - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r89r = __msa_ilvr_w((v4i32)_r9, (v4i32)_r8); - v4i32 _r89l = __msa_ilvl_w((v4i32)_r9, (v4i32)_r8); - v4i32 _rabr = __msa_ilvr_w((v4i32)_rb, (v4i32)_ra); - v4i32 _rabl = __msa_ilvl_w((v4i32)_rb, (v4i32)_ra); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r89ab_0 = __msa_ilvr_d((v2i64)_rabr, (v2i64)_r89r); - v2i64 _r89ab_1 = __msa_ilvl_d((v2i64)_rabr, (v2i64)_r89r); - v2i64 _r89ab_2 = __msa_ilvr_d((v2i64)_rabl, (v2i64)_r89l); - v2i64 _r89ab_3 = __msa_ilvl_d((v2i64)_rabl, (v2i64)_r89l); - - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); - __msa_st_w((v4i32)_r89ab_0, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 3, 0); - __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 4, 0); - __msa_st_w((v4i32)_r89ab_1, tmpptr + 4 * 5, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 6, 0); - __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 7, 0); - __msa_st_w((v4i32)_r89ab_2, tmpptr + 4 * 8, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 9, 0); - __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 10, 0); - __msa_st_w((v4i32)_r89ab_3, tmpptr + 4 * 11, 0); + convolution_winograd_dot_pack4_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); + // END dot - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 48; - } - } - for (; i + 7 < tiles; i += 8) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8); + // BEGIN transform output + Mat top_blob_bordered; + if (outw == top_blob.w && outh == top_blob.h) + { + top_blob_bordered = top_blob; + } + else + { + top_blob_bordered.create(outw, outh, outch, elemsize, elempack, opt.workspace_allocator); + } + { + conv3x3s1_winograd43_transform_output_pack4_msa(top_blob_tm, top_blob_bordered, bias, opt); + } + // END transform output - const float* r0 = bottom_blob_tm; + // cut result pad + copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt); +} - r0 += (r * tiles + i) * 4; +static void conv3x3s1_winograd23_transform_kernel_pack4_msa(const Mat& kernel, Mat& kernel_tm_pack4, int inch, int outch, const Option& opt) +{ + // winograd23 transform kernel + Mat kernel_tm(4 * 4, inch, outch); - for (int q = 0; q < inch; q++) - { - // transpose 4x8 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); - v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); - v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); - v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); + const float ktm[4][3] = { + {1.0f, 0.0f, 0.0f}, + {1.0f / 2, 1.0f / 2, 1.0f / 2}, + {1.0f / 2, -1.0f / 2, 1.0f / 2}, + {0.0f, 0.0f, 1.0f} + }; - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); - v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); - v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); - v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); - v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + for (int q = 0; q < inch; q++) + { + const float* kernel0 = (const float*)kernel + p * inch * 9 + q * 9; + float* kernel_tm0 = kernel_tm.channel(p).row(q); - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 3, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 4, 0); - __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 5, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 6, 0); - __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 7, 0); + // transform kernel + const float* k0 = kernel0; + const float* k1 = kernel0 + 3; + const float* k2 = kernel0 + 6; - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 32; - } - } - for (; i + 3 < tiles; i += 4) + // h + float tmp[4][3]; + for (int i = 0; i < 4; i++) { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x4 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); - v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); - - v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); - v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); - v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); - v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); - v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); - - __msa_st_w((v4i32)_r0123_0, tmpptr, 0); - __msa_st_w((v4i32)_r0123_1, tmpptr + 4, 0); - __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 2, 0); - __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 3, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 16; - } + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; } - for (; i + 1 < tiles; i += 2) - { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; - - for (int q = 0; q < inch; q++) - { - // transpose 4x2 - v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); - v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); - - v4i32 _r01_0 = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4i32 _r01_1 = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - __msa_st_w((v4i32)_r01_0, tmpptr, 0); - __msa_st_w((v4i32)_r01_1, tmpptr + 4, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 8; - } - } - for (; i < tiles; i++) + // U + for (int j = 0; j < 4; j++) { - float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - - const float* r0 = bottom_blob_tm; - - r0 += (r * tiles + i) * 4; + float* tmpp = &tmp[j][0]; - for (int q = 0; q < inch; q++) + for (int i = 0; i < 4; i++) { - v4f32 _val = (v4f32)__msa_ld_w(r0, 0); - __msa_st_w((v4i32)_val, tmpptr, 0); - - r0 += bottom_blob_tm.cstep * 4; - tmpptr += 4; + kernel_tm0[j * 4 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; } } } + } - bottom_blob_tm = Mat(); - // permute end + // interleave + // src = 16-inch-outch + // dst = pb-pa-inch/pa-16-outch/pb + kernel_tm_pack4.create(inch / 4, 16, outch / 4, (size_t)4u * 4 * 4, 4 * 4); - top_blob_tm.create(tiles, 36, outch, 4u * elempack, elempack, opt.workspace_allocator); + for (int q = 0; q + 3 < outch; q += 4) + { + Mat g0 = kernel_tm_pack4.channel(q / 4); - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) + for (int k = 0; k < 16; k++) { - float* output0_tm = top_blob_tm.channel(p); + float* g00 = g0.row(k); - const Mat kernel0_tm = kernel_tm.channel(p); - - for (int r = 0; r < 36; r++) + for (int p = 0; p + 3 < inch; p += 4) { - const Mat bb2 = bottom_blob_tm2.channel(r); - - int i = 0; - for (; i + 11 < tiles; i += 12) - { - const float* r0 = bb2.row(i / 12); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - v4f32 _sum8 = (v4f32)__msa_fill_w(0); - v4f32 _sum9 = (v4f32)__msa_fill_w(0); - v4f32 _suma = (v4f32)__msa_fill_w(0); - v4f32 _sumb = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 48); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); - v4i32 _val89ab = __msa_ld_w(r0 + 8, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); - _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); - _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); - _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); - _sum8 = __msa_fmadd_w(_sum8, (v4f32)__msa_splati_w(_val89ab, 0), _w0); - _sum9 = __msa_fmadd_w(_sum9, (v4f32)__msa_splati_w(_val89ab, 1), _w0); - _suma = __msa_fmadd_w(_suma, (v4f32)__msa_splati_w(_val89ab, 2), _w0); - _sumb = __msa_fmadd_w(_sumb, (v4f32)__msa_splati_w(_val89ab, 3), _w0); - - r0 += 12; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); - __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); - __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); - __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); - __msa_st_w((v4i32)_sum8, output0_tm + 4 * 8, 0); - __msa_st_w((v4i32)_sum9, output0_tm + 4 * 9, 0); - __msa_st_w((v4i32)_suma, output0_tm + 4 * 10, 0); - __msa_st_w((v4i32)_sumb, output0_tm + 4 * 11, 0); - - output0_tm += 4 * 12; - } - for (; i + 7 < tiles; i += 8) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - v4f32 _sum4 = (v4f32)__msa_fill_w(0); - v4f32 _sum5 = (v4f32)__msa_fill_w(0); - v4f32 _sum6 = (v4f32)__msa_fill_w(0); - v4f32 _sum7 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 32); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); - _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); - _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); - _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); - - r0 += 8; - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); - __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); - __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); - __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); - - output0_tm += 4 * 8; - } - for (; i + 3 < tiles; i += 4) + for (int i = 0; i < 4; i++) { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - v4f32 _sum2 = (v4f32)__msa_fill_w(0); - v4f32 _sum3 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) + for (int j = 0; j < 4; j++) { - __builtin_prefetch(r0 + 16); - __builtin_prefetch(k0 + 16); - v4i32 _val0123 = __msa_ld_w(r0, 0); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); - _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); - _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); - _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); - - r0 += 4; - k0 += 4; + const float* k00 = kernel_tm.channel(q + j).row(p + i); + g00[0] = k00[k]; + g00++; } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); - __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); - __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); - - output0_tm += 4 * 4; } - for (; i + 1 < tiles; i += 2) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); - const float* k0 = kernel0_tm.row(r); - - int nn = inch * 4; // inch always > 0 - - v4f32 _sum0 = (v4f32)__msa_fill_w(0); - v4f32 _sum1 = (v4f32)__msa_fill_w(0); - - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 8); - __builtin_prefetch(k0 + 16); - v4f32 _val0 = __msa_fill_w_f32(*r0++); - v4f32 _val1 = __msa_fill_w_f32(*r0++); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum0 = __msa_fmadd_w(_sum0, _val0, _w0); - _sum1 = __msa_fmadd_w(_sum1, _val1, _w0); - - k0 += 4; - } - - __msa_st_w((v4i32)_sum0, output0_tm, 0); - __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); + } + } + } +} - output0_tm += 4 * 2; - } - for (; i < tiles; i++) - { - const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); - const float* k0 = kernel0_tm.row(r); +static void conv3x3s1_winograd23_pack4_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int inch = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; - int nn = inch * 4; // inch always > 0 + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; - v4f32 _sum = (v4f32)__msa_fill_w(0); + // pad to 2n+2 + Mat bottom_blob_bordered = bottom_blob; - for (int j = 0; j < nn; j++) - { - __builtin_prefetch(r0 + 4); - __builtin_prefetch(k0 + 16); - v4f32 _val0 = __msa_fill_w_f32(*r0++); - v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); - _sum = __msa_fmadd_w(_sum, _val0, _w0); + outw = (outw + 1) / 2 * 2; + outh = (outh + 1) / 2 * 2; - k0 += 4; - } + w = outw + 2; + h = outh + 2; + copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt); - __msa_st_w((v4i32)_sum, output0_tm, 0); + // BEGIN transform input + Mat bottom_blob_tm; + { + int w_tiles = outw / 2; + int h_tiles = outh / 2; + const int tiles = w_tiles * h_tiles; - output0_tm += 4; - } - } - } + bottom_blob_tm.create(tiles, 16, inch, elemsize, elempack, opt.workspace_allocator); + conv3x3s1_winograd23_transform_input_pack4_msa(bottom_blob_bordered, bottom_blob_tm, opt); } - bottom_blob_tm = Mat(); + bottom_blob_bordered = Mat(); + // END transform input + + // BEGIN dot + Mat top_blob_tm; + convolution_winograd_dot_pack4_msa(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt); // END dot // BEGIN transform output @@ -1147,7 +416,7 @@ static void conv3x3s1_winograd43_pack4_msa(const Mat& bottom_blob, Mat& top_blob top_blob_bordered.create(outw, outh, outch, elemsize, elempack, opt.workspace_allocator); } { - conv3x3s1_winograd43_transform_output_pack4_msa(top_blob_tm, top_blob_bordered, bias, opt); + conv3x3s1_winograd23_transform_output_pack4_msa(top_blob_tm, top_blob_bordered, bias, opt); } // END transform output diff --git a/src/layer/mips/convolution_mips.cpp b/src/layer/mips/convolution_mips.cpp index 5acb933f4..73bce37a1 100644 --- a/src/layer/mips/convolution_mips.cpp +++ b/src/layer/mips/convolution_mips.cpp @@ -31,6 +31,7 @@ namespace ncnn { #include "convolution_sgemm.h" #include "convolution_winograd_transform.h" +#include "convolution_winograd_dot.h" #include "convolution_1x1.h" #include "convolution_3x3.h" @@ -48,6 +49,7 @@ namespace ncnn { #include "convolution_sgemm_pack4.h" #include "convolution_sgemm_pack4to1.h" #include "convolution_winograd_transform_pack4.h" +#include "convolution_winograd_dot_pack4.h" #include "convolution_1x1_pack4.h" #include "convolution_1x1_pack4to1.h" #include "convolution_3x3_pack4.h" @@ -145,12 +147,14 @@ int Convolution_mips::create_pipeline(const Option& opt) // pack4 if (elempack == 4 && out_elempack == 4) { - if (opt.use_winograd_convolution && (opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) + if (opt.use_winograd_convolution && (opt.use_winograd23_convolution || opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - if (opt.use_winograd63_convolution) + if ((opt.use_winograd63_convolution && num_input >= 8 && num_output >= 8 && num_input <= 64 && num_output <= 64) || (!opt.use_winograd43_convolution && !opt.use_winograd23_convolution)) conv3x3s1_winograd63_transform_kernel_pack4_msa(weight_data, weight_winograd63_data, num_input, num_output, opt); - if (opt.use_winograd43_convolution) + else if ((opt.use_winograd43_convolution && num_input >= 8 && num_output >= 8) || (!opt.use_winograd63_convolution && !opt.use_winograd23_convolution)) conv3x3s1_winograd43_transform_kernel_pack4_msa(weight_data, weight_winograd43_data, num_input, num_output, opt); + else // if (opt.use_winograd23_convolution) + conv3x3s1_winograd23_transform_kernel_pack4_msa(weight_data, weight_winograd23_data, num_input, num_output, opt); } else { @@ -340,17 +344,14 @@ int Convolution_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Optio activation->forward_inplace(top_blob, opt); } } - else if (opt.use_winograd_convolution && (opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) + else if (opt.use_winograd_convolution && (opt.use_winograd23_convolution || opt.use_winograd43_convolution || opt.use_winograd63_convolution) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - // we need more proper conditions - if ((opt.use_winograd43_convolution && (w <= 10 || (w >= 15 && w <= 18) || w == 21 || w == 22) && (h <= 10 || (h >= 15 && h <= 18) || h == 21 || h == 22)) || !opt.use_winograd63_convolution) - { - conv3x3s1_winograd43_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt); - } - else - { + if ((opt.use_winograd63_convolution && num_input >= 8 && num_output >= 8 && num_input <= 64 && num_output <= 64) || (!opt.use_winograd43_convolution && !opt.use_winograd23_convolution)) conv3x3s1_winograd63_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd63_data, bias_data, opt); - } + else if ((opt.use_winograd43_convolution && num_input >= 8 && num_output >= 8) || (!opt.use_winograd63_convolution && !opt.use_winograd23_convolution)) + conv3x3s1_winograd43_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt); + else // if (opt.use_winograd23_convolution) + conv3x3s1_winograd23_pack4_msa(bottom_blob_bordered, top_blob, weight_winograd23_data, bias_data, opt); if (activation) { diff --git a/src/layer/mips/convolution_winograd_dot.h b/src/layer/mips/convolution_winograd_dot.h new file mode 100644 index 000000000..c4bede4b2 --- /dev/null +++ b/src/layer/mips/convolution_winograd_dot.h @@ -0,0 +1,495 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_winograd_dot_msa(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt) +{ + // Mat bottom_blob_tm(tiles, 16/36/64, inch, 4u, opt.workspace_allocator); + + const int tiles = bottom_blob_tm.w; + const int batch = bottom_blob_tm.h; + const int inch = bottom_blob_tm.c; + + // permute + Mat bottom_blob_tm2; + if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 4u, opt.workspace_allocator); + else + bottom_blob_tm2.create(1 * inch, tiles, batch, 4u, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < batch; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; + for (; i + 3 < tiles; i += 4) + { + float* tmpptr = tm2.row(i / 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { +#if __mips_msa + __msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0); +#else + tmpptr[0] = r0[0]; + tmpptr[1] = r0[1]; + tmpptr[2] = r0[2]; + tmpptr[3] = r0[3]; +#endif + + r0 += bottom_blob_tm.cstep; + tmpptr += 4; + } + } + for (; i < tiles; i++) + { + float* tmpptr = tm2.row(i / 4 + i % 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { + tmpptr[0] = r0[0]; + + r0 += bottom_blob_tm.cstep; + tmpptr += 1; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, batch, outch, 4u, opt.workspace_allocator); + +#if __mips_msa + int nn_outch = outch >> 3; + int remain_outch_start = nn_outch << 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 8; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + float* output4_tm = top_blob_tm.channel(p + 4); + float* output5_tm = top_blob_tm.channel(p + 5); + float* output6_tm = top_blob_tm.channel(p + 6); + float* output7_tm = top_blob_tm.channel(p + 7); + + const Mat kernel0_tm = kernel_tm.channel(p / 8); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + v4f32 _sum4 = (v4f32)__msa_fill_w(0); + v4f32 _sum5 = (v4f32)__msa_fill_w(0); + v4f32 _sum6 = (v4f32)__msa_fill_w(0); + v4f32 _sum7 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 32); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + v4i32 _w4567 = __msa_ld_w(k0 + 4, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + _sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0)); + _sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1)); + _sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2)); + _sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3)); + + r0 += 4; + k0 += 8; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + __msa_st_w((v4i32)_sum4, output4_tm, 0); + __msa_st_w((v4i32)_sum5, output5_tm, 0); + __msa_st_w((v4i32)_sum6, output6_tm, 0); + __msa_st_w((v4i32)_sum7, output7_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + output4_tm += 4; + output5_tm += 4; + output6_tm += 4; + output7_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + float sum4 = 0.f; + float sum5 = 0.f; + float sum6 = 0.f; + float sum7 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + sum4 += r0[0] * k0[4]; + sum5 += r0[0] * k0[5]; + sum6 += r0[0] * k0[6]; + sum7 += r0[0] * k0[7]; + + r0 += 1; + k0 += 8; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + output4_tm[0] = sum4; + output5_tm[0] = sum5; + output6_tm[0] = sum6; + output7_tm[0] = sum7; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + output4_tm++; + output5_tm++; + output6_tm++; + output7_tm++; + } + } + } + + nn_outch = (outch - remain_outch_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = remain_outch_start + pp * 4; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 16); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + + r0 += 4; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + + r0 += 1; + k0 += 4; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + } + } + } + + remain_outch_start += nn_outch << 2; +#else + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + + const Mat kernel0_tm = kernel_tm.channel(p / 2); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum01 = 0.f; + float sum02 = 0.f; + float sum03 = 0.f; + float sum10 = 0.f; + float sum11 = 0.f; + float sum12 = 0.f; + float sum13 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 8); + float w0 = k0[0]; + float w1 = k0[1]; + sum00 += r0[0] * w0; + sum01 += r0[1] * w0; + sum02 += r0[2] * w0; + sum03 += r0[3] * w0; + sum10 += r0[0] * w1; + sum11 += r0[1] * w1; + sum12 += r0[2] * w1; + sum13 += r0[3] * w1; + + r0 += 4; + k0 += 2; + } + + output0_tm[0] = sum00; + output0_tm[1] = sum01; + output0_tm[2] = sum02; + output0_tm[3] = sum03; + output1_tm[0] = sum10; + output1_tm[1] = sum11; + output1_tm[2] = sum12; + output1_tm[3] = sum13; + + output0_tm += 4; + output1_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum10 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 4); + __builtin_prefetch(k0 + 8); + float val0 = r0[0]; + sum00 += val0 * k0[0]; + sum10 += val0 * k0[1]; + + r0 += 1; + k0 += 2; + } + + output0_tm[0] = sum00; + output1_tm[0] = sum10; + output0_tm++; + output1_tm++; + } + } + } +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + float* output0_tm = top_blob_tm.channel(p); + +#if __mips_msa + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); +#endif + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int j = 0; +#if __mips_msa + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + + for (; j < nn; j++) + { + _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0)); + r0 += 4; + k0++; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + output0_tm += 4; +#else // __mips_msa + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 4); + float w0 = k0[0]; + sum0 += r0[0] * w0; + sum1 += r0[1] * w0; + sum2 += r0[2] * w0; + sum3 += r0[3] * w0; + + r0 += 4; + k0++; + } + + output0_tm[0] = sum0; + output0_tm[1] = sum1; + output0_tm[2] = sum2; + output0_tm[3] = sum3; + output0_tm += 4; +#endif // __mips_msa + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum = 0.f; + + for (int j = 0; j < nn; j++) + { + float w0 = k0[0]; + float val0 = r0[0]; + sum += val0 * w0; + + r0 += 1; + k0 += 1; + } + + output0_tm[0] = sum; + output0_tm += 1; + } + } + } +} diff --git a/src/layer/mips/convolution_winograd_dot_pack4.h b/src/layer/mips/convolution_winograd_dot_pack4.h new file mode 100644 index 000000000..2d64ea7b4 --- /dev/null +++ b/src/layer/mips/convolution_winograd_dot_pack4.h @@ -0,0 +1,448 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void convolution_winograd_dot_pack4_msa(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt) +{ + // Mat bottom_blob_tm(tiles, 16/36/64, inch, 16u, 4, opt.workspace_allocator); + + const int tiles = bottom_blob_tm.w; + const int batch = bottom_blob_tm.h; + const int inch = bottom_blob_tm.c; + + // permute + Mat bottom_blob_tm2; + if (tiles >= 12) + bottom_blob_tm2.create(12 * inch, tiles / 12 + (tiles % 12) / 8 + (tiles % 12 % 8) / 4 + (tiles % 12 % 4) / 2 + tiles % 12 % 2, batch, 16u, 4, opt.workspace_allocator); + else if (tiles >= 8) + bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator); + else if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + (tiles % 4) / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator); + else if (tiles >= 2) + bottom_blob_tm2.create(2 * inch, tiles / 2 + tiles % 2, batch, 16u, 4, opt.workspace_allocator); + else // if (tiles >= 1) + bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 4, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < batch; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; + for (; i + 11 < tiles; i += 12) + { + float* tmpptr = tm2.row(i / 12); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 4; + + for (int q = 0; q < inch; q++) + { + // transpose 4x8 + v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); + v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); + v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); + v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); + v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); + v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); + v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); + v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); + v4f32 _r8 = (v4f32)__msa_ld_w(r0 + 4 * 8, 0); + v4f32 _r9 = (v4f32)__msa_ld_w(r0 + 4 * 9, 0); + v4f32 _ra = (v4f32)__msa_ld_w(r0 + 4 * 10, 0); + v4f32 _rb = (v4f32)__msa_ld_w(r0 + 4 * 11, 0); + + v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); + v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); + v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); + v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); + v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); + v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); + v4i32 _r89r = __msa_ilvr_w((v4i32)_r9, (v4i32)_r8); + v4i32 _r89l = __msa_ilvl_w((v4i32)_r9, (v4i32)_r8); + v4i32 _rabr = __msa_ilvr_w((v4i32)_rb, (v4i32)_ra); + v4i32 _rabl = __msa_ilvl_w((v4i32)_rb, (v4i32)_ra); + v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); + v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); + v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); + v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); + v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); + v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); + v2i64 _r89ab_0 = __msa_ilvr_d((v2i64)_rabr, (v2i64)_r89r); + v2i64 _r89ab_1 = __msa_ilvl_d((v2i64)_rabr, (v2i64)_r89r); + v2i64 _r89ab_2 = __msa_ilvr_d((v2i64)_rabl, (v2i64)_r89l); + v2i64 _r89ab_3 = __msa_ilvl_d((v2i64)_rabl, (v2i64)_r89l); + + __msa_st_w((v4i32)_r0123_0, tmpptr, 0); + __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); + __msa_st_w((v4i32)_r89ab_0, tmpptr + 4 * 2, 0); + __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 3, 0); + __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 4, 0); + __msa_st_w((v4i32)_r89ab_1, tmpptr + 4 * 5, 0); + __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 6, 0); + __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 7, 0); + __msa_st_w((v4i32)_r89ab_2, tmpptr + 4 * 8, 0); + __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 9, 0); + __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 10, 0); + __msa_st_w((v4i32)_r89ab_3, tmpptr + 4 * 11, 0); + + r0 += bottom_blob_tm.cstep * 4; + tmpptr += 48; + } + } + for (; i + 7 < tiles; i += 8) + { + float* tmpptr = tm2.row(i / 12 + (i % 12) / 8); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 4; + + for (int q = 0; q < inch; q++) + { + // transpose 4x8 + v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); + v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); + v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); + v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); + v4f32 _r4 = (v4f32)__msa_ld_w(r0 + 4 * 4, 0); + v4f32 _r5 = (v4f32)__msa_ld_w(r0 + 4 * 5, 0); + v4f32 _r6 = (v4f32)__msa_ld_w(r0 + 4 * 6, 0); + v4f32 _r7 = (v4f32)__msa_ld_w(r0 + 4 * 7, 0); + + v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); + v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); + v4i32 _r45r = __msa_ilvr_w((v4i32)_r5, (v4i32)_r4); + v4i32 _r45l = __msa_ilvl_w((v4i32)_r5, (v4i32)_r4); + v4i32 _r67r = __msa_ilvr_w((v4i32)_r7, (v4i32)_r6); + v4i32 _r67l = __msa_ilvl_w((v4i32)_r7, (v4i32)_r6); + v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); + v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); + v2i64 _r4567_0 = __msa_ilvr_d((v2i64)_r67r, (v2i64)_r45r); + v2i64 _r4567_1 = __msa_ilvl_d((v2i64)_r67r, (v2i64)_r45r); + v2i64 _r4567_2 = __msa_ilvr_d((v2i64)_r67l, (v2i64)_r45l); + v2i64 _r4567_3 = __msa_ilvl_d((v2i64)_r67l, (v2i64)_r45l); + + __msa_st_w((v4i32)_r0123_0, tmpptr, 0); + __msa_st_w((v4i32)_r4567_0, tmpptr + 4, 0); + __msa_st_w((v4i32)_r0123_1, tmpptr + 4 * 2, 0); + __msa_st_w((v4i32)_r4567_1, tmpptr + 4 * 3, 0); + __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 4, 0); + __msa_st_w((v4i32)_r4567_2, tmpptr + 4 * 5, 0); + __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 6, 0); + __msa_st_w((v4i32)_r4567_3, tmpptr + 4 * 7, 0); + + r0 += bottom_blob_tm.cstep * 4; + tmpptr += 32; + } + } + for (; i + 3 < tiles; i += 4) + { + float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 4; + + for (int q = 0; q < inch; q++) + { + // transpose 4x4 + v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); + v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); + v4f32 _r2 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); + v4f32 _r3 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); + + v4i32 _r01r = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r01l = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r23r = __msa_ilvr_w((v4i32)_r3, (v4i32)_r2); + v4i32 _r23l = __msa_ilvl_w((v4i32)_r3, (v4i32)_r2); + v2i64 _r0123_0 = __msa_ilvr_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_1 = __msa_ilvl_d((v2i64)_r23r, (v2i64)_r01r); + v2i64 _r0123_2 = __msa_ilvr_d((v2i64)_r23l, (v2i64)_r01l); + v2i64 _r0123_3 = __msa_ilvl_d((v2i64)_r23l, (v2i64)_r01l); + + __msa_st_w((v4i32)_r0123_0, tmpptr, 0); + __msa_st_w((v4i32)_r0123_1, tmpptr + 4, 0); + __msa_st_w((v4i32)_r0123_2, tmpptr + 4 * 2, 0); + __msa_st_w((v4i32)_r0123_3, tmpptr + 4 * 3, 0); + + r0 += bottom_blob_tm.cstep * 4; + tmpptr += 16; + } + } + for (; i + 1 < tiles; i += 2) + { + float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 4; + + for (int q = 0; q < inch; q++) + { + // transpose 4x2 + v4f32 _r0 = (v4f32)__msa_ld_w(r0, 0); + v4f32 _r1 = (v4f32)__msa_ld_w(r0 + 4, 0); + + v4i32 _r01_0 = __msa_ilvr_w((v4i32)_r1, (v4i32)_r0); + v4i32 _r01_1 = __msa_ilvl_w((v4i32)_r1, (v4i32)_r0); + + __msa_st_w((v4i32)_r01_0, tmpptr, 0); + __msa_st_w((v4i32)_r01_1, tmpptr + 4, 0); + + r0 += bottom_blob_tm.cstep * 4; + tmpptr += 8; + } + } + for (; i < tiles; i++) + { + float* tmpptr = tm2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i) * 4; + + for (int q = 0; q < inch; q++) + { + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + __msa_st_w((v4i32)_val, tmpptr, 0); + + r0 += bottom_blob_tm.cstep * 4; + tmpptr += 4; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, batch, outch, 16u, 4, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* output0_tm = top_blob_tm.channel(p); + + const Mat kernel0_tm = kernel_tm.channel(p); + + for (int r = 0; r < batch; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 11 < tiles; i += 12) + { + const float* r0 = bb2.row(i / 12); + const float* k0 = kernel0_tm.row(r); + + int nn = inch * 4; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + v4f32 _sum4 = (v4f32)__msa_fill_w(0); + v4f32 _sum5 = (v4f32)__msa_fill_w(0); + v4f32 _sum6 = (v4f32)__msa_fill_w(0); + v4f32 _sum7 = (v4f32)__msa_fill_w(0); + v4f32 _sum8 = (v4f32)__msa_fill_w(0); + v4f32 _sum9 = (v4f32)__msa_fill_w(0); + v4f32 _suma = (v4f32)__msa_fill_w(0); + v4f32 _sumb = (v4f32)__msa_fill_w(0); + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 48); + __builtin_prefetch(k0 + 16); + v4i32 _val0123 = __msa_ld_w(r0, 0); + v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); + v4i32 _val89ab = __msa_ld_w(r0 + 8, 0); + v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); + _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); + _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); + _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); + _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); + _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); + _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); + _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); + _sum8 = __msa_fmadd_w(_sum8, (v4f32)__msa_splati_w(_val89ab, 0), _w0); + _sum9 = __msa_fmadd_w(_sum9, (v4f32)__msa_splati_w(_val89ab, 1), _w0); + _suma = __msa_fmadd_w(_suma, (v4f32)__msa_splati_w(_val89ab, 2), _w0); + _sumb = __msa_fmadd_w(_sumb, (v4f32)__msa_splati_w(_val89ab, 3), _w0); + + r0 += 12; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); + __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); + __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); + __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); + __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); + __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); + __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); + __msa_st_w((v4i32)_sum8, output0_tm + 4 * 8, 0); + __msa_st_w((v4i32)_sum9, output0_tm + 4 * 9, 0); + __msa_st_w((v4i32)_suma, output0_tm + 4 * 10, 0); + __msa_st_w((v4i32)_sumb, output0_tm + 4 * 11, 0); + + output0_tm += 4 * 12; + } + for (; i + 7 < tiles; i += 8) + { + const float* r0 = bb2.row(i / 12 + (i % 12) / 8); + const float* k0 = kernel0_tm.row(r); + + int nn = inch * 4; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + v4f32 _sum4 = (v4f32)__msa_fill_w(0); + v4f32 _sum5 = (v4f32)__msa_fill_w(0); + v4f32 _sum6 = (v4f32)__msa_fill_w(0); + v4f32 _sum7 = (v4f32)__msa_fill_w(0); + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 32); + __builtin_prefetch(k0 + 16); + v4i32 _val0123 = __msa_ld_w(r0, 0); + v4i32 _val4567 = __msa_ld_w(r0 + 4, 0); + v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); + _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); + _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); + _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); + _sum4 = __msa_fmadd_w(_sum4, (v4f32)__msa_splati_w(_val4567, 0), _w0); + _sum5 = __msa_fmadd_w(_sum5, (v4f32)__msa_splati_w(_val4567, 1), _w0); + _sum6 = __msa_fmadd_w(_sum6, (v4f32)__msa_splati_w(_val4567, 2), _w0); + _sum7 = __msa_fmadd_w(_sum7, (v4f32)__msa_splati_w(_val4567, 3), _w0); + + r0 += 8; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); + __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); + __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); + __msa_st_w((v4i32)_sum4, output0_tm + 4 * 4, 0); + __msa_st_w((v4i32)_sum5, output0_tm + 4 * 5, 0); + __msa_st_w((v4i32)_sum6, output0_tm + 4 * 6, 0); + __msa_st_w((v4i32)_sum7, output0_tm + 4 * 7, 0); + + output0_tm += 4 * 8; + } + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch * 4; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 16); + v4i32 _val0123 = __msa_ld_w(r0, 0); + v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, (v4f32)__msa_splati_w(_val0123, 0), _w0); + _sum1 = __msa_fmadd_w(_sum1, (v4f32)__msa_splati_w(_val0123, 1), _w0); + _sum2 = __msa_fmadd_w(_sum2, (v4f32)__msa_splati_w(_val0123, 2), _w0); + _sum3 = __msa_fmadd_w(_sum3, (v4f32)__msa_splati_w(_val0123, 3), _w0); + + r0 += 4; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); + __msa_st_w((v4i32)_sum2, output0_tm + 4 * 2, 0); + __msa_st_w((v4i32)_sum3, output0_tm + 4 * 3, 0); + + output0_tm += 4 * 4; + } + for (; i + 1 < tiles; i += 2) + { + const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2); + const float* k0 = kernel0_tm.row(r); + + int nn = inch * 4; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 8); + __builtin_prefetch(k0 + 16); + v4f32 _val0 = __msa_fill_w_f32(*r0++); + v4f32 _val1 = __msa_fill_w_f32(*r0++); + v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, _val0, _w0); + _sum1 = __msa_fmadd_w(_sum1, _val1, _w0); + + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output0_tm + 4, 0); + + output0_tm += 4 * 2; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 12 + (i % 12) / 8 + (i % 12 % 8) / 4 + (i % 12 % 4) / 2 + i % 12 % 2); + const float* k0 = kernel0_tm.row(r); + + int nn = inch * 4; // inch always > 0 + + v4f32 _sum = (v4f32)__msa_fill_w(0); + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 4); + __builtin_prefetch(k0 + 16); + v4f32 _val0 = __msa_fill_w_f32(*r0++); + v4f32 _w0 = (v4f32)__msa_ld_w(k0, 0); + _sum = __msa_fmadd_w(_sum, _val0, _w0); + + k0 += 4; + } + + __msa_st_w((v4i32)_sum, output0_tm, 0); + + output0_tm += 4; + } + } + } +} diff --git a/src/layer/mips/convolution_winograd_transform_pack4.h b/src/layer/mips/convolution_winograd_transform_pack4.h index 627fe6128..353dd9304 100644 --- a/src/layer/mips/convolution_winograd_transform_pack4.h +++ b/src/layer/mips/convolution_winograd_transform_pack4.h @@ -558,3 +558,173 @@ static void conv3x3s1_winograd43_transform_output_pack4_msa(const Mat& top_blob_ } } } + +static void conv3x3s1_winograd23_transform_input_pack4_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt) +{ + const int w = bottom_blob.w; + const int h = bottom_blob.h; + const int inch = bottom_blob.c; + + const int w_tiles = (w - 2) / 2; + const int h_tiles = (h - 2) / 2; + const int tiles = w_tiles * h_tiles; + + // const float itm[4][4] = { + // {1.0f, 0.0f, -1.0f, 0.0f}, + // {0.0f, 1.0f, 1.00f, 0.0f}, + // {0.0f, -1.0f, 1.00f, 0.0f}, + // {0.0f, -1.0f, 0.00f, 1.0f} + // }; + + // 0 = r00 - r02 + // 1 = r01 + r02 + // 2 = r02 - r01 + // 3 = r03 - r01 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < inch; q++) + { + const Mat img0 = bottom_blob.channel(q); + Mat img0_tm = bottom_blob_tm.channel(q); + + float tmp[4][4][4]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* r0 = img0.row(i * 2) + (j * 2) * 4; + + for (int m = 0; m < 4; m++) + { + v4f32 _r00 = (v4f32)__msa_ld_w(r0, 0); + v4f32 _r01 = (v4f32)__msa_ld_w(r0 + 4, 0); + v4f32 _r02 = (v4f32)__msa_ld_w(r0 + 4 * 2, 0); + v4f32 _r03 = (v4f32)__msa_ld_w(r0 + 4 * 3, 0); + + v4f32 _tmp0m = __msa_fsub_w(_r00, _r02); + v4f32 _tmp1m = __msa_fadd_w(_r01, _r02); + v4f32 _tmp2m = __msa_fsub_w(_r02, _r01); + v4f32 _tmp3m = __msa_fsub_w(_r03, _r01); + + __msa_st_w((v4i32)_tmp0m, tmp[0][m], 0); + __msa_st_w((v4i32)_tmp1m, tmp[1][m], 0); + __msa_st_w((v4i32)_tmp2m, tmp[2][m], 0); + __msa_st_w((v4i32)_tmp3m, tmp[3][m], 0); + + r0 += w * 4; + } + + float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j) * 4; + float* r0_tm_1 = r0_tm_0 + tiles * 4; + float* r0_tm_2 = r0_tm_0 + tiles * 4 * 2; + float* r0_tm_3 = r0_tm_0 + tiles * 4 * 3; + + for (int m = 0; m < 4; m++) + { + v4f32 _tmp00 = (v4f32)__msa_ld_w(tmp[m][0], 0); + v4f32 _tmp01 = (v4f32)__msa_ld_w(tmp[m][1], 0); + v4f32 _tmp02 = (v4f32)__msa_ld_w(tmp[m][2], 0); + v4f32 _tmp03 = (v4f32)__msa_ld_w(tmp[m][3], 0); + + v4f32 _r0tm0 = __msa_fsub_w(_tmp00, _tmp02); + v4f32 _r0tm1 = __msa_fadd_w(_tmp01, _tmp02); + v4f32 _r0tm2 = __msa_fsub_w(_tmp02, _tmp01); + v4f32 _r0tm3 = __msa_fsub_w(_tmp03, _tmp01); + + __msa_st_w((v4i32)_r0tm0, r0_tm_0, 0); + __msa_st_w((v4i32)_r0tm1, r0_tm_1, 0); + __msa_st_w((v4i32)_r0tm2, r0_tm_2, 0); + __msa_st_w((v4i32)_r0tm3, r0_tm_3, 0); + + r0_tm_0 += tiles * 4 * 4; + r0_tm_1 += tiles * 4 * 4; + r0_tm_2 += tiles * 4 * 4; + r0_tm_3 += tiles * 4 * 4; + } + } + } + } +} + +static void conv3x3s1_winograd23_transform_output_pack4_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt) +{ + const int outw = top_blob.w; + const int outh = top_blob.h; + const int outch = top_blob.c; + + const int w_tiles = outw / 2; + const int h_tiles = outh / 2; + const int tiles = w_tiles * h_tiles; + + const float* biasptr = bias; + + // const float otm[2][4] = { + // {1.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 1.0f} + // }; + + // 0 = r00 + r01 + r02 + // 1 = r01 - r02 + r03 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + const Mat out0_tm = top_blob_tm.channel(p); + Mat out0 = top_blob.channel(p); + + v4f32 _bias0 = biasptr ? (v4f32)__msa_ld_w(biasptr + p * 4, 0) : (v4f32)__msa_fill_w(0); + + float tmp[2][4][4]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j) * 4; + const float* output0_tm_1 = output0_tm_0 + tiles * 4; + const float* output0_tm_2 = output0_tm_0 + tiles * 4 * 2; + const float* output0_tm_3 = output0_tm_0 + tiles * 4 * 3; + + float* output0 = out0.row(i * 2) + (j * 2) * 4; + + for (int m = 0; m < 4; m++) + { + v4f32 _out0tm0 = (v4f32)__msa_ld_w(output0_tm_0, 0); + v4f32 _out0tm1 = (v4f32)__msa_ld_w(output0_tm_1, 0); + v4f32 _out0tm2 = (v4f32)__msa_ld_w(output0_tm_2, 0); + v4f32 _out0tm3 = (v4f32)__msa_ld_w(output0_tm_3, 0); + + v4f32 _tmp0m = __msa_fadd_w(__msa_fadd_w(_out0tm0, _out0tm1), _out0tm2); + v4f32 _tmp1m = __msa_fadd_w(__msa_fsub_w(_out0tm1, _out0tm2), _out0tm3); + + __msa_st_w((v4i32)_tmp0m, tmp[0][m], 0); + __msa_st_w((v4i32)_tmp1m, tmp[1][m], 0); + + output0_tm_0 += tiles * 4 * 4; + output0_tm_1 += tiles * 4 * 4; + output0_tm_2 += tiles * 4 * 4; + output0_tm_3 += tiles * 4 * 4; + } + + for (int m = 0; m < 2; m++) + { + v4f32 _tmp00 = (v4f32)__msa_ld_w(tmp[m][0], 0); + v4f32 _tmp01 = (v4f32)__msa_ld_w(tmp[m][1], 0); + v4f32 _tmp02 = (v4f32)__msa_ld_w(tmp[m][2], 0); + v4f32 _tmp03 = (v4f32)__msa_ld_w(tmp[m][3], 0); + + v4f32 _out00 = __msa_fadd_w(_bias0, __msa_fadd_w(__msa_fadd_w(_tmp00, _tmp01), _tmp02)); + v4f32 _out01 = __msa_fadd_w(_bias0, __msa_fadd_w(__msa_fsub_w(_tmp01, _tmp02), _tmp03)); + + __msa_st_w((v4i32)_out00, output0, 0); + __msa_st_w((v4i32)_out01, output0 + 4, 0); + + output0 += outw * 4; + } + } + } + } +}