From c3adbcf9f340264eded498cfa3b6d2b9caf7dde9 Mon Sep 17 00:00:00 2001 From: nihui Date: Sat, 28 May 2022 18:42:30 +0800 Subject: [PATCH] mips optimization for convolution sgemm (#3853) * mips optimization for convolution sgemm * mips optimization for general convolution int8 gemm * mips optmization for convolution winograd pack1 * preload magic --- src/layer/mips/convolution_3x3.h | 1370 +++++++++++++++++ src/layer/mips/convolution_mips.cpp | 29 + src/layer/mips/convolution_mips.h | 1 + src/layer/mips/convolution_sgemm.h | 190 ++- src/layer/mips/convolution_sgemm_int8.h | 214 ++- .../mips/convolution_winograd_transform.h | 405 +++++ 6 files changed, 2095 insertions(+), 114 deletions(-) create mode 100644 src/layer/mips/convolution_3x3.h create mode 100644 src/layer/mips/convolution_winograd_transform.h diff --git a/src/layer/mips/convolution_3x3.h b/src/layer/mips/convolution_3x3.h new file mode 100644 index 000000000..5e4cc4be2 --- /dev/null +++ b/src/layer/mips/convolution_3x3.h @@ -0,0 +1,1370 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s1_winograd23_transform_kernel_msa(const Mat& kernel, Mat& kernel_tm2, int inch, int outch, const Option& opt) +{ + Mat kernel_tm(4 * 4, inch, outch); + + // G + const float ktm[4][3] = { + {1.0f, 0.0f, 0.0f}, + {1.0f / 2, 1.0f / 2, 1.0f / 2}, + {1.0f / 2, -1.0f / 2, 1.0f / 2}, + {0.0f, 0.0f, 1.0f} + }; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + for (int q = 0; q < inch; q++) + { + const float* kernel0 = (const float*)kernel + p * inch * 9 + q * 9; + float* kernel_tm0 = kernel_tm.channel(p).row(q); + + // transform kernel + const float* k0 = kernel0; + const float* k1 = kernel0 + 3; + const float* k2 = kernel0 + 6; + + // h + float tmp[4][3]; + for (int i = 0; i < 4; i++) + { + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; + } + + // U + for (int j = 0; j < 4; j++) + { + float* tmpp = &tmp[j][0]; + + for (int i = 0; i < 4; i++) + { + kernel_tm0[j * 4 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; + } + } + } + } + + // interleave + // src = 16-inch-outch + // dst = inch-16-outch +#if __mips_msa + kernel_tm2.create(8 * inch, 16, outch / 8 + (outch % 8) / 4 + outch % 4); +#else + kernel_tm2.create(2 * inch, 16, outch / 2 + outch % 2); +#endif + + int q = 0; +#if __mips_msa + for (; q + 7 < outch; q += 8) + { + Mat g0 = kernel_tm2.channel(q / 8); + + for (int k = 0; k < 16; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 8; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } + for (; q + 3 < outch; q += 4) + { + Mat g0 = kernel_tm2.channel(q / 8 + (q % 8) / 4); + + for (int k = 0; k < 16; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 4; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } +#else // __mips_msa + for (; q + 1 < outch; q += 2) + { + Mat g0 = kernel_tm2.channel(q / 2); + + for (int k = 0; k < 16; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 2; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } +#endif // __mips_msa + for (; q < outch; q++) + { +#if __mips_msa + Mat g0 = kernel_tm2.channel(q / 8 + (q % 8) / 4 + q % 4); +#else + Mat g0 = kernel_tm2.channel(q / 2 + q % 2); +#endif + + for (int k = 0; k < 16; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + const float* k00 = kernel_tm.channel(q).row(p); + g00[0] = k00[k]; + g00++; + } + } + } +} + +static void conv3x3s1_winograd23_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + // pad to 2n+2, winograd F(2,3) + Mat bottom_blob_bordered = bottom_blob; + + outw = (outw + 1) / 2 * 2; + outh = (outh + 1) / 2 * 2; + + w = outw + 2; + h = outh + 2; + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt_b); + + // BEGIN transform input + Mat bottom_blob_tm; + { + int w_tiles = outw / 2; + int h_tiles = outh / 2; + int tiles = w_tiles * h_tiles; + + bottom_blob_tm.create(tiles, 16, inch, 4u, opt.workspace_allocator); + conv3x3s1_winograd23_transform_input_msa(bottom_blob_bordered, bottom_blob_tm, opt); + } + bottom_blob_bordered = Mat(); + // END transform input + + // BEGIN dot + Mat top_blob_tm; + { + int w_tm = outw / 2 * 4; + int h_tm = outh / 2 * 4; + + const int tiles = h_tm / 4 * w_tm / 4; + + // permute + Mat bottom_blob_tm2; + if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 16, 4u, opt.workspace_allocator); + else + bottom_blob_tm2.create(1 * inch, tiles, 16, 4u, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < 16; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; + for (; i + 3 < tiles; i += 4) + { + float* tmpptr = tm2.row(i / 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { +#if __mips_msa + __msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0); +#else + tmpptr[0] = r0[0]; + tmpptr[1] = r0[1]; + tmpptr[2] = r0[2]; + tmpptr[3] = r0[3]; +#endif + + r0 += bottom_blob_tm.cstep; + tmpptr += 4; + } + } + for (; i < tiles; i++) + { + float* tmpptr = tm2.row(i / 4 + i % 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { + tmpptr[0] = r0[0]; + + r0 += bottom_blob_tm.cstep; + tmpptr += 1; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, 16, outch, 4u, opt.workspace_allocator); + +#if __mips_msa + int nn_outch = outch >> 3; + int remain_outch_start = nn_outch << 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 8; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + float* output4_tm = top_blob_tm.channel(p + 4); + float* output5_tm = top_blob_tm.channel(p + 5); + float* output6_tm = top_blob_tm.channel(p + 6); + float* output7_tm = top_blob_tm.channel(p + 7); + + const Mat kernel0_tm = kernel_tm.channel(p / 8); + + for (int r = 0; r < 16; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + v4f32 _sum4 = (v4f32)__msa_fill_w(0); + v4f32 _sum5 = (v4f32)__msa_fill_w(0); + v4f32 _sum6 = (v4f32)__msa_fill_w(0); + v4f32 _sum7 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 32); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + v4i32 _w4567 = __msa_ld_w(k0 + 4, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + _sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0)); + _sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1)); + _sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2)); + _sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3)); + + r0 += 4; + k0 += 8; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + __msa_st_w((v4i32)_sum4, output4_tm, 0); + __msa_st_w((v4i32)_sum5, output5_tm, 0); + __msa_st_w((v4i32)_sum6, output6_tm, 0); + __msa_st_w((v4i32)_sum7, output7_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + output4_tm += 4; + output5_tm += 4; + output6_tm += 4; + output7_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + float sum4 = 0.f; + float sum5 = 0.f; + float sum6 = 0.f; + float sum7 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + sum4 += r0[0] * k0[4]; + sum5 += r0[0] * k0[5]; + sum6 += r0[0] * k0[6]; + sum7 += r0[0] * k0[7]; + + r0 += 1; + k0 += 8; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + output4_tm[0] = sum4; + output5_tm[0] = sum5; + output6_tm[0] = sum6; + output7_tm[0] = sum7; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + output4_tm++; + output5_tm++; + output6_tm++; + output7_tm++; + } + } + } + + nn_outch = (outch - remain_outch_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = remain_outch_start + pp * 4; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); + + for (int r = 0; r < 16; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 16); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + + r0 += 4; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + + r0 += 1; + k0 += 4; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + } + } + } + + remain_outch_start += nn_outch << 2; +#else + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + + const Mat kernel0_tm = kernel_tm.channel(p / 2); + + for (int r = 0; r < 16; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum01 = 0.f; + float sum02 = 0.f; + float sum03 = 0.f; + float sum10 = 0.f; + float sum11 = 0.f; + float sum12 = 0.f; + float sum13 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 8); + float w0 = k0[0]; + float w1 = k0[1]; + sum00 += r0[0] * w0; + sum01 += r0[1] * w0; + sum02 += r0[2] * w0; + sum03 += r0[3] * w0; + sum10 += r0[0] * w1; + sum11 += r0[1] * w1; + sum12 += r0[2] * w1; + sum13 += r0[3] * w1; + + r0 += 4; + k0 += 2; + } + + output0_tm[0] = sum00; + output0_tm[1] = sum01; + output0_tm[2] = sum02; + output0_tm[3] = sum03; + output1_tm[0] = sum10; + output1_tm[1] = sum11; + output1_tm[2] = sum12; + output1_tm[3] = sum13; + + output0_tm += 4; + output1_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum10 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 4); + __builtin_prefetch(k0 + 8); + float val0 = r0[0]; + sum00 += val0 * k0[0]; + sum10 += val0 * k0[1]; + + r0 += 1; + k0 += 2; + } + + output0_tm[0] = sum00; + output1_tm[0] = sum10; + output0_tm++; + output1_tm++; + } + } + } +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + float* output0_tm = top_blob_tm.channel(p); + +#if __mips_msa + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); +#endif + + for (int r = 0; r < 16; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int j = 0; +#if __mips_msa + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + + for (; j < nn; j++) + { + _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0)); + r0 += 4; + k0++; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + output0_tm += 4; +#else // __mips_msa + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 4); + float w0 = k0[0]; + sum0 += r0[0] * w0; + sum1 += r0[1] * w0; + sum2 += r0[2] * w0; + sum3 += r0[3] * w0; + + r0 += 4; + k0++; + } + + output0_tm[0] = sum0; + output0_tm[1] = sum1; + output0_tm[2] = sum2; + output0_tm[3] = sum3; + output0_tm += 4; +#endif // __mips_msa + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum = 0.f; + + for (int j = 0; j < nn; j++) + { + float w0 = k0[0]; + float val0 = r0[0]; + sum += val0 * w0; + + r0 += 1; + k0 += 1; + } + + output0_tm[0] = sum; + output0_tm += 1; + } + } + } + } + bottom_blob_tm = Mat(); + // END dot + + // BEGIN transform output + Mat top_blob_bordered; + if (outw == top_blob.w && outh == top_blob.h) + { + top_blob_bordered = top_blob; + } + else + { + top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator); + } + { + conv3x3s1_winograd23_transform_output_msa(top_blob_tm, top_blob_bordered, bias, opt); + } + // END transform output + + // cut result pad + copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt); +} + +static void conv3x3s1_winograd43_transform_kernel_msa(const Mat& kernel, Mat& kernel_tm2, int inch, int outch, const Option& opt) +{ + Mat kernel_tm(6 * 6, inch, outch); + + // G + const float ktm[6][3] = { + {1.0f / 4, 0.0f, 0.0f}, + {-1.0f / 6, -1.0f / 6, -1.0f / 6}, + {-1.0f / 6, 1.0f / 6, -1.0f / 6}, + {1.0f / 24, 1.0f / 12, 1.0f / 6}, + {1.0f / 24, -1.0f / 12, 1.0f / 6}, + {0.0f, 0.0f, 1.0f} + }; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + for (int q = 0; q < inch; q++) + { + const float* kernel0 = (const float*)kernel + p * inch * 9 + q * 9; + float* kernel_tm0 = kernel_tm.channel(p).row(q); + + // transform kernel + const float* k0 = kernel0; + const float* k1 = kernel0 + 3; + const float* k2 = kernel0 + 6; + + // h + float tmp[6][3]; + for (int i = 0; i < 6; i++) + { + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; + } + + // U + for (int j = 0; j < 6; j++) + { + float* tmpp = &tmp[j][0]; + + for (int i = 0; i < 6; i++) + { + kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; + } + } + } + } + + // interleave + // src = 36-inch-outch + // dst = inch-36-outch +#if __mips_msa + kernel_tm2.create(8 * inch, 36, outch / 8 + (outch % 8) / 4 + outch % 4); +#else + kernel_tm2.create(2 * inch, 36, outch / 2 + outch % 2); +#endif + + int q = 0; +#if __mips_msa + for (; q + 7 < outch; q += 8) + { + Mat g0 = kernel_tm2.channel(q / 8); + + for (int k = 0; k < 36; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 8; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } + for (; q + 3 < outch; q += 4) + { + Mat g0 = kernel_tm2.channel(q / 8 + (q % 8) / 4); + + for (int k = 0; k < 36; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 4; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } +#else // __mips_msa + for (; q + 1 < outch; q += 2) + { + Mat g0 = kernel_tm2.channel(q / 2); + + for (int k = 0; k < 36; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + for (int i = 0; i < 2; i++) + { + const float* k00 = kernel_tm.channel(q + i).row(p); + g00[0] = k00[k]; + g00++; + } + } + } + } +#endif // __mips_msa + for (; q < outch; q++) + { +#if __mips_msa + Mat g0 = kernel_tm2.channel(q / 8 + (q % 8) / 4 + q % 4); +#else + Mat g0 = kernel_tm2.channel(q / 2 + q % 2); +#endif + + for (int k = 0; k < 36; k++) + { + float* g00 = g0.row(k); + + for (int p = 0; p < inch; p++) + { + const float* k00 = kernel_tm.channel(q).row(p); + g00[0] = k00[k]; + g00++; + } + } + } +} + +static void conv3x3s1_winograd43_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Mat& bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + int outch = top_blob.c; + + // pad to 4n+2, winograd F(4,3) + Mat bottom_blob_bordered = bottom_blob; + + outw = (outw + 3) / 4 * 4; + outh = (outh + 3) / 4 * 4; + + w = outw + 2; + h = outh + 2; + + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, 0, 0.f, opt_b); + + // BEGIN transform input + Mat bottom_blob_tm; + { + int w_tiles = outw / 4; + int h_tiles = outh / 4; + int tiles = w_tiles * h_tiles; + + bottom_blob_tm.create(tiles, 36, inch, 4u, opt.workspace_allocator); + conv3x3s1_winograd43_transform_input_msa(bottom_blob_bordered, bottom_blob_tm, opt); + } + bottom_blob_bordered = Mat(); + // END transform input + + // BEGIN dot + Mat top_blob_tm; + { + int w_tm = outw / 4 * 6; + int h_tm = outh / 4 * 6; + + const int tiles = h_tm / 6 * w_tm / 6; + + // permute + Mat bottom_blob_tm2; + if (tiles >= 4) + bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, 36, 4u, opt.workspace_allocator); + else + bottom_blob_tm2.create(1 * inch, tiles, 36, 4u, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int r = 0; r < 36; r++) + { + Mat tm2 = bottom_blob_tm2.channel(r); + + // tile + int i = 0; + for (; i + 3 < tiles; i += 4) + { + float* tmpptr = tm2.row(i / 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { +#if __mips_msa + __msa_st_w(__msa_ld_w(r0, 0), tmpptr, 0); +#else + tmpptr[0] = r0[0]; + tmpptr[1] = r0[1]; + tmpptr[2] = r0[2]; + tmpptr[3] = r0[3]; +#endif + + r0 += bottom_blob_tm.cstep; + tmpptr += 4; + } + } + for (; i < tiles; i++) + { + float* tmpptr = tm2.row(i / 4 + i % 4); + + const float* r0 = bottom_blob_tm; + + r0 += (r * tiles + i); + + for (int q = 0; q < inch; q++) + { + tmpptr[0] = r0[0]; + + r0 += bottom_blob_tm.cstep; + tmpptr += 1; + } + } + } + + bottom_blob_tm = Mat(); + // permute end + + top_blob_tm.create(tiles, 36, outch, 4u, opt.workspace_allocator); + +#if __mips_msa + int nn_outch = outch >> 3; + int remain_outch_start = nn_outch << 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 8; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + float* output4_tm = top_blob_tm.channel(p + 4); + float* output5_tm = top_blob_tm.channel(p + 5); + float* output6_tm = top_blob_tm.channel(p + 6); + float* output7_tm = top_blob_tm.channel(p + 7); + + const Mat kernel0_tm = kernel_tm.channel(p / 8); + + for (int r = 0; r < 36; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + v4f32 _sum4 = (v4f32)__msa_fill_w(0); + v4f32 _sum5 = (v4f32)__msa_fill_w(0); + v4f32 _sum6 = (v4f32)__msa_fill_w(0); + v4f32 _sum7 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 32); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + v4i32 _w4567 = __msa_ld_w(k0 + 4, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + _sum4 = __msa_fmadd_w(_sum4, _val, (v4f32)__msa_splati_w(_w4567, 0)); + _sum5 = __msa_fmadd_w(_sum5, _val, (v4f32)__msa_splati_w(_w4567, 1)); + _sum6 = __msa_fmadd_w(_sum6, _val, (v4f32)__msa_splati_w(_w4567, 2)); + _sum7 = __msa_fmadd_w(_sum7, _val, (v4f32)__msa_splati_w(_w4567, 3)); + + r0 += 4; + k0 += 8; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + __msa_st_w((v4i32)_sum4, output4_tm, 0); + __msa_st_w((v4i32)_sum5, output5_tm, 0); + __msa_st_w((v4i32)_sum6, output6_tm, 0); + __msa_st_w((v4i32)_sum7, output7_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + output4_tm += 4; + output5_tm += 4; + output6_tm += 4; + output7_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + float sum4 = 0.f; + float sum5 = 0.f; + float sum6 = 0.f; + float sum7 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + sum4 += r0[0] * k0[4]; + sum5 += r0[0] * k0[5]; + sum6 += r0[0] * k0[6]; + sum7 += r0[0] * k0[7]; + + r0 += 1; + k0 += 8; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + output4_tm[0] = sum4; + output5_tm[0] = sum5; + output6_tm[0] = sum6; + output7_tm[0] = sum7; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + output4_tm++; + output5_tm++; + output6_tm++; + output7_tm++; + } + } + } + + nn_outch = (outch - remain_outch_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = remain_outch_start + pp * 4; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + float* output2_tm = top_blob_tm.channel(p + 2); + float* output3_tm = top_blob_tm.channel(p + 3); + + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4); + + for (int r = 0; r < 36; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + v4f32 _sum1 = (v4f32)__msa_fill_w(0); + v4f32 _sum2 = (v4f32)__msa_fill_w(0); + v4f32 _sum3 = (v4f32)__msa_fill_w(0); + + int j = 0; + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 16); + v4f32 _val = (v4f32)__msa_ld_w(r0, 0); + v4i32 _w0123 = __msa_ld_w(k0, 0); + _sum0 = __msa_fmadd_w(_sum0, _val, (v4f32)__msa_splati_w(_w0123, 0)); + _sum1 = __msa_fmadd_w(_sum1, _val, (v4f32)__msa_splati_w(_w0123, 1)); + _sum2 = __msa_fmadd_w(_sum2, _val, (v4f32)__msa_splati_w(_w0123, 2)); + _sum3 = __msa_fmadd_w(_sum3, _val, (v4f32)__msa_splati_w(_w0123, 3)); + + r0 += 4; + k0 += 4; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + __msa_st_w((v4i32)_sum1, output1_tm, 0); + __msa_st_w((v4i32)_sum2, output2_tm, 0); + __msa_st_w((v4i32)_sum3, output3_tm, 0); + + output0_tm += 4; + output1_tm += 4; + output2_tm += 4; + output3_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + int j = 0; + for (; j < nn; j++) + { + sum0 += r0[0] * k0[0]; + sum1 += r0[0] * k0[1]; + sum2 += r0[0] * k0[2]; + sum3 += r0[0] * k0[3]; + + r0 += 1; + k0 += 4; + } + + output0_tm[0] = sum0; + output1_tm[0] = sum1; + output2_tm[0] = sum2; + output3_tm[0] = sum3; + + output0_tm++; + output1_tm++; + output2_tm++; + output3_tm++; + } + } + } + + remain_outch_start += nn_outch << 2; +#else + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + float* output0_tm = top_blob_tm.channel(p); + float* output1_tm = top_blob_tm.channel(p + 1); + + const Mat kernel0_tm = kernel_tm.channel(p / 2); + + for (int r = 0; r < 36; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum01 = 0.f; + float sum02 = 0.f; + float sum03 = 0.f; + float sum10 = 0.f; + float sum11 = 0.f; + float sum12 = 0.f; + float sum13 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 8); + float w0 = k0[0]; + float w1 = k0[1]; + sum00 += r0[0] * w0; + sum01 += r0[1] * w0; + sum02 += r0[2] * w0; + sum03 += r0[3] * w0; + sum10 += r0[0] * w1; + sum11 += r0[1] * w1; + sum12 += r0[2] * w1; + sum13 += r0[3] * w1; + + r0 += 4; + k0 += 2; + } + + output0_tm[0] = sum00; + output0_tm[1] = sum01; + output0_tm[2] = sum02; + output0_tm[3] = sum03; + output1_tm[0] = sum10; + output1_tm[1] = sum11; + output1_tm[2] = sum12; + output1_tm[3] = sum13; + + output0_tm += 4; + output1_tm += 4; + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum00 = 0.f; + float sum10 = 0.f; + + for (int j = 0; j < nn; j++) + { + __builtin_prefetch(r0 + 4); + __builtin_prefetch(k0 + 8); + float val0 = r0[0]; + sum00 += val0 * k0[0]; + sum10 += val0 * k0[1]; + + r0 += 1; + k0 += 2; + } + + output0_tm[0] = sum00; + output1_tm[0] = sum10; + output0_tm++; + output1_tm++; + } + } + } +#endif + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_outch_start; p < outch; p++) + { + float* output0_tm = top_blob_tm.channel(p); + +#if __mips_msa + const Mat kernel0_tm = kernel_tm.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const Mat kernel0_tm = kernel_tm.channel(p / 2 + p % 2); +#endif + + for (int r = 0; r < 36; r++) + { + const Mat bb2 = bottom_blob_tm2.channel(r); + + int i = 0; + for (; i + 3 < tiles; i += 4) + { + const float* r0 = bb2.row(i / 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + int j = 0; +#if __mips_msa + v4f32 _sum0 = (v4f32)__msa_fill_w(0); + + for (; j < nn; j++) + { + _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(k0[0]), (v4f32)__msa_ld_w(r0, 0)); + r0 += 4; + k0++; + } + + __msa_st_w((v4i32)_sum0, output0_tm, 0); + output0_tm += 4; +#else // __mips_msa + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + for (; j < nn; j++) + { + __builtin_prefetch(r0 + 16); + __builtin_prefetch(k0 + 4); + float w0 = k0[0]; + sum0 += r0[0] * w0; + sum1 += r0[1] * w0; + sum2 += r0[2] * w0; + sum3 += r0[3] * w0; + + r0 += 4; + k0++; + } + + output0_tm[0] = sum0; + output0_tm[1] = sum1; + output0_tm[2] = sum2; + output0_tm[3] = sum3; + output0_tm += 4; +#endif // __mips_msa + } + for (; i < tiles; i++) + { + const float* r0 = bb2.row(i / 4 + i % 4); + const float* k0 = kernel0_tm.row(r); + + int nn = inch; // inch always > 0 + + float sum = 0.f; + + for (int j = 0; j < nn; j++) + { + float w0 = k0[0]; + float val0 = r0[0]; + sum += val0 * w0; + + r0 += 1; + k0 += 1; + } + + output0_tm[0] = sum; + output0_tm += 1; + } + } + } + } + bottom_blob_tm = Mat(); + // END dot + + // BEGIN transform output + Mat top_blob_bordered; + if (outw == top_blob.w && outh == top_blob.h) + { + top_blob_bordered = top_blob; + } + else + { + top_blob_bordered.create(outw, outh, outch, 4u, opt.workspace_allocator); + } + { + conv3x3s1_winograd43_transform_output_msa(top_blob_tm, top_blob_bordered, bias, opt); + } + // END transform output + + // cut result pad + copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt); +} diff --git a/src/layer/mips/convolution_mips.cpp b/src/layer/mips/convolution_mips.cpp index 676fb9cba..2f280af84 100644 --- a/src/layer/mips/convolution_mips.cpp +++ b/src/layer/mips/convolution_mips.cpp @@ -30,7 +30,9 @@ namespace ncnn { #include "convolution_sgemm.h" +#include "convolution_winograd_transform.h" #include "convolution_1x1.h" +#include "convolution_3x3.h" #if NCNN_INT8 #include "convolution_sgemm_int8.h" @@ -189,6 +191,17 @@ int Convolution_mips::create_pipeline(const Option& opt) { convolution_im2col_sgemm_transform_kernel_msa(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); } + if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + if (num_input >= 16 && num_output >= 16) + { + conv3x3s1_winograd43_transform_kernel_msa(weight_data, weight_winograd43_data, num_input, num_output, opt); + } + else + { + conv3x3s1_winograd23_transform_kernel_msa(weight_data, weight_winograd23_data, num_input, num_output, opt); + } + } else if (opt.use_sgemm_convolution) { convolution_im2col_sgemm_transform_kernel_msa(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); @@ -395,6 +408,22 @@ int Convolution_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Optio activation->forward_inplace(top_blob, opt); } } + else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + if (num_input >= 16 && num_output >= 16) + { + conv3x3s1_winograd43_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt); + } + else + { + conv3x3s1_winograd23_msa(bottom_blob_bordered, top_blob, weight_winograd23_data, bias_data, opt); + } + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } else if (opt.use_sgemm_convolution) { convolution_im2col_sgemm_msa(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); diff --git a/src/layer/mips/convolution_mips.h b/src/layer/mips/convolution_mips.h index 8d4bbd553..5ba3fabb6 100644 --- a/src/layer/mips/convolution_mips.h +++ b/src/layer/mips/convolution_mips.h @@ -41,6 +41,7 @@ public: Layer* activation; Mat weight_sgemm_data; + Mat weight_winograd23_data; Mat weight_winograd43_data; Mat weight_winograd63_data; diff --git a/src/layer/mips/convolution_sgemm.h b/src/layer/mips/convolution_sgemm.h index 6b2d870a3..2a0324912 100644 --- a/src/layer/mips/convolution_sgemm.h +++ b/src/layer/mips/convolution_sgemm.h @@ -26,7 +26,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& // permute Mat tmp; -#if __mips_msa if (size >= 4) tmp.create(4 * maxk, inch, size / 4 + size % 4, 4u, 1, opt.workspace_allocator); else @@ -47,7 +46,14 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& for (int k = 0; k < maxk; k++) { +#if __mips_msa __msa_st_w(__msa_ld_w(img0, 0), tmpptr, 0); +#else + tmpptr[0] = img0[0]; + tmpptr[1] = img0[1]; + tmpptr[2] = img0[2]; + tmpptr[3] = img0[3]; +#endif img0 += size; tmpptr += 4; } @@ -74,28 +80,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& } } } -#else // __mips_msa - tmp.create(maxk, inch, size, 4u, 1, opt.workspace_allocator); - { - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = 0; i < size; i++) - { - float* tmpptr = tmp.channel(i); - - for (int q = 0; q < inch; q++) - { - const float* img0 = (const float*)bottom_im2col.channel(q) + i; - - for (int k = 0; k < maxk; k++) - { - tmpptr[0] = img0[0]; - img0 += size; - tmpptr += 1; - } - } - } - } -#endif // __mips_msa #if __mips_msa int nn_outch = outch >> 3; @@ -311,68 +295,163 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& } remain_outch_start += nn_outch << 2; +#else // __mips_msa + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_outch_start; p < outch; p++) + for (int pp = 0; pp < nn_outch; pp++) { + int p = pp * 2; + float* outptr0 = top_blob.channel(p); + float* outptr1 = top_blob.channel(p + 1); - const float bias0 = bias ? bias[p] : 0.f; + const float zeros[2] = {0.f, 0.f}; + const float* biasptr = bias ? bias + p : zeros; int i = 0; for (; i + 3 < size; i += 4) { const float* tmpptr = tmp.channel(i / 4); - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); + const float* kptr = kernel.channel(p / 2); int nn = inch * maxk; // inch always > 0 - v4f32 _sum0 = __msa_fill_w_f32(bias0); + float sum00 = biasptr[0]; + float sum01 = biasptr[0]; + float sum02 = biasptr[0]; + float sum03 = biasptr[0]; + float sum10 = biasptr[1]; + float sum11 = biasptr[1]; + float sum12 = biasptr[1]; + float sum13 = biasptr[1]; for (int q = 0; q < nn; q++) { - _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(kptr[0]), (v4f32)__msa_ld_w(tmpptr, 0)); + __builtin_prefetch(tmpptr + 16); + __builtin_prefetch(kptr + 8); + float k0 = kptr[0]; + float k1 = kptr[1]; + sum00 += tmpptr[0] * k0; + sum01 += tmpptr[1] * k0; + sum02 += tmpptr[2] * k0; + sum03 += tmpptr[3] * k0; + sum10 += tmpptr[0] * k1; + sum11 += tmpptr[1] * k1; + sum12 += tmpptr[2] * k1; + sum13 += tmpptr[3] * k1; tmpptr += 4; - kptr++; + kptr += 2; } - __msa_st_w((v4i32)_sum0, outptr0, 0); + outptr0[0] = sum00; + outptr0[1] = sum01; + outptr0[2] = sum02; + outptr0[3] = sum03; + outptr1[0] = sum10; + outptr1[1] = sum11; + outptr1[2] = sum12; + outptr1[3] = sum13; outptr0 += 4; + outptr1 += 4; } for (; i < size; i++) { const float* tmpptr = tmp.channel(i / 4 + i % 4); - const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); + const float* kptr = kernel.channel(p / 2); int nn = inch * maxk; // inch always > 0 - float sum0 = bias0; + float sum0 = biasptr[0]; + float sum1 = biasptr[1]; for (int q = 0; q < nn; q++) { + __builtin_prefetch(tmpptr + 4); + __builtin_prefetch(kptr + 8); sum0 += tmpptr[0] * kptr[0]; + sum1 += tmpptr[0] * kptr[1]; tmpptr++; - kptr++; + kptr += 2; } outptr0[0] = sum0; + outptr1[0] = sum1; outptr0++; + outptr1++; } } -#else // __mips_msa +#endif // __mips_msa + #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < outch; p++) + for (int p = remain_outch_start; p < outch; p++) { float* outptr0 = top_blob.channel(p); const float bias0 = bias ? bias[p] : 0.f; - for (int i = 0; i < size; i++) + int i = 0; + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 4); +#if __mips_msa + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const float* kptr = kernel.channel(p / 2 + p % 2); +#endif + + int nn = inch * maxk; // inch always > 0 + +#if __mips_msa + v4f32 _sum0 = __msa_fill_w_f32(bias0); + + for (int q = 0; q < nn; q++) + { + _sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(kptr[0]), (v4f32)__msa_ld_w(tmpptr, 0)); + tmpptr += 4; + kptr++; + } + + __msa_st_w((v4i32)_sum0, outptr0, 0); + + outptr0 += 4; +#else + float sum0 = bias0; + float sum1 = bias0; + float sum2 = bias0; + float sum3 = bias0; + + for (int q = 0; q < nn; q++) + { + __builtin_prefetch(tmpptr + 16); + __builtin_prefetch(kptr + 4); + sum0 += tmpptr[0] * kptr[0]; + sum1 += tmpptr[1] * kptr[0]; + sum2 += tmpptr[2] * kptr[0]; + sum3 += tmpptr[3] * kptr[0]; + tmpptr += 4; + kptr++; + } + + outptr0[0] = sum0; + outptr0[1] = sum1; + outptr0[2] = sum2; + outptr0[3] = sum3; + + outptr0 += 4; +#endif // __mips_msa + } + for (; i < size; i++) { - const float* tmpptr = tmp.channel(i); - const float* kptr = kernel.channel(p); + const float* tmpptr = tmp.channel(i / 4 + i % 4); +#if __mips_msa + const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4); +#else + const float* kptr = kernel.channel(p / 2 + p % 2); +#endif int nn = inch * maxk; // inch always > 0 @@ -390,7 +469,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat& outptr0++; } } -#endif // __mips_msa } static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h) @@ -403,8 +481,12 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma Mat kernel = _kernel.reshape(maxk, inch, outch); #if __mips_msa kernel_tm.create(8 * maxk, inch, outch / 8 + (outch % 8) / 4 + outch % 4); +#else + kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2); +#endif int q = 0; +#if __mips_msa for (; q + 7 < outch; q += 8) { const Mat k0 = kernel.channel(q); @@ -471,11 +553,38 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma } } } +#else + for (; q + 1 < outch; q += 2) + { + const Mat k0 = kernel.channel(q); + const Mat k1 = kernel.channel(q + 1); + + float* g00 = kernel_tm.channel(q / 2); + + for (int p = 0; p < inch; p++) + { + const float* k00 = k0.row(p); + const float* k10 = k1.row(p); + + for (int k = 0; k < maxk; k++) + { + g00[0] = k00[k]; + g00[1] = k10[k]; + + g00 += 2; + } + } + } +#endif // __mips_msa for (; q < outch; q++) { const Mat k0 = kernel.channel(q); +#if __mips_msa float* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4 + q % 4); +#else + float* g00 = kernel_tm.channel(q / 2 + q % 2); +#endif for (int p = 0; p < inch; p++) { @@ -489,9 +598,6 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma } } } -#else - kernel_tm = kernel; -#endif // __mips_msa } static void convolution_im2col_sgemm_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) diff --git a/src/layer/mips/convolution_sgemm_int8.h b/src/layer/mips/convolution_sgemm_int8.h index 297c4b1b0..ee39f6e8f 100644 --- a/src/layer/mips/convolution_sgemm_int8.h +++ b/src/layer/mips/convolution_sgemm_int8.h @@ -33,6 +33,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const tmp.create(maxk, inch / 4 + inch % 4, size, 4u, 4, opt.workspace_allocator); } else +#endif // __mips_msa { if (size >= 2) tmp.create(2 * maxk, inch, size / 2 + size % 2, 1u, 1, opt.workspace_allocator); @@ -51,6 +52,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const signed char* tmpptr = tmp.channel(i / 2); int q = 0; +#if __mips_msa for (; q + 3 < inch; q += 4) { const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; @@ -76,6 +78,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const img3 += size; } } +#endif // __mips_msa for (; q < inch; q++) { const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; @@ -100,6 +103,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const signed char* tmpptr = tmp.channel(i / 2 + i % 2); int q = 0; +#if __mips_msa for (; q + 3 < inch; q += 4) { const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; @@ -121,6 +125,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const img3 += size; } } +#endif // __mips_msa for (; q < inch; q++) { const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; @@ -136,37 +141,10 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const } } } -#else // __mips_msa - tmp.create(maxk, inch, size, 1u, 1, opt.workspace_allocator); - { - #pragma omp parallel for num_threads(opt.num_threads) - for (int i = 0; i < size; i++) - { - signed char* tmpptr = tmp.channel(i); - - int q = 0; - for (; q < inch; q++) - { - const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i; - - for (int k = 0; k < maxk; k++) - { - tmpptr[0] = img0[0]; - - tmpptr += 1; - - img0 += size; - } - } - } - } -#endif // __mips_msa - - int nn_outch = 0; - int remain_outch_start = 0; #if __mips_msa - nn_outch = outch >> 2; + int nn_outch = outch >> 2; + int remain_outch_start = nn_outch << 2; #pragma omp parallel for num_threads(opt.num_threads) for (int pp = 0; pp < nn_outch; pp++) @@ -414,8 +392,85 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const outptr3 += 1; } } +#else // __mips_msa + int nn_outch = outch >> 1; + int remain_outch_start = nn_outch << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_outch; pp++) + { + int p = pp * 2; + + int* outptr0 = top_blob.channel(p); + int* outptr1 = top_blob.channel(p + 1); + + int i = 0; + for (; i + 1 < size; i += 2) + { + const signed char* tmpptr = tmp.channel(i / 2); + const signed char* kptr = kernel.channel(p / 2); + + int nn1 = inch * maxk; + + int sum00 = 0; + int sum01 = 0; + int sum10 = 0; + int sum11 = 0; + + int j = 0; + for (; j < nn1; j++) + { + signed char val0 = tmpptr[0]; + signed char val1 = tmpptr[1]; + signed char w0 = kptr[0]; + signed char w1 = kptr[1]; + + sum00 += val0 * w0; + sum01 += val1 * w0; + sum10 += val0 * w1; + sum11 += val1 * w1; + + tmpptr += 2; + kptr += 2; + } - remain_outch_start += nn_outch << 2; + outptr0[0] = sum00; + outptr0[1] = sum01; + outptr1[0] = sum10; + outptr1[1] = sum11; + outptr0 += 2; + outptr1 += 2; + } + for (; i < size; i++) + { + const signed char* tmpptr = tmp.channel(i / 2 + i % 2); + const signed char* kptr = kernel.channel(p / 2); + + int nn1 = inch * maxk; + + int sum00 = 0; + int sum10 = 0; + + int j = 0; + for (; j < nn1; j++) + { + signed char val0 = tmpptr[0]; + signed char w0 = kptr[0]; + signed char w1 = kptr[1]; + + sum00 += val0 * w0; + sum10 += val0 * w1; + + tmpptr += 1; + kptr += 2; + } + + outptr0[0] = sum00; + outptr1[0] = sum10; + outptr0 += 1; + outptr1 += 1; + } + } #endif // __mips_msa #pragma omp parallel for num_threads(opt.num_threads) @@ -424,18 +479,22 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const int* outptr0 = top_blob.channel(p); int i = 0; -#if __mips_msa for (; i + 1 < size; i += 2) { const signed char* tmpptr = tmp.channel(i / 2); +#if __mips_msa const signed char* kptr = kernel.channel(p / 4 + p % 4); - - int nn4 = (inch / 4) * maxk; - int nn1 = (inch % 4) * maxk; +#else + const signed char* kptr = kernel.channel(p / 2 + p % 2); +#endif int sum0 = 0; int sum1 = 0; +#if __mips_msa + int nn4 = (inch / 4) * maxk; + int nn1 = (inch % 4) * maxk; + if (nn4 > 0) { v4i32 _sum0 = __msa_fill_w(0); @@ -467,6 +526,9 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const sum0 = _sum0[0] + _sum0[1] + _sum0[2] + _sum0[3]; sum1 = _sum1[0] + _sum1[1] + _sum1[2] + _sum1[3]; } +#else + int nn1 = inch * maxk; +#endif // __mips_msa int j = 0; for (; j < nn1; j++) @@ -489,13 +551,18 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const for (; i < size; i++) { const signed char* tmpptr = tmp.channel(i / 2 + i % 2); +#if __mips_msa const signed char* kptr = kernel.channel(p / 4 + p % 4); +#else + const signed char* kptr = kernel.channel(p / 2 + p % 2); +#endif + + int sum = 0; +#if __mips_msa int nn4 = (inch / 4) * maxk; int nn1 = (inch % 4) * maxk; - int sum = 0; - if (nn4 > 0) { v4i32 _sum = __msa_fill_w(0); @@ -520,31 +587,10 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const sum = _sum[0] + _sum[1] + _sum[2] + _sum[3]; } - - int j = 0; - for (; j < nn1; j++) - { - signed char val = tmpptr[0]; - signed char w = kptr[0]; - - sum += val * w; - - tmpptr += 1; - kptr += 1; - } - - outptr0[0] = sum; - outptr0 += 1; - } -#else // __mips_msa - for (; i < size; i++) - { - const signed char* tmpptr = tmp.channel(i); - const signed char* kptr = kernel.channel(p); - +#else int nn1 = inch * maxk; +#endif // __mips_msa - int sum = 0; int j = 0; for (; j < nn1; j++) { @@ -560,7 +606,6 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const outptr0[0] = sum; outptr0 += 1; } -#endif // __mips_msa } } @@ -568,11 +613,11 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne { const int maxk = kernel_w * kernel_h; -#if __mips_msa // interleave // src = maxk-inch-outch // dst = 4a-4b-maxk-inch/4a-outch/4b Mat kernel = _kernel.reshape(maxk, inch, outch); +#if __mips_msa if (outch >= 4) { if (inch >= 4) @@ -580,15 +625,26 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne else kernel_tm.create(4 * maxk, inch, outch / 4 + outch % 4, (size_t)1u); } +#else + if (outch >= 2) + { + kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2, (size_t)1u); + } +#endif // __mips_msa else { +#if __mips_msa if (inch >= 4) kernel_tm.create(4 * maxk, inch / 4 + inch % 4, outch, (size_t)1u); else +#endif // __mips_msa + { kernel_tm.create(1 * maxk, inch, outch, (size_t)1u); + } } int q = 0; +#if __mips_msa for (; q + 3 < outch; q += 4) { signed char* g00 = kernel_tm.channel(q / 4); @@ -603,9 +659,7 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne for (int j = 0; j < 4; j++) { const signed char* k00 = kernel.channel(q + i).row(p + j); - g00[0] = k00[k]; - g00++; } } @@ -618,20 +672,42 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne for (int i = 0; i < 4; i++) { const signed char* k00 = kernel.channel(q + i).row(p); - g00[0] = k00[k]; + g00++; + } + } + } + } +#else // __mips_msa + for (; q + 1 < outch; q += 2) + { + signed char* g00 = kernel_tm.channel(q / 2); + int p = 0; + for (; p < inch; p++) + { + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < 2; i++) + { + const signed char* k00 = kernel.channel(q + i).row(p); + g00[0] = k00[k]; g00++; } } } } - // TODO unroll 2 +#endif // __mips_msa for (; q < outch; q++) { +#if __mips_msa signed char* g00 = kernel_tm.channel(q / 4 + q % 4); +#else + signed char* g00 = kernel_tm.channel(q / 2 + q % 2); +#endif int p = 0; +#if __mips_msa for (; p + 3 < inch; p += 4) { for (int k = 0; k < maxk; k++) @@ -639,28 +715,22 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne for (int j = 0; j < 4; j++) { const signed char* k00 = kernel.channel(q).row(p + j); - g00[0] = k00[k]; - g00++; } } } +#endif // __mips_msa for (; p < inch; p++) { for (int k = 0; k < maxk; k++) { const signed char* k00 = kernel.channel(q).row(p); - g00[0] = k00[k]; - g00++; } } } -#else // __mips_msa - kernel_tm = _kernel.reshape(maxk, inch, outch); -#endif // __mips_msa } static void convolution_im2col_sgemm_int8_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) diff --git a/src/layer/mips/convolution_winograd_transform.h b/src/layer/mips/convolution_winograd_transform.h new file mode 100644 index 000000000..14578df51 --- /dev/null +++ b/src/layer/mips/convolution_winograd_transform.h @@ -0,0 +1,405 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv3x3s1_winograd43_transform_input_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt) +{ + const int w = bottom_blob.w; + const int h = bottom_blob.h; + const int inch = bottom_blob.c; + + const int w_tiles = (w - 2) / 4; + const int h_tiles = (h - 2) / 4; + const int tiles = w_tiles * h_tiles; + + // const float itm[6][6] = { + // {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f}, + // {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f}, + // {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f}, + // {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f}, + // {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f} + // }; + + // 0 = 4 * r00 - 5 * r02 + r04 + // 1 = -4 * (r01 + r02) + r04 + r03 + // 2 = 4 * (r01 - r02) + r04 - r03 + // 3 = -2 * (r01 - r03) + r04 - r02 + // 4 = 2 * (r01 - r03) + r04 - r02 + // 5 = 4 * r01 - 5 * r03 + r05 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < inch; q++) + { + const Mat img0 = bottom_blob.channel(q); + Mat img0_tm = bottom_blob_tm.channel(q); + + float tmp[6][6]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* r0 = img0.row(i * 4) + (j * 4); + + for (int m = 0; m < 6; m++) + { + float r00 = r0[0]; + float r01 = r0[1]; + float r02 = r0[2]; + float r03 = r0[3]; + float r04 = r0[4]; + float r05 = r0[5]; + + float tmp0m = 4 * r00 - 5 * r02 + r04; + float tmp1m = -4 * (r01 + r02) + r04 + r03; + float tmp2m = 4 * (r01 - r02) + r04 - r03; + float tmp3m = -2 * (r01 - r03) + r04 - r02; + float tmp4m = 2 * (r01 - r03) + r04 - r02; + float tmp5m = 4 * r01 - 5 * r03 + r05; + + tmp[0][m] = tmp0m; + tmp[1][m] = tmp1m; + tmp[2][m] = tmp2m; + tmp[3][m] = tmp3m; + tmp[4][m] = tmp4m; + tmp[5][m] = tmp5m; + + r0 += w; + } + + float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j); + float* r0_tm_1 = r0_tm_0 + tiles; + float* r0_tm_2 = r0_tm_0 + tiles * 2; + float* r0_tm_3 = r0_tm_0 + tiles * 3; + float* r0_tm_4 = r0_tm_0 + tiles * 4; + float* r0_tm_5 = r0_tm_0 + tiles * 5; + + for (int m = 0; m < 6; m++) + { + float tmp00 = tmp[m][0]; + float tmp01 = tmp[m][1]; + float tmp02 = tmp[m][2]; + float tmp03 = tmp[m][3]; + float tmp04 = tmp[m][4]; + float tmp05 = tmp[m][5]; + + float r0tm0 = 4 * tmp00 - 5 * tmp02 + tmp04; + float r0tm1 = -4 * (tmp01 + tmp02) + tmp04 + tmp03; + float r0tm2 = 4 * (tmp01 - tmp02) + tmp04 - tmp03; + float r0tm3 = -2 * (tmp01 - tmp03) + tmp04 - tmp02; + float r0tm4 = 2 * (tmp01 - tmp03) + tmp04 - tmp02; + float r0tm5 = 4 * tmp01 - 5 * tmp03 + tmp05; + + r0_tm_0[0] = r0tm0; + r0_tm_1[0] = r0tm1; + r0_tm_2[0] = r0tm2; + r0_tm_3[0] = r0tm3; + r0_tm_4[0] = r0tm4; + r0_tm_5[0] = r0tm5; + + r0_tm_0 += tiles * 6; + r0_tm_1 += tiles * 6; + r0_tm_2 += tiles * 6; + r0_tm_3 += tiles * 6; + r0_tm_4 += tiles * 6; + r0_tm_5 += tiles * 6; + } + } + } + } +} + +static void conv3x3s1_winograd43_transform_output_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt) +{ + const int outw = top_blob.w; + const int outh = top_blob.h; + const int outch = top_blob.c; + + const int w_tiles = outw / 4; + const int h_tiles = outh / 4; + const int tiles = w_tiles * h_tiles; + + const float* biasptr = bias; + + // const float otm[4][6] = { + // {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f}, + // {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f} + // }; + + // 0 = r00 + (r01 + r02) + (r03 + r04) + // 1 = (r01 - r02) + (r03 - r04) * 2 + // 2 = (r01 + r02) + (r03 + r04) * 4 + // 3 = r05 + (r01 - r02) + (r03 - r04) * 8 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + const Mat out0_tm = top_blob_tm.channel(p); + Mat out0 = top_blob.channel(p); + + float bias0 = biasptr ? biasptr[p] : 0.f; + + float tmp[4][6]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j); + const float* output0_tm_1 = output0_tm_0 + tiles; + const float* output0_tm_2 = output0_tm_0 + tiles * 2; + const float* output0_tm_3 = output0_tm_0 + tiles * 3; + const float* output0_tm_4 = output0_tm_0 + tiles * 4; + const float* output0_tm_5 = output0_tm_0 + tiles * 5; + + float* output0 = out0.row(i * 4) + (j * 4); + + for (int m = 0; m < 6; m++) + { + float out0tm0 = output0_tm_0[0]; + float out0tm1 = output0_tm_1[0]; + float out0tm2 = output0_tm_2[0]; + float out0tm3 = output0_tm_3[0]; + float out0tm4 = output0_tm_4[0]; + float out0tm5 = output0_tm_5[0]; + + float tmp02a = out0tm1 + out0tm2; + float tmp13a = out0tm1 - out0tm2; + + float tmp02b = out0tm3 + out0tm4; + float tmp13b = out0tm3 - out0tm4; + + float tmp0m = out0tm0 + tmp02a + tmp02b; + float tmp1m = tmp13a + tmp13b * 2; + float tmp2m = tmp02a + tmp02b * 4; + float tmp3m = out0tm5 + tmp13a + tmp13b * 8; + + tmp[0][m] = tmp0m; + tmp[1][m] = tmp1m; + tmp[2][m] = tmp2m; + tmp[3][m] = tmp3m; + + output0_tm_0 += tiles * 6; + output0_tm_1 += tiles * 6; + output0_tm_2 += tiles * 6; + output0_tm_3 += tiles * 6; + output0_tm_4 += tiles * 6; + output0_tm_5 += tiles * 6; + } + + for (int m = 0; m < 4; m++) + { + float tmp00 = tmp[m][0]; + float tmp01 = tmp[m][1]; + float tmp02 = tmp[m][2]; + float tmp03 = tmp[m][3]; + float tmp04 = tmp[m][4]; + float tmp05 = tmp[m][5]; + + float tmp02a = tmp01 + tmp02; + float tmp13a = tmp01 - tmp02; + + float tmp02b = tmp03 + tmp04; + float tmp13b = tmp03 - tmp04; + + float out00 = bias0 + tmp00 + tmp02a + tmp02b; + float out01 = bias0 + tmp13a + tmp13b * 2; + float out02 = bias0 + tmp02a + tmp02b * 4; + float out03 = bias0 + tmp05 + tmp13a + tmp13b * 8; + + output0[0] = out00; + output0[1] = out01; + output0[2] = out02; + output0[3] = out03; + + output0 += outw; + } + } + } + } +} + +static void conv3x3s1_winograd23_transform_input_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt) +{ + const int w = bottom_blob.w; + const int h = bottom_blob.h; + const int inch = bottom_blob.c; + + const int w_tiles = (w - 2) / 2; + const int h_tiles = (h - 2) / 2; + const int tiles = w_tiles * h_tiles; + + // const float itm[4][4] = { + // {1.0f, 0.0f, -1.0f, 0.0f}, + // {0.0f, 1.0f, 1.00f, 0.0f}, + // {0.0f, -1.0f, 1.00f, 0.0f}, + // {0.0f, -1.0f, 0.00f, 1.0f} + // }; + + // 0 = r00 - r02 + // 1 = r01 + r02 + // 2 = r02 - r01 + // 3 = r03 - r01 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < inch; q++) + { + const Mat img0 = bottom_blob.channel(q); + Mat img0_tm = bottom_blob_tm.channel(q); + + float tmp[4][4]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* r0 = img0.row(i * 2) + (j * 2); + + for (int m = 0; m < 4; m++) + { + float r00 = r0[0]; + float r01 = r0[1]; + float r02 = r0[2]; + float r03 = r0[3]; + + float tmp0m = r00 - r02; + float tmp1m = r01 + r02; + float tmp2m = r02 - r01; + float tmp3m = r03 - r01; + + tmp[0][m] = tmp0m; + tmp[1][m] = tmp1m; + tmp[2][m] = tmp2m; + tmp[3][m] = tmp3m; + + r0 += w; + } + + float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j); + float* r0_tm_1 = r0_tm_0 + tiles; + float* r0_tm_2 = r0_tm_0 + tiles * 2; + float* r0_tm_3 = r0_tm_0 + tiles * 3; + + for (int m = 0; m < 4; m++) + { + float tmp00 = tmp[m][0]; + float tmp01 = tmp[m][1]; + float tmp02 = tmp[m][2]; + float tmp03 = tmp[m][3]; + + float r0tm0 = tmp00 - tmp02; + float r0tm1 = tmp01 + tmp02; + float r0tm2 = tmp02 - tmp01; + float r0tm3 = tmp03 - tmp01; + + r0_tm_0[0] = r0tm0; + r0_tm_1[0] = r0tm1; + r0_tm_2[0] = r0tm2; + r0_tm_3[0] = r0tm3; + + r0_tm_0 += tiles * 4; + r0_tm_1 += tiles * 4; + r0_tm_2 += tiles * 4; + r0_tm_3 += tiles * 4; + } + } + } + } +} + +static void conv3x3s1_winograd23_transform_output_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt) +{ + const int outw = top_blob.w; + const int outh = top_blob.h; + const int outch = top_blob.c; + + const int w_tiles = outw / 2; + const int h_tiles = outh / 2; + const int tiles = w_tiles * h_tiles; + + const float* biasptr = bias; + + // const float otm[2][4] = { + // {1.0f, 1.0f, 1.0f, 0.0f}, + // {0.0f, 1.0f, -1.0f, 1.0f} + // }; + + // 0 = r00 + r01 + r02 + // 1 = r01 - r02 + r03 + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + const Mat out0_tm = top_blob_tm.channel(p); + Mat out0 = top_blob.channel(p); + + float bias0 = biasptr ? biasptr[p] : 0.f; + + float tmp[2][4]; + + // tile + for (int i = 0; i < h_tiles; i++) + { + for (int j = 0; j < w_tiles; j++) + { + const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j); + const float* output0_tm_1 = output0_tm_0 + tiles; + const float* output0_tm_2 = output0_tm_0 + tiles * 2; + const float* output0_tm_3 = output0_tm_0 + tiles * 3; + + float* output0 = out0.row(i * 2) + (j * 2); + + for (int m = 0; m < 4; m++) + { + float out0tm0 = output0_tm_0[0]; + float out0tm1 = output0_tm_1[0]; + float out0tm2 = output0_tm_2[0]; + float out0tm3 = output0_tm_3[0]; + + float tmp0m = out0tm0 + out0tm1 + out0tm2; + float tmp1m = out0tm1 - out0tm2 + out0tm3; + + tmp[0][m] = tmp0m; + tmp[1][m] = tmp1m; + + output0_tm_0 += tiles * 4; + output0_tm_1 += tiles * 4; + output0_tm_2 += tiles * 4; + output0_tm_3 += tiles * 4; + } + + for (int m = 0; m < 2; m++) + { + float tmp00 = tmp[m][0]; + float tmp01 = tmp[m][1]; + float tmp02 = tmp[m][2]; + float tmp03 = tmp[m][3]; + + float out00 = bias0 + tmp00 + tmp01 + tmp02; + float out01 = bias0 + tmp01 - tmp02 + tmp03; + + output0[0] = out00; + output0[1] = out01; + + output0 += outw; + } + } + } + } +}