nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

mips optimization for convolution sgemm (#3853) 

* mips optimization for convolution sgemm

* mips optimization for general convolution int8 gemm

* mips optmization for convolution winograd pack1

* preload magic
tags/20220701
nihui GitHub 4 years ago
parent
a5bcc8895f
commit
c3adbcf9f3
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 2095 additions and 114 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1370
    -0
      src/layer/mips/convolution_3x3.h
  2. +29
    -0
      src/layer/mips/convolution_mips.cpp
  3. +1
    -0
      src/layer/mips/convolution_mips.h
  4. +148
    -42
      src/layer/mips/convolution_sgemm.h
  5. +142
    -72
      src/layer/mips/convolution_sgemm_int8.h
  6. +405
    -0
      src/layer/mips/convolution_winograd_transform.h

+ 1370
- 0
src/layer/mips/convolution_3x3.h
File diff suppressed because it is too large
View File


+ 29
- 0
src/layer/mips/convolution_mips.cpp View File

@@ -30,7 +30,9 @@
namespace ncnn {

#include "convolution_sgemm.h"
#include "convolution_winograd_transform.h"
#include "convolution_1x1.h"
#include "convolution_3x3.h"

#if NCNN_INT8
#include "convolution_sgemm_int8.h"
@@ -189,6 +191,17 @@ int Convolution_mips::create_pipeline(const Option& opt)
{
convolution_im2col_sgemm_transform_kernel_msa(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h);
}
if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
if (num_input >= 16 && num_output >= 16)
{
conv3x3s1_winograd43_transform_kernel_msa(weight_data, weight_winograd43_data, num_input, num_output, opt);
}
else
{
conv3x3s1_winograd23_transform_kernel_msa(weight_data, weight_winograd23_data, num_input, num_output, opt);
}
}
else if (opt.use_sgemm_convolution)
{
convolution_im2col_sgemm_transform_kernel_msa(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h);
@@ -395,6 +408,22 @@ int Convolution_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
activation->forward_inplace(top_blob, opt);
}
}
else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
if (num_input >= 16 && num_output >= 16)
{
conv3x3s1_winograd43_msa(bottom_blob_bordered, top_blob, weight_winograd43_data, bias_data, opt);
}
else
{
conv3x3s1_winograd23_msa(bottom_blob_bordered, top_blob, weight_winograd23_data, bias_data, opt);
}

if (activation)
{
activation->forward_inplace(top_blob, opt);
}
}
else if (opt.use_sgemm_convolution)
{
convolution_im2col_sgemm_msa(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);


+ 1
- 0
src/layer/mips/convolution_mips.h View File

@@ -41,6 +41,7 @@ public:
Layer* activation;

Mat weight_sgemm_data;
Mat weight_winograd23_data;
Mat weight_winograd43_data;
Mat weight_winograd63_data;



+ 148
- 42
src/layer/mips/convolution_sgemm.h View File

@@ -26,7 +26,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat&

// permute
Mat tmp;
#if __mips_msa
if (size >= 4)
tmp.create(4 * maxk, inch, size / 4 + size % 4, 4u, 1, opt.workspace_allocator);
else
@@ -47,7 +46,14 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat&

for (int k = 0; k < maxk; k++)
{
#if __mips_msa
__msa_st_w(__msa_ld_w(img0, 0), tmpptr, 0);
#else
tmpptr[0] = img0[0];
tmpptr[1] = img0[1];
tmpptr[2] = img0[2];
tmpptr[3] = img0[3];
#endif
img0 += size;
tmpptr += 4;
}
@@ -74,28 +80,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat&
}
}
}
#else // __mips_msa
tmp.create(maxk, inch, size, 4u, 1, opt.workspace_allocator);
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int i = 0; i < size; i++)
{
float* tmpptr = tmp.channel(i);

for (int q = 0; q < inch; q++)
{
const float* img0 = (const float*)bottom_im2col.channel(q) + i;

for (int k = 0; k < maxk; k++)
{
tmpptr[0] = img0[0];
img0 += size;
tmpptr += 1;
}
}
}
}
#endif // __mips_msa

#if __mips_msa
int nn_outch = outch >> 3;
@@ -311,68 +295,163 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat&
}

remain_outch_start += nn_outch << 2;
#else // __mips_msa
int nn_outch = outch >> 1;
int remain_outch_start = nn_outch << 1;

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_outch_start; p < outch; p++)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 2;

float* outptr0 = top_blob.channel(p);
float* outptr1 = top_blob.channel(p + 1);

const float bias0 = bias ? bias[p] : 0.f;
const float zeros[2] = {0.f, 0.f};
const float* biasptr = bias ? bias + p : zeros;

int i = 0;
for (; i + 3 < size; i += 4)
{
const float* tmpptr = tmp.channel(i / 4);
const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4);
const float* kptr = kernel.channel(p / 2);

int nn = inch * maxk; // inch always > 0

v4f32 _sum0 = __msa_fill_w_f32(bias0);
float sum00 = biasptr[0];
float sum01 = biasptr[0];
float sum02 = biasptr[0];
float sum03 = biasptr[0];
float sum10 = biasptr[1];
float sum11 = biasptr[1];
float sum12 = biasptr[1];
float sum13 = biasptr[1];

for (int q = 0; q < nn; q++)
{
_sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(kptr[0]), (v4f32)__msa_ld_w(tmpptr, 0));
__builtin_prefetch(tmpptr + 16);
__builtin_prefetch(kptr + 8);
float k0 = kptr[0];
float k1 = kptr[1];
sum00 += tmpptr[0] * k0;
sum01 += tmpptr[1] * k0;
sum02 += tmpptr[2] * k0;
sum03 += tmpptr[3] * k0;
sum10 += tmpptr[0] * k1;
sum11 += tmpptr[1] * k1;
sum12 += tmpptr[2] * k1;
sum13 += tmpptr[3] * k1;
tmpptr += 4;
kptr++;
kptr += 2;
}

__msa_st_w((v4i32)_sum0, outptr0, 0);
outptr0[0] = sum00;
outptr0[1] = sum01;
outptr0[2] = sum02;
outptr0[3] = sum03;
outptr1[0] = sum10;
outptr1[1] = sum11;
outptr1[2] = sum12;
outptr1[3] = sum13;

outptr0 += 4;
outptr1 += 4;
}
for (; i < size; i++)
{
const float* tmpptr = tmp.channel(i / 4 + i % 4);
const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4);
const float* kptr = kernel.channel(p / 2);

int nn = inch * maxk; // inch always > 0

float sum0 = bias0;
float sum0 = biasptr[0];
float sum1 = biasptr[1];

for (int q = 0; q < nn; q++)
{
__builtin_prefetch(tmpptr + 4);
__builtin_prefetch(kptr + 8);
sum0 += tmpptr[0] * kptr[0];
sum1 += tmpptr[0] * kptr[1];
tmpptr++;
kptr++;
kptr += 2;
}

outptr0[0] = sum0;
outptr1[0] = sum1;

outptr0++;
outptr1++;
}
}
#else // __mips_msa
#endif // __mips_msa

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
for (int p = remain_outch_start; p < outch; p++)
{
float* outptr0 = top_blob.channel(p);

const float bias0 = bias ? bias[p] : 0.f;

for (int i = 0; i < size; i++)
int i = 0;
for (; i + 3 < size; i += 4)
{
const float* tmpptr = tmp.channel(i / 4);
#if __mips_msa
const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
const float* kptr = kernel.channel(p / 2 + p % 2);
#endif

int nn = inch * maxk; // inch always > 0

#if __mips_msa
v4f32 _sum0 = __msa_fill_w_f32(bias0);

for (int q = 0; q < nn; q++)
{
_sum0 = __msa_fmadd_w(_sum0, __msa_fill_w_f32(kptr[0]), (v4f32)__msa_ld_w(tmpptr, 0));
tmpptr += 4;
kptr++;
}

__msa_st_w((v4i32)_sum0, outptr0, 0);

outptr0 += 4;
#else
float sum0 = bias0;
float sum1 = bias0;
float sum2 = bias0;
float sum3 = bias0;

for (int q = 0; q < nn; q++)
{
__builtin_prefetch(tmpptr + 16);
__builtin_prefetch(kptr + 4);
sum0 += tmpptr[0] * kptr[0];
sum1 += tmpptr[1] * kptr[0];
sum2 += tmpptr[2] * kptr[0];
sum3 += tmpptr[3] * kptr[0];
tmpptr += 4;
kptr++;
}

outptr0[0] = sum0;
outptr0[1] = sum1;
outptr0[2] = sum2;
outptr0[3] = sum3;

outptr0 += 4;
#endif // __mips_msa
}
for (; i < size; i++)
{
const float* tmpptr = tmp.channel(i);
const float* kptr = kernel.channel(p);
const float* tmpptr = tmp.channel(i / 4 + i % 4);
#if __mips_msa
const float* kptr = kernel.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
const float* kptr = kernel.channel(p / 2 + p % 2);
#endif

int nn = inch * maxk; // inch always > 0

@@ -390,7 +469,6 @@ static void im2col_sgemm_msa(const Mat& bottom_im2col, Mat& top_blob, const Mat&
outptr0++;
}
}
#endif // __mips_msa
}

static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_w, int kernel_h)
@@ -403,8 +481,12 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma
Mat kernel = _kernel.reshape(maxk, inch, outch);
#if __mips_msa
kernel_tm.create(8 * maxk, inch, outch / 8 + (outch % 8) / 4 + outch % 4);
#else
kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2);
#endif

int q = 0;
#if __mips_msa
for (; q + 7 < outch; q += 8)
{
const Mat k0 = kernel.channel(q);
@@ -471,11 +553,38 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma
}
}
}
#else
for (; q + 1 < outch; q += 2)
{
const Mat k0 = kernel.channel(q);
const Mat k1 = kernel.channel(q + 1);

float* g00 = kernel_tm.channel(q / 2);

for (int p = 0; p < inch; p++)
{
const float* k00 = k0.row(p);
const float* k10 = k1.row(p);

for (int k = 0; k < maxk; k++)
{
g00[0] = k00[k];
g00[1] = k10[k];

g00 += 2;
}
}
}
#endif // __mips_msa
for (; q < outch; q++)
{
const Mat k0 = kernel.channel(q);

#if __mips_msa
float* g00 = kernel_tm.channel(q / 8 + (q % 8) / 4 + q % 4);
#else
float* g00 = kernel_tm.channel(q / 2 + q % 2);
#endif

for (int p = 0; p < inch; p++)
{
@@ -489,9 +598,6 @@ static void convolution_im2col_sgemm_transform_kernel_msa(const Mat& _kernel, Ma
}
}
}
#else
kernel_tm = kernel;
#endif // __mips_msa
}

static void convolution_im2col_sgemm_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt)


+ 142
- 72
src/layer/mips/convolution_sgemm_int8.h View File

@@ -33,6 +33,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
tmp.create(maxk, inch / 4 + inch % 4, size, 4u, 4, opt.workspace_allocator);
}
else
#endif // __mips_msa
{
if (size >= 2)
tmp.create(2 * maxk, inch, size / 2 + size % 2, 1u, 1, opt.workspace_allocator);
@@ -51,6 +52,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
signed char* tmpptr = tmp.channel(i / 2);

int q = 0;
#if __mips_msa
for (; q + 3 < inch; q += 4)
{
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i;
@@ -76,6 +78,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
img3 += size;
}
}
#endif // __mips_msa
for (; q < inch; q++)
{
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i;
@@ -100,6 +103,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
signed char* tmpptr = tmp.channel(i / 2 + i % 2);

int q = 0;
#if __mips_msa
for (; q + 3 < inch; q += 4)
{
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i;
@@ -121,6 +125,7 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
img3 += size;
}
}
#endif // __mips_msa
for (; q < inch; q++)
{
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i;
@@ -136,37 +141,10 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
}
}
}
#else // __mips_msa
tmp.create(maxk, inch, size, 1u, 1, opt.workspace_allocator);
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int i = 0; i < size; i++)
{
signed char* tmpptr = tmp.channel(i);

int q = 0;
for (; q < inch; q++)
{
const signed char* img0 = (const signed char*)bottom_im2col.channel(q) + i;

for (int k = 0; k < maxk; k++)
{
tmpptr[0] = img0[0];

tmpptr += 1;

img0 += size;
}
}
}
}
#endif // __mips_msa

int nn_outch = 0;
int remain_outch_start = 0;

#if __mips_msa
nn_outch = outch >> 2;
int nn_outch = outch >> 2;
int remain_outch_start = nn_outch << 2;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
@@ -414,8 +392,85 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
outptr3 += 1;
}
}
#else // __mips_msa
int nn_outch = outch >> 1;
int remain_outch_start = nn_outch << 1;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 2;

int* outptr0 = top_blob.channel(p);
int* outptr1 = top_blob.channel(p + 1);

int i = 0;
for (; i + 1 < size; i += 2)
{
const signed char* tmpptr = tmp.channel(i / 2);
const signed char* kptr = kernel.channel(p / 2);

int nn1 = inch * maxk;

int sum00 = 0;
int sum01 = 0;
int sum10 = 0;
int sum11 = 0;

int j = 0;
for (; j < nn1; j++)
{
signed char val0 = tmpptr[0];
signed char val1 = tmpptr[1];
signed char w0 = kptr[0];
signed char w1 = kptr[1];

sum00 += val0 * w0;
sum01 += val1 * w0;
sum10 += val0 * w1;
sum11 += val1 * w1;

tmpptr += 2;
kptr += 2;
}

remain_outch_start += nn_outch << 2;
outptr0[0] = sum00;
outptr0[1] = sum01;
outptr1[0] = sum10;
outptr1[1] = sum11;
outptr0 += 2;
outptr1 += 2;
}
for (; i < size; i++)
{
const signed char* tmpptr = tmp.channel(i / 2 + i % 2);
const signed char* kptr = kernel.channel(p / 2);

int nn1 = inch * maxk;

int sum00 = 0;
int sum10 = 0;

int j = 0;
for (; j < nn1; j++)
{
signed char val0 = tmpptr[0];
signed char w0 = kptr[0];
signed char w1 = kptr[1];

sum00 += val0 * w0;
sum10 += val0 * w1;

tmpptr += 1;
kptr += 2;
}

outptr0[0] = sum00;
outptr1[0] = sum10;
outptr0 += 1;
outptr1 += 1;
}
}
#endif // __mips_msa

#pragma omp parallel for num_threads(opt.num_threads)
@@ -424,18 +479,22 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
int* outptr0 = top_blob.channel(p);

int i = 0;
#if __mips_msa
for (; i + 1 < size; i += 2)
{
const signed char* tmpptr = tmp.channel(i / 2);
#if __mips_msa
const signed char* kptr = kernel.channel(p / 4 + p % 4);
int nn4 = (inch / 4) * maxk;
int nn1 = (inch % 4) * maxk;
#else
const signed char* kptr = kernel.channel(p / 2 + p % 2);
#endif

int sum0 = 0;
int sum1 = 0;

#if __mips_msa
int nn4 = (inch / 4) * maxk;
int nn1 = (inch % 4) * maxk;

if (nn4 > 0)
{
v4i32 _sum0 = __msa_fill_w(0);
@@ -467,6 +526,9 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
sum0 = _sum0[0] + _sum0[1] + _sum0[2] + _sum0[3];
sum1 = _sum1[0] + _sum1[1] + _sum1[2] + _sum1[3];
}
#else
int nn1 = inch * maxk;
#endif // __mips_msa

int j = 0;
for (; j < nn1; j++)
@@ -489,13 +551,18 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
for (; i < size; i++)
{
const signed char* tmpptr = tmp.channel(i / 2 + i % 2);
#if __mips_msa
const signed char* kptr = kernel.channel(p / 4 + p % 4);
#else
const signed char* kptr = kernel.channel(p / 2 + p % 2);
#endif

int sum = 0;

#if __mips_msa
int nn4 = (inch / 4) * maxk;
int nn1 = (inch % 4) * maxk;

int sum = 0;

if (nn4 > 0)
{
v4i32 _sum = __msa_fill_w(0);
@@ -520,31 +587,10 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const

sum = _sum[0] + _sum[1] + _sum[2] + _sum[3];
}

int j = 0;
for (; j < nn1; j++)
{
signed char val = tmpptr[0];
signed char w = kptr[0];

sum += val * w;

tmpptr += 1;
kptr += 1;
}

outptr0[0] = sum;
outptr0 += 1;
}
#else // __mips_msa
for (; i < size; i++)
{
const signed char* tmpptr = tmp.channel(i);
const signed char* kptr = kernel.channel(p);

#else
int nn1 = inch * maxk;
#endif // __mips_msa

int sum = 0;
int j = 0;
for (; j < nn1; j++)
{
@@ -560,7 +606,6 @@ static void im2col_sgemm_int8_msa(const Mat& bottom_im2col, Mat& top_blob, const
outptr0[0] = sum;
outptr0 += 1;
}
#endif // __mips_msa
}
}

@@ -568,11 +613,11 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne
{
const int maxk = kernel_w * kernel_h;

#if __mips_msa
// interleave
// src = maxk-inch-outch
// dst = 4a-4b-maxk-inch/4a-outch/4b
Mat kernel = _kernel.reshape(maxk, inch, outch);
#if __mips_msa
if (outch >= 4)
{
if (inch >= 4)
@@ -580,15 +625,26 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne
else
kernel_tm.create(4 * maxk, inch, outch / 4 + outch % 4, (size_t)1u);
}
#else
if (outch >= 2)
{
kernel_tm.create(2 * maxk, inch, outch / 2 + outch % 2, (size_t)1u);
}
#endif // __mips_msa
else
{
#if __mips_msa
if (inch >= 4)
kernel_tm.create(4 * maxk, inch / 4 + inch % 4, outch, (size_t)1u);
else
#endif // __mips_msa
{
kernel_tm.create(1 * maxk, inch, outch, (size_t)1u);
}
}

int q = 0;
#if __mips_msa
for (; q + 3 < outch; q += 4)
{
signed char* g00 = kernel_tm.channel(q / 4);
@@ -603,9 +659,7 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne
for (int j = 0; j < 4; j++)
{
const signed char* k00 = kernel.channel(q + i).row<const signed char>(p + j);

g00[0] = k00[k];

g00++;
}
}
@@ -618,20 +672,42 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne
for (int i = 0; i < 4; i++)
{
const signed char* k00 = kernel.channel(q + i).row<const signed char>(p);

g00[0] = k00[k];
g00++;
}
}
}
}
#else // __mips_msa
for (; q + 1 < outch; q += 2)
{
signed char* g00 = kernel_tm.channel(q / 2);

int p = 0;
for (; p < inch; p++)
{
for (int k = 0; k < maxk; k++)
{
for (int i = 0; i < 2; i++)
{
const signed char* k00 = kernel.channel(q + i).row<const signed char>(p);
g00[0] = k00[k];
g00++;
}
}
}
}
// TODO unroll 2
#endif // __mips_msa
for (; q < outch; q++)
{
#if __mips_msa
signed char* g00 = kernel_tm.channel(q / 4 + q % 4);
#else
signed char* g00 = kernel_tm.channel(q / 2 + q % 2);
#endif

int p = 0;
#if __mips_msa
for (; p + 3 < inch; p += 4)
{
for (int k = 0; k < maxk; k++)
@@ -639,28 +715,22 @@ static void convolution_im2col_sgemm_transform_kernel_int8_msa(const Mat& _kerne
for (int j = 0; j < 4; j++)
{
const signed char* k00 = kernel.channel(q).row<const signed char>(p + j);

g00[0] = k00[k];

g00++;
}
}
}
#endif // __mips_msa
for (; p < inch; p++)
{
for (int k = 0; k < maxk; k++)
{
const signed char* k00 = kernel.channel(q).row<const signed char>(p);

g00[0] = k00[k];

g00++;
}
}
}
#else // __mips_msa
kernel_tm = _kernel.reshape(maxk, inch, outch);
#endif // __mips_msa
}

static void convolution_im2col_sgemm_int8_msa(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt)


+ 405
- 0
src/layer/mips/convolution_winograd_transform.h View File

@@ -0,0 +1,405 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_input_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
const int w = bottom_blob.w;
const int h = bottom_blob.h;
const int inch = bottom_blob.c;

const int w_tiles = (w - 2) / 4;
const int h_tiles = (h - 2) / 4;
const int tiles = w_tiles * h_tiles;

// const float itm[6][6] = {
// {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f},
// {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f},
// {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f},
// {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f}
// };

// 0 = 4 * r00 - 5 * r02 + r04
// 1 = -4 * (r01 + r02) + r04 + r03
// 2 = 4 * (r01 - r02) + r04 - r03
// 3 = -2 * (r01 - r03) + r04 - r02
// 4 = 2 * (r01 - r03) + r04 - r02
// 5 = 4 * r01 - 5 * r03 + r05

#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < inch; q++)
{
const Mat img0 = bottom_blob.channel(q);
Mat img0_tm = bottom_blob_tm.channel(q);

float tmp[6][6];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* r0 = img0.row(i * 4) + (j * 4);

for (int m = 0; m < 6; m++)
{
float r00 = r0[0];
float r01 = r0[1];
float r02 = r0[2];
float r03 = r0[3];
float r04 = r0[4];
float r05 = r0[5];

float tmp0m = 4 * r00 - 5 * r02 + r04;
float tmp1m = -4 * (r01 + r02) + r04 + r03;
float tmp2m = 4 * (r01 - r02) + r04 - r03;
float tmp3m = -2 * (r01 - r03) + r04 - r02;
float tmp4m = 2 * (r01 - r03) + r04 - r02;
float tmp5m = 4 * r01 - 5 * r03 + r05;

tmp[0][m] = tmp0m;
tmp[1][m] = tmp1m;
tmp[2][m] = tmp2m;
tmp[3][m] = tmp3m;
tmp[4][m] = tmp4m;
tmp[5][m] = tmp5m;

r0 += w;
}

float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j);
float* r0_tm_1 = r0_tm_0 + tiles;
float* r0_tm_2 = r0_tm_0 + tiles * 2;
float* r0_tm_3 = r0_tm_0 + tiles * 3;
float* r0_tm_4 = r0_tm_0 + tiles * 4;
float* r0_tm_5 = r0_tm_0 + tiles * 5;

for (int m = 0; m < 6; m++)
{
float tmp00 = tmp[m][0];
float tmp01 = tmp[m][1];
float tmp02 = tmp[m][2];
float tmp03 = tmp[m][3];
float tmp04 = tmp[m][4];
float tmp05 = tmp[m][5];

float r0tm0 = 4 * tmp00 - 5 * tmp02 + tmp04;
float r0tm1 = -4 * (tmp01 + tmp02) + tmp04 + tmp03;
float r0tm2 = 4 * (tmp01 - tmp02) + tmp04 - tmp03;
float r0tm3 = -2 * (tmp01 - tmp03) + tmp04 - tmp02;
float r0tm4 = 2 * (tmp01 - tmp03) + tmp04 - tmp02;
float r0tm5 = 4 * tmp01 - 5 * tmp03 + tmp05;

r0_tm_0[0] = r0tm0;
r0_tm_1[0] = r0tm1;
r0_tm_2[0] = r0tm2;
r0_tm_3[0] = r0tm3;
r0_tm_4[0] = r0tm4;
r0_tm_5[0] = r0tm5;

r0_tm_0 += tiles * 6;
r0_tm_1 += tiles * 6;
r0_tm_2 += tiles * 6;
r0_tm_3 += tiles * 6;
r0_tm_4 += tiles * 6;
r0_tm_5 += tiles * 6;
}
}
}
}
}

static void conv3x3s1_winograd43_transform_output_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt)
{
const int outw = top_blob.w;
const int outh = top_blob.h;
const int outch = top_blob.c;

const int w_tiles = outw / 4;
const int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

const float* biasptr = bias;

// const float otm[4][6] = {
// {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f},
// {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f}
// };

// 0 = r00 + (r01 + r02) + (r03 + r04)
// 1 = (r01 - r02) + (r03 - r04) * 2
// 2 = (r01 + r02) + (r03 + r04) * 4
// 3 = r05 + (r01 - r02) + (r03 - r04) * 8

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
const Mat out0_tm = top_blob_tm.channel(p);
Mat out0 = top_blob.channel(p);

float bias0 = biasptr ? biasptr[p] : 0.f;

float tmp[4][6];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j);
const float* output0_tm_1 = output0_tm_0 + tiles;
const float* output0_tm_2 = output0_tm_0 + tiles * 2;
const float* output0_tm_3 = output0_tm_0 + tiles * 3;
const float* output0_tm_4 = output0_tm_0 + tiles * 4;
const float* output0_tm_5 = output0_tm_0 + tiles * 5;

float* output0 = out0.row(i * 4) + (j * 4);

for (int m = 0; m < 6; m++)
{
float out0tm0 = output0_tm_0[0];
float out0tm1 = output0_tm_1[0];
float out0tm2 = output0_tm_2[0];
float out0tm3 = output0_tm_3[0];
float out0tm4 = output0_tm_4[0];
float out0tm5 = output0_tm_5[0];

float tmp02a = out0tm1 + out0tm2;
float tmp13a = out0tm1 - out0tm2;

float tmp02b = out0tm3 + out0tm4;
float tmp13b = out0tm3 - out0tm4;

float tmp0m = out0tm0 + tmp02a + tmp02b;
float tmp1m = tmp13a + tmp13b * 2;
float tmp2m = tmp02a + tmp02b * 4;
float tmp3m = out0tm5 + tmp13a + tmp13b * 8;

tmp[0][m] = tmp0m;
tmp[1][m] = tmp1m;
tmp[2][m] = tmp2m;
tmp[3][m] = tmp3m;

output0_tm_0 += tiles * 6;
output0_tm_1 += tiles * 6;
output0_tm_2 += tiles * 6;
output0_tm_3 += tiles * 6;
output0_tm_4 += tiles * 6;
output0_tm_5 += tiles * 6;
}

for (int m = 0; m < 4; m++)
{
float tmp00 = tmp[m][0];
float tmp01 = tmp[m][1];
float tmp02 = tmp[m][2];
float tmp03 = tmp[m][3];
float tmp04 = tmp[m][4];
float tmp05 = tmp[m][5];

float tmp02a = tmp01 + tmp02;
float tmp13a = tmp01 - tmp02;

float tmp02b = tmp03 + tmp04;
float tmp13b = tmp03 - tmp04;

float out00 = bias0 + tmp00 + tmp02a + tmp02b;
float out01 = bias0 + tmp13a + tmp13b * 2;
float out02 = bias0 + tmp02a + tmp02b * 4;
float out03 = bias0 + tmp05 + tmp13a + tmp13b * 8;

output0[0] = out00;
output0[1] = out01;
output0[2] = out02;
output0[3] = out03;

output0 += outw;
}
}
}
}
}

static void conv3x3s1_winograd23_transform_input_msa(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
const int w = bottom_blob.w;
const int h = bottom_blob.h;
const int inch = bottom_blob.c;

const int w_tiles = (w - 2) / 2;
const int h_tiles = (h - 2) / 2;
const int tiles = w_tiles * h_tiles;

// const float itm[4][4] = {
// {1.0f, 0.0f, -1.0f, 0.0f},
// {0.0f, 1.0f, 1.00f, 0.0f},
// {0.0f, -1.0f, 1.00f, 0.0f},
// {0.0f, -1.0f, 0.00f, 1.0f}
// };

// 0 = r00 - r02
// 1 = r01 + r02
// 2 = r02 - r01
// 3 = r03 - r01

#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < inch; q++)
{
const Mat img0 = bottom_blob.channel(q);
Mat img0_tm = bottom_blob_tm.channel(q);

float tmp[4][4];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* r0 = img0.row(i * 2) + (j * 2);

for (int m = 0; m < 4; m++)
{
float r00 = r0[0];
float r01 = r0[1];
float r02 = r0[2];
float r03 = r0[3];

float tmp0m = r00 - r02;
float tmp1m = r01 + r02;
float tmp2m = r02 - r01;
float tmp3m = r03 - r01;

tmp[0][m] = tmp0m;
tmp[1][m] = tmp1m;
tmp[2][m] = tmp2m;
tmp[3][m] = tmp3m;

r0 += w;
}

float* r0_tm_0 = (float*)img0_tm + (i * w_tiles + j);
float* r0_tm_1 = r0_tm_0 + tiles;
float* r0_tm_2 = r0_tm_0 + tiles * 2;
float* r0_tm_3 = r0_tm_0 + tiles * 3;

for (int m = 0; m < 4; m++)
{
float tmp00 = tmp[m][0];
float tmp01 = tmp[m][1];
float tmp02 = tmp[m][2];
float tmp03 = tmp[m][3];

float r0tm0 = tmp00 - tmp02;
float r0tm1 = tmp01 + tmp02;
float r0tm2 = tmp02 - tmp01;
float r0tm3 = tmp03 - tmp01;

r0_tm_0[0] = r0tm0;
r0_tm_1[0] = r0tm1;
r0_tm_2[0] = r0tm2;
r0_tm_3[0] = r0tm3;

r0_tm_0 += tiles * 4;
r0_tm_1 += tiles * 4;
r0_tm_2 += tiles * 4;
r0_tm_3 += tiles * 4;
}
}
}
}
}

static void conv3x3s1_winograd23_transform_output_msa(const Mat& top_blob_tm, Mat& top_blob, const Mat& bias, const Option& opt)
{
const int outw = top_blob.w;
const int outh = top_blob.h;
const int outch = top_blob.c;

const int w_tiles = outw / 2;
const int h_tiles = outh / 2;
const int tiles = w_tiles * h_tiles;

const float* biasptr = bias;

// const float otm[2][4] = {
// {1.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 1.0f}
// };

// 0 = r00 + r01 + r02
// 1 = r01 - r02 + r03

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
const Mat out0_tm = top_blob_tm.channel(p);
Mat out0 = top_blob.channel(p);

float bias0 = biasptr ? biasptr[p] : 0.f;

float tmp[2][4];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const float* output0_tm_0 = (const float*)out0_tm + (i * w_tiles + j);
const float* output0_tm_1 = output0_tm_0 + tiles;
const float* output0_tm_2 = output0_tm_0 + tiles * 2;
const float* output0_tm_3 = output0_tm_0 + tiles * 3;

float* output0 = out0.row(i * 2) + (j * 2);

for (int m = 0; m < 4; m++)
{
float out0tm0 = output0_tm_0[0];
float out0tm1 = output0_tm_1[0];
float out0tm2 = output0_tm_2[0];
float out0tm3 = output0_tm_3[0];

float tmp0m = out0tm0 + out0tm1 + out0tm2;
float tmp1m = out0tm1 - out0tm2 + out0tm3;

tmp[0][m] = tmp0m;
tmp[1][m] = tmp1m;

output0_tm_0 += tiles * 4;
output0_tm_1 += tiles * 4;
output0_tm_2 += tiles * 4;
output0_tm_3 += tiles * 4;
}

for (int m = 0; m < 2; m++)
{
float tmp00 = tmp[m][0];
float tmp01 = tmp[m][1];
float tmp02 = tmp[m][2];
float tmp03 = tmp[m][3];

float out00 = bias0 + tmp00 + tmp01 + tmp02;
float out01 = bias0 + tmp01 - tmp02 + tmp03;

output0[0] = out00;
output0[1] = out01;

output0 += outw;
}
}
}
}
}

Write Preview
Loading…
Cancel
Save