nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

x86 and arm optimization for convolution1d packed unified elempack (#4615)

tags/20230517
nihui GitHub 3 years ago
parent
868ea52bea
commit
1133a18ca8
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 7289 additions and 1926 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +10
    -472
      src/layer/arm/convolution1d_arm.cpp
  2. +1
    -8
      src/layer/arm/convolution1d_arm.h
  3. +5
    -658
      src/layer/arm/convolution1d_arm_asimdhp.cpp
  4. +1272
    -0
      src/layer/arm/convolution1d_packed.h
  5. +1311
    -0
      src/layer/arm/convolution1d_packed_bf16s.h
  6. +1848
    -0
      src/layer/arm/convolution1d_packed_fp16s.h
  7. +2
    -4
      src/layer/arm/convolution_packed_fp16s.h
  8. +2702
    -0
      src/layer/x86/convolution1d_packed.h
  9. +8
    -516
      src/layer/x86/convolution1d_x86.cpp
  10. +1
    -1
      src/layer/x86/convolution1d_x86.h
  11. +87
    -237
      src/layer/x86/convolution_packed.h
  12. +42
    -30
      tests/test_convolution1d.cpp

+ 10
- 472
src/layer/arm/convolution1d_arm.cpp View File

@@ -26,6 +26,11 @@

namespace ncnn {

#include "convolution1d_packed.h"
#if NCNN_BF16
#include "convolution1d_packed_bf16s.h"
#endif // NCNN_BF16

Convolution1D_arm::Convolution1D_arm()
{
#if __ARM_NEON
@@ -61,47 +66,7 @@ int Convolution1D_arm::create_pipeline(const Option& opt)

const int num_input = weight_data_size / kernel_w / num_output;

int elempack = 1;
int out_elempack = 1;

#if __ARM_NEON
if (opt.use_packing_layout)
{
elempack = num_input % 4 == 0 ? 4 : 1;
out_elempack = num_output % 4 == 0 ? 4 : 1;
}
#endif

// src = kw-inch-outch
// dst = pb-pa-kw-inch/pa-outch/pb
{
Mat weight_data_r2 = weight_data.reshape(kernel_w, num_input, num_output);

weight_data_packed.create(kernel_w, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
float* g00 = weight_data_packed.channel(q / out_elempack);

for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
{
for (int k = 0; k < kernel_w; k++)
{
for (int i = 0; i < elempack; i++)
{
for (int j = 0; j < out_elempack; j++)
{
const float* k00 = weight_data_r2.channel(q + j).row(p + i);

g00[0] = k00[k];

g00++;
}
}
}
}
}
}
convolution1d_transform_kernel_packed(weight_data, weight_data_tm, num_input, num_output, kernel_w);

return 0;
}
@@ -131,7 +96,6 @@ int Convolution1D_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
#endif

int w = bottom_blob.w;
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

@@ -143,7 +107,6 @@ int Convolution1D_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
return -100;

w = bottom_blob_bordered.w;
h = bottom_blob_bordered.h;

int out_elempack = 1;
#if __ARM_NEON
@@ -161,199 +124,7 @@ int Convolution1D_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
if (top_blob.empty())
return -100;

#if __ARM_NEON
if (elempack == 4 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vld1q_f32(sptr);

float32x4_t _w0 = vld1q_f32(kptr);
float32x4_t _w1 = vld1q_f32(kptr + 4);
float32x4_t _w2 = vld1q_f32(kptr + 8);
float32x4_t _w3 = vld1q_f32(kptr + 12);

#if __aarch64__
_sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);
_sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);
_sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);
_sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);
#else
_sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);
_sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);
_sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);
_sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);
#endif

sptr += dilation_w * 4;
kptr += 16;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f32(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 1 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vdupq_n_f32(sptr[0]);
float32x4_t _w = vld1q_f32(kptr);
_sum = vmlaq_f32(_sum, _val, _w);

sptr += dilation_w;
kptr += 4;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f32(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++) // 29.23
{
float32x4_t _val = vld1q_f32(sptr);
float32x4_t _w = vld1q_f32(kptr);
float32x4_t _s4 = vmulq_f32(_val, _w);
#if __aarch64__
sum += vaddvq_f32(_s4); // dot
#else
float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));
_ss = vpadd_f32(_ss, _ss);
sum += vget_lane_f32(_ss, 0);
#endif

sptr += dilation_w * 4;
kptr += 4;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}
#endif // __ARM_NEON

if (elempack == 1 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const float* kptr = (const float*)weight_data + kernel_w * h * p;

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float val = sptr[0];
float wt = kptr[0];
sum += val * wt;

sptr += dilation_w;
kptr += 1;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}
convolution1d_packed(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, dilation_w, stride_w, activation_type, activation_params, opt);

return 0;
}
@@ -460,50 +231,11 @@ int Convolution1D_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector
}

#if NCNN_BF16
int Convolution1D_arm::create_pipeline_bf16s(const Option& opt)
int Convolution1D_arm::create_pipeline_bf16s(const Option& /*opt*/)
{
const int num_input = weight_data_size / kernel_w / num_output;

int elempack = 1;
int out_elempack = 1;
#if __ARM_NEON
if (opt.use_packing_layout)
{
elempack = num_input % 4 == 0 ? 4 : 1;
out_elempack = num_output % 4 == 0 ? 4 : 1;
}
#endif

// src = kw-inch-outch
// dst = pb-pa-kw-inch/pa-outch/pb
{
Mat weight_data_r2 = weight_data.reshape(kernel_w, num_input, num_output);

weight_data_bf16.create(kernel_w, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
unsigned short* g00 = weight_data_bf16.channel(q / out_elempack);

for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
{
for (int k = 0; k < kernel_w; k++)
{
for (int i = 0; i < elempack; i++)
{
for (int j = 0; j < out_elempack; j++)
{
const float* k00 = weight_data_r2.channel(q + j).row(p + i);

g00[0] = float32_to_bfloat16(k00[k]);

g00++;
}
}
}
}
}
}
convolution1d_transform_kernel_packed_bf16s(weight_data, weight_data_tm, num_input, num_output, kernel_w);

return 0;
}
@@ -511,7 +243,6 @@ int Convolution1D_arm::create_pipeline_bf16s(const Option& opt)
int Convolution1D_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
int w = bottom_blob.w;
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

@@ -523,7 +254,6 @@ int Convolution1D_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
return -100;

w = bottom_blob_bordered.w;
h = bottom_blob_bordered.h;

int out_elempack = 1;
#if __ARM_NEON
@@ -541,199 +271,7 @@ int Convolution1D_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
if (top_blob.empty())
return -100;

#if __ARM_NEON
if (elempack == 4 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
unsigned short* outptr = top_blob.row<unsigned short>(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const unsigned short* kptr = weight_data_bf16.channel(p);

for (int q = 0; q < h; q++)
{
const unsigned short* sptr = bottom_blob_bordered.row<const unsigned short>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = bfloat2float(vld1_u16(sptr));

float32x4_t _w0 = bfloat2float(vld1_u16(kptr));
float32x4_t _w1 = bfloat2float(vld1_u16(kptr + 4));
float32x4_t _w2 = bfloat2float(vld1_u16(kptr + 8));
float32x4_t _w3 = bfloat2float(vld1_u16(kptr + 12));

#if __aarch64__
_sum = vmlaq_laneq_f32(_sum, _w0, _val, 0);
_sum = vmlaq_laneq_f32(_sum, _w1, _val, 1);
_sum = vmlaq_laneq_f32(_sum, _w2, _val, 2);
_sum = vmlaq_laneq_f32(_sum, _w3, _val, 3);
#else
_sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0);
_sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1);
_sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0);
_sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1);
#endif

sptr += dilation_w * 4;
kptr += 16;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_u16(outptr, float2bfloat(_sum));
outptr += 4;
}
}
}
}

if (elempack == 1 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
unsigned short* outptr = top_blob.row<unsigned short>(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const unsigned short* kptr = weight_data_bf16.channel(p);

for (int q = 0; q < h; q++)
{
const unsigned short* sptr = bottom_blob_bordered.row<const unsigned short>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vdupq_n_f32(bfloat16_to_float32(sptr[0]));
float32x4_t _w = bfloat2float(vld1_u16(kptr));
_sum = vmlaq_f32(_sum, _val, _w);

sptr += dilation_w;
kptr += 4;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_u16(outptr, float2bfloat(_sum));
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
unsigned short* outptr = top_blob.row<unsigned short>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const unsigned short* kptr = weight_data_bf16.channel(p);

for (int q = 0; q < h; q++)
{
const unsigned short* sptr = bottom_blob_bordered.row<const unsigned short>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = bfloat2float(vld1_u16(sptr));
float32x4_t _w = bfloat2float(vld1_u16(kptr));
float32x4_t _s4 = vmulq_f32(_val, _w);
#if __aarch64__
sum += vaddvq_f32(_s4); // dot
#else
float32x2_t _ss = vadd_f32(vget_low_f32(_s4), vget_high_f32(_s4));
_ss = vpadd_f32(_ss, _ss);
sum += vget_lane_f32(_ss, 0);
#endif

sptr += dilation_w * 4;
kptr += 4;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = float32_to_bfloat16(sum);
}
}
}
}
#endif // __ARM_NEON

if (elempack == 1 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
unsigned short* outptr = top_blob.row<unsigned short>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const unsigned short* kptr = weight_data_bf16.channel(p);

for (int q = 0; q < h; q++)
{
const unsigned short* sptr = bottom_blob_bordered.row<unsigned short>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float val = bfloat16_to_float32(sptr[0]);
float wt = bfloat16_to_float32(kptr[0]);
sum += val * wt;

sptr += dilation_w;
kptr += 1;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = float32_to_bfloat16(sum);
}
}
}
}
convolution1d_packed_bf16s(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, dilation_w, stride_w, activation_type, activation_params, opt);

return 0;
}


+ 1
- 8
src/layer/arm/convolution1d_arm.h View File

@@ -43,17 +43,10 @@ protected:
#endif

public:
// pack4
Mat weight_data_packed;
Mat weight_data_tm;

// fp16
Mat weight_data_fp16;
Mat bias_data_fp16;

#if NCNN_BF16
// bf16
Mat weight_data_bf16;
#endif
};

} // namespace ncnn


+ 5
- 658
src/layer/arm/convolution1d_arm_asimdhp.cpp View File

@@ -26,49 +26,13 @@
namespace ncnn {

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
#include "convolution1d_packed_fp16s.h"

int Convolution1D_arm::create_pipeline_fp16s(const Option& opt)
{
const int num_input = weight_data_size / kernel_w / num_output;

int elempack = 1;
int out_elempack = 1;

if (opt.use_packing_layout)
{
elempack = opt.use_fp16_arithmetic && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
}

// src = kw-inch-outch
// dst = pb-pa-kw-inch/pa-outch/pb
{
Mat weight_data_r2 = weight_data.reshape(kernel_w, num_input, num_output);

weight_data_fp16.create(kernel_w, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
__fp16* g00 = weight_data_fp16.channel(q / out_elempack);

for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
{
for (int k = 0; k < kernel_w; k++)
{
for (int i = 0; i < elempack; i++)
{
for (int j = 0; j < out_elempack; j++)
{
const float* k00 = weight_data_r2.channel(q + j).row(p + i);

g00[0] = (__fp16)k00[k];

g00++;
}
}
}
}
}
}
convolution1d_transform_kernel_packed_fp16s(weight_data, weight_data_tm, num_input, num_output, kernel_w);

ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);

@@ -78,7 +42,6 @@ int Convolution1D_arm::create_pipeline_fp16s(const Option& opt)
int Convolution1D_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
int w = bottom_blob.w;
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

@@ -90,7 +53,6 @@ int Convolution1D_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
return -100;

w = bottom_blob_bordered.w;
h = bottom_blob_bordered.h;

int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;
size_t out_elemsize = elemsize / elempack * out_elempack;
@@ -102,185 +64,7 @@ int Convolution1D_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
if (top_blob.empty())
return -100;

if (elempack == 4 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));

float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr));
float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4));
float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8));
float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12));

_sum = vfmaq_laneq_f32(_sum, _w0, _val, 0);
_sum = vfmaq_laneq_f32(_sum, _w1, _val, 1);
_sum = vfmaq_laneq_f32(_sum, _w2, _val, 2);
_sum = vfmaq_laneq_f32(_sum, _w3, _val, 3);

sptr += dilation_w * 4;
kptr += 16;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, vcvt_f16_f32(_sum));
outptr += 4;
}
}
}
}

if (elempack == 1 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vcvt_f32_f16(vdup_n_f16(sptr[0]));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
_sum = vfmaq_f32(_sum, _val, _w);

sptr += dilation_w;
kptr += 4;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, vcvt_f16_f32(_sum));
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
float32x4_t _s4 = vmulq_f32(_val, _w);

sum += vaddvq_f32(_s4); // dot

sptr += dilation_w * 4;
kptr += 4;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = (__fp16)sum;
}
}
}
}

if (elempack == 1 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float val = (float)sptr[0];
float w = (float)kptr[0];
sum += val * w;

sptr += dilation_w;
kptr += 1;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = (__fp16)sum;
}
}
}
}
convolution1d_packed_fp16s(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, dilation_w, stride_w, activation_type, activation_params, opt);

return 0;
}
@@ -288,7 +72,6 @@ int Convolution1D_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
int Convolution1D_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
int w = bottom_blob.w;
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

@@ -300,7 +83,6 @@ int Convolution1D_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
return -100;

w = bottom_blob_bordered.w;
h = bottom_blob_bordered.h;

int out_elempack = 1;
if (opt.use_packing_layout)
@@ -316,442 +98,7 @@ int Convolution1D_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
if (top_blob.empty())
return -100;

if (elempack == 8 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++)
{
float16x8_t _val = vld1q_f16(sptr);

float16x8_t _w0 = vld1q_f16(kptr);
float16x8_t _w1 = vld1q_f16(kptr + 8);
float16x8_t _w2 = vld1q_f16(kptr + 16);
float16x8_t _w3 = vld1q_f16(kptr + 24);
float16x8_t _w4 = vld1q_f16(kptr + 32);
float16x8_t _w5 = vld1q_f16(kptr + 40);
float16x8_t _w6 = vld1q_f16(kptr + 48);
float16x8_t _w7 = vld1q_f16(kptr + 56);

_sum = vfmaq_laneq_f16(_sum, _w0, _val, 0);
_sum = vfmaq_laneq_f16(_sum, _w1, _val, 1);
_sum = vfmaq_laneq_f16(_sum, _w2, _val, 2);
_sum = vfmaq_laneq_f16(_sum, _w3, _val, 3);
_sum = vfmaq_laneq_f16(_sum, _w4, _val, 4);
_sum = vfmaq_laneq_f16(_sum, _w5, _val, 5);
_sum = vfmaq_laneq_f16(_sum, _w6, _val, 6);
_sum = vfmaq_laneq_f16(_sum, _w7, _val, 7);

sptr += dilation_w * 8;
kptr += 64;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 1 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float16x8_t _val = vdupq_n_f16(sptr[0]);
float16x8_t _w = vld1q_f16(kptr);
_sum = vfmaq_f16(_sum, _val, _w);

sptr += dilation_w;
kptr += 8;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 4 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float16x4_t _val = vld1_f16(sptr);

float16x8_t _w0 = vld1q_f16(kptr);
float16x8_t _w1 = vld1q_f16(kptr + 8);
float16x8_t _w2 = vld1q_f16(kptr + 16);
float16x8_t _w3 = vld1q_f16(kptr + 24);

_sum = vfmaq_lane_f16(_sum, _w0, _val, 0);
_sum = vfmaq_lane_f16(_sum, _w1, _val, 1);
_sum = vfmaq_lane_f16(_sum, _w2, _val, 2);
_sum = vfmaq_lane_f16(_sum, _w3, _val, 3);

sptr += dilation_w * 4;
kptr += 32;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 8 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = ((const __fp16*)bias_data_fp16)[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++)
{
float16x8_t _val = vld1q_f16(sptr);
float16x8_t _w = vld1q_f16(kptr);
float16x8_t _s8 = vmulq_f16(_val, _w);

float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8));
sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot

sptr += dilation_w * 8;
kptr += 8;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}

if (elempack == 8 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x4_t _sum = vdup_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++)
{
float16x8_t _val = vld1q_f16(sptr);

float16x4_t _w0 = vld1_f16(kptr);
float16x4_t _w1 = vld1_f16(kptr + 4);
float16x4_t _w2 = vld1_f16(kptr + 8);
float16x4_t _w3 = vld1_f16(kptr + 12);
float16x4_t _w4 = vld1_f16(kptr + 16);
float16x4_t _w5 = vld1_f16(kptr + 20);
float16x4_t _w6 = vld1_f16(kptr + 24);
float16x4_t _w7 = vld1_f16(kptr + 28);

_sum = vfma_laneq_f16(_sum, _w0, _val, 0);
_sum = vfma_laneq_f16(_sum, _w1, _val, 1);
_sum = vfma_laneq_f16(_sum, _w2, _val, 2);
_sum = vfma_laneq_f16(_sum, _w3, _val, 3);
_sum = vfma_laneq_f16(_sum, _w4, _val, 4);
_sum = vfma_laneq_f16(_sum, _w5, _val, 5);
_sum = vfma_laneq_f16(_sum, _w6, _val, 6);
_sum = vfma_laneq_f16(_sum, _w7, _val, 7);

sptr += dilation_w * 8;
kptr += 32;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x4_t _sum = vdup_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float16x4_t _val = vld1_f16(sptr);

float16x4_t _w0 = vld1_f16(kptr);
float16x4_t _w1 = vld1_f16(kptr + 4);
float16x4_t _w2 = vld1_f16(kptr + 8);
float16x4_t _w3 = vld1_f16(kptr + 12);

_sum = vfma_lane_f16(_sum, _w0, _val, 0);
_sum = vfma_lane_f16(_sum, _w1, _val, 1);
_sum = vfma_lane_f16(_sum, _w2, _val, 2);
_sum = vfma_lane_f16(_sum, _w3, _val, 3);

sptr += dilation_w * 4;
kptr += 16;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 1 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float16x4_t _sum = vdup_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float16x4_t _val = vdup_n_f16(sptr[0]);
float16x4_t _w = vld1_f16(kptr);
_sum = vfma_f16(_sum, _val, _w);

sptr += dilation_w;
kptr += 4;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = ((const __fp16*)bias_data_fp16)[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
float16x4_t _val = vld1_f16(sptr);
float16x4_t _w = vld1_f16(kptr);
float16x4_t _s4 = vmul_f16(_val, _w);

sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot

sptr += dilation_w * 4;
kptr += 4;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}

if (elempack == 1 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
__fp16* outptr = top_blob.row<__fp16>(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

for (int q = 0; q < h; q++)
{
const __fp16* sptr = bottom_blob_bordered.row<const __fp16>(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float val = (float)sptr[0];
float w = (float)kptr[0];
sum += val * w;

sptr += dilation_w;
kptr += 1;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = (__fp16)sum;
}
}
}
}
convolution1d_packed_fp16sa(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, kernel_w, dilation_w, stride_w, activation_type, activation_params, opt);

return 0;
}


+ 1272
- 0
src/layer/arm/convolution1d_packed.h
File diff suppressed because it is too large
View File


+ 1311
- 0
src/layer/arm/convolution1d_packed_bf16s.h
File diff suppressed because it is too large
View File


+ 1848
- 0
src/layer/arm/convolution1d_packed_fp16s.h
File diff suppressed because it is too large
View File


+ 2
- 4
src/layer/arm/convolution_packed_fp16s.h View File

@@ -32,8 +32,7 @@ static void convolution_transform_kernel_packed_fp16s(const Mat& kernel, Mat& ke
else
kernel_tm.create(8 * maxk, inch, outch / 8 + (outch % 8) / 4 + (outch % 4) / 2 + outch % 2, (size_t)2u);
}
else
if (outch >= 4)
else if (outch >= 4)
{
if (inch >= 8)
kernel_tm.create(4 * 8 * maxk, inch / 8 + (inch % 8) / 4 + (inch % 4) / 2 + inch % 2, outch / 4 + (outch % 4) / 2 + outch % 2, (size_t)2u);
@@ -44,8 +43,7 @@ static void convolution_transform_kernel_packed_fp16s(const Mat& kernel, Mat& ke
else
kernel_tm.create(4 * maxk, inch, outch / 4 + (outch % 4) / 2 + outch % 2, (size_t)2u);
}
else
if (outch >= 2)
else if (outch >= 2)
{
if (inch >= 8)
kernel_tm.create(2 * 8 * maxk, inch / 8 + (inch % 8) / 4 + (inch % 4) / 2 + inch % 2, outch / 2 + outch % 2, (size_t)2u);


+ 2702
- 0
src/layer/x86/convolution1d_packed.h
File diff suppressed because it is too large
View File


+ 8
- 516
src/layer/x86/convolution1d_x86.cpp View File

@@ -25,6 +25,8 @@

namespace ncnn {

#include "convolution1d_packed.h"

Convolution1D_x86::Convolution1D_x86()
{
#if __SSE2__
@@ -32,59 +34,14 @@ Convolution1D_x86::Convolution1D_x86()
#endif // __SSE2__
}

int Convolution1D_x86::create_pipeline(const Option& opt)
int Convolution1D_x86::create_pipeline(const Option& /*opt*/)
{
if (dynamic_weight)
return 0;

int num_input = weight_data_size / kernel_w / num_output;

int elempack = 1;
int out_elempack = 1;

#if __SSE2__
if (opt.use_packing_layout)
{
#if __AVX__
elempack = num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
out_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
#else
elempack = num_input % 4 == 0 ? 4 : 1;
out_elempack = num_output % 4 == 0 ? 4 : 1;
#endif
}
#endif // __SSE2__

// src = kw-inch-outch
// dst = pb-pa-kw-inch/pa-outch/pb
{
Mat weight_data_r2 = weight_data.reshape(kernel_w, num_input, num_output);

weight_data_packed.create(kernel_w, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
float* g00 = weight_data_packed.channel(q / out_elempack);

for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
{
for (int k = 0; k < kernel_w; k++)
{
for (int i = 0; i < elempack; i++)
{
for (int j = 0; j < out_elempack; j++)
{
const float* k00 = weight_data_r2.channel(q + j).row(p + i);

g00[0] = k00[k];

g00++;
}
}
}
}
}
}
convolution1d_transform_kernel_packed(weight_data, weight_data_tm, num_input, num_output, kernel_w);

return 0;
}
@@ -97,25 +54,9 @@ int Convolution1D_x86::destroy_pipeline(const Option& /*opt*/)
int Convolution1D_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
int w = bottom_blob.w;
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

#if __AVX512F__
if (elempack == 16)
{
Mat tmp;
convert_packing(bottom_blob, tmp, 8, opt);

Mat tmpout;
forward(tmp, tmpout, opt);

convert_packing(tmpout, top_blob, 16, opt);

return 0;
}
#endif // __AVX512F__

const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;

Mat bottom_blob_bordered;
@@ -124,13 +65,14 @@ int Convolution1D_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
return -100;

w = bottom_blob_bordered.w;
h = bottom_blob_bordered.h;

int out_elempack = 1;
#if __SSE2__
if (opt.use_packing_layout)
{
#if __AVX__
#if __AVX512F__
out_elempack = num_output % 16 == 0 ? 16 : num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
#elif __AVX__
out_elempack = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
#else
out_elempack = num_output % 4 == 0 ? 4 : 1;
@@ -146,457 +88,7 @@ int Convolution1D_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
if (top_blob.empty())
return -100;

#if __SSE2__
#if __AVX__
if (elempack == 8 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m256 _sum = _mm256_set1_ps(0.f);

if (bias_term)
{
_sum = _mm256_loadu_ps(((const float*)bias_data) + p * 8);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++)
{
__m256 _val0 = _mm256_broadcast_ss(sptr);
__m256 _val1 = _mm256_broadcast_ss(sptr + 1);
__m256 _val2 = _mm256_broadcast_ss(sptr + 2);
__m256 _val3 = _mm256_broadcast_ss(sptr + 3);
__m256 _val4 = _mm256_broadcast_ss(sptr + 4);
__m256 _val5 = _mm256_broadcast_ss(sptr + 5);
__m256 _val6 = _mm256_broadcast_ss(sptr + 6);
__m256 _val7 = _mm256_broadcast_ss(sptr + 7);

__m256 _w0 = _mm256_loadu_ps(kptr);
__m256 _w1 = _mm256_loadu_ps(kptr + 8);
__m256 _w2 = _mm256_loadu_ps(kptr + 16);
__m256 _w3 = _mm256_loadu_ps(kptr + 24);
__m256 _w4 = _mm256_loadu_ps(kptr + 32);
__m256 _w5 = _mm256_loadu_ps(kptr + 40);
__m256 _w6 = _mm256_loadu_ps(kptr + 48);
__m256 _w7 = _mm256_loadu_ps(kptr + 56);

_mm256_comp_fmadd_ps8(_sum,
_val0, _val1, _val2, _val3, _val4, _val5, _val6, _val7,
_w0, _w1, _w2, _w3, _w4, _w5, _w6, _w7);

sptr += dilation_w * 8;
kptr += 64;
}
}

_sum = activation_avx(_sum, activation_type, activation_params);

_mm256_storeu_ps(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 1 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m256 _sum = _mm256_set1_ps(0.f);

if (bias_term)
{
_sum = _mm256_loadu_ps(((const float*)bias_data) + p * 8);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
__m256 _val = _mm256_set1_ps(sptr[0]);
__m256 _w = _mm256_loadu_ps(kptr);
_sum = _mm256_comp_fmadd_ps(_val, _w, _sum);

sptr += dilation_w;
kptr += 8;
}
}

_sum = activation_avx(_sum, activation_type, activation_params);

_mm256_storeu_ps(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 4 && out_elempack == 8)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m256 _sum = _mm256_set1_ps(0.f);

if (bias_term)
{
_sum = _mm256_loadu_ps((const float*)bias_data + p * 8);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
__m256 _val0 = _mm256_broadcast_ss(sptr);
__m256 _val1 = _mm256_broadcast_ss(sptr + 1);
__m256 _val2 = _mm256_broadcast_ss(sptr + 2);
__m256 _val3 = _mm256_broadcast_ss(sptr + 3);

__m256 _w0 = _mm256_loadu_ps(kptr);
_sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum);
__m256 _w1 = _mm256_loadu_ps(kptr + 8);
_sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum);
__m256 _w2 = _mm256_loadu_ps(kptr + 16);
_sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum);
__m256 _w3 = _mm256_loadu_ps(kptr + 24);
_sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum);

sptr += dilation_w * 4;
kptr += 32;
}
}

_sum = activation_avx(_sum, activation_type, activation_params);

_mm256_storeu_ps(outptr, _sum);
outptr += 8;
}
}
}
}

if (elempack == 8 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const float* kptr = weight_data_packed.channel(p);

__m256 _sum8 = _mm256_set1_ps(0);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++) // 29.23
{
__m256 _val = _mm256_loadu_ps(sptr);
__m256 _w = _mm256_loadu_ps(kptr);
__m256 _s8 = _mm256_mul_ps(_val, _w);
_sum8 = _mm256_add_ps(_sum8, _s8);

sptr += dilation_w * 8;
kptr += 8;
}
}
sum += _mm256_reduce_add_ps(_sum8); // dot
sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}

if (elempack == 8 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m128 _sum = _mm_set1_ps(0.f);

if (bias_term)
{
_sum = _mm_loadu_ps((const float*)bias_data + p * 4);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 8;

for (int k = 0; k < kernel_w; k++)
{
__m128 _val0 = _mm_broadcast_ss(sptr);
__m128 _val1 = _mm_broadcast_ss(sptr + 1);
__m128 _val2 = _mm_broadcast_ss(sptr + 2);
__m128 _val3 = _mm_broadcast_ss(sptr + 3);
__m128 _val4 = _mm_broadcast_ss(sptr + 4);
__m128 _val5 = _mm_broadcast_ss(sptr + 5);
__m128 _val6 = _mm_broadcast_ss(sptr + 6);
__m128 _val7 = _mm_broadcast_ss(sptr + 7);

__m128 _w0 = _mm_loadu_ps(kptr);
_sum = _mm_comp_fmadd_ps(_val0, _w0, _sum);
__m128 _w1 = _mm_loadu_ps(kptr + 4);
_sum = _mm_comp_fmadd_ps(_val1, _w1, _sum);
__m128 _w2 = _mm_loadu_ps(kptr + 8);
_sum = _mm_comp_fmadd_ps(_val2, _w2, _sum);
__m128 _w3 = _mm_loadu_ps(kptr + 12);
_sum = _mm_comp_fmadd_ps(_val3, _w3, _sum);
__m128 _w4 = _mm_loadu_ps(kptr + 16);
_sum = _mm_comp_fmadd_ps(_val4, _w4, _sum);
__m128 _w5 = _mm_loadu_ps(kptr + 20);
_sum = _mm_comp_fmadd_ps(_val5, _w5, _sum);
__m128 _w6 = _mm_loadu_ps(kptr + 24);
_sum = _mm_comp_fmadd_ps(_val6, _w6, _sum);
__m128 _w7 = _mm_loadu_ps(kptr + 28);
_sum = _mm_comp_fmadd_ps(_val7, _w7, _sum);

sptr += dilation_w * 8;
kptr += 32;
}
}

_sum = activation_sse(_sum, activation_type, activation_params);

_mm_storeu_ps(outptr, _sum);
outptr += 4;
}
}
}
}
#endif

if (elempack == 4 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m128 _sum = _mm_set1_ps(0.f);

if (bias_term)
{
_sum = _mm_loadu_ps((const float*)bias_data + p * 4);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
__m128 _val0 = _mm_set1_ps(sptr[0]);
__m128 _val1 = _mm_set1_ps(sptr[1]);
__m128 _val2 = _mm_set1_ps(sptr[2]);
__m128 _val3 = _mm_set1_ps(sptr[3]);

__m128 _w0 = _mm_loadu_ps(kptr);
_sum = _mm_add_ps(_mm_mul_ps(_val0, _w0), _sum);
__m128 _w1 = _mm_loadu_ps(kptr + 4);
_sum = _mm_add_ps(_mm_mul_ps(_val1, _w1), _sum);
__m128 _w2 = _mm_loadu_ps(kptr + 8);
_sum = _mm_add_ps(_mm_mul_ps(_val2, _w2), _sum);
__m128 _w3 = _mm_loadu_ps(kptr + 12);
_sum = _mm_add_ps(_mm_mul_ps(_val3, _w3), _sum);

sptr += dilation_w * 4;
kptr += 16;
}
}

_sum = activation_sse(_sum, activation_type, activation_params);

_mm_storeu_ps(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 1 && out_elempack == 4)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
__m128 _sum = _mm_set1_ps(0.f);

if (bias_term)
{
_sum = _mm_loadu_ps((const float*)bias_data + p * 4);
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
__m128 _val = _mm_set1_ps(sptr[0]);
__m128 _w = _mm_loadu_ps(kptr);
_sum = _mm_add_ps(_mm_mul_ps(_val, _w), _sum);

sptr += dilation_w;
kptr += 4;
}
}

_sum = activation_sse(_sum, activation_type, activation_params);

_mm_storeu_ps(outptr, _sum);
outptr += 4;
}
}
}
}

if (elempack == 4 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const float* kptr = weight_data_packed.channel(p);

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w * 4;

for (int k = 0; k < kernel_w; k++)
{
__m128 _val = _mm_loadu_ps(sptr);
__m128 _w = _mm_loadu_ps(kptr);
__m128 _s4 = _mm_mul_ps(_val, _w);
sum += _mm_reduce_add_ps(_s4); // dot

sptr += dilation_w * 4;
kptr += 4;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}
#endif // __SSE2__

if (elempack == 1 && out_elempack == 1)
{
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outh; p++)
{
float* outptr = top_blob.row(p);

for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const float* kptr = (const float*)weight_data + kernel_w * h * p;

for (int q = 0; q < h; q++)
{
const float* sptr = bottom_blob_bordered.row(q) + j * stride_w;

for (int k = 0; k < kernel_w; k++)
{
float val = sptr[0];
float wt = kptr[0];
sum += val * wt;

sptr += dilation_w;
kptr += 1;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}
}
}
}
convolution1d_packed(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, dilation_w, stride_w, activation_type, activation_params, opt);

return 0;
}


+ 1
- 1
src/layer/x86/convolution1d_x86.h View File

@@ -32,7 +32,7 @@ public:
virtual int forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const;

public:
Mat weight_data_packed;
Mat weight_data_tm;
};

} // namespace ncnn


+ 87
- 237
src/layer/x86/convolution_packed.h View File

@@ -1188,22 +1188,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;
const float* r8 = r0 + N * 8;
const float* r9 = r0 + N * 9;
const float* ra = r0 + N * 10;
const float* rb = r0 + N * 11;
const float* rc = r0 + N * 12;
const float* rd = r0 + N * 13;
const float* re = r0 + N * 14;
const float* rf = r0 + N * 15;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1226,21 +1210,21 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m512 _wf = _mm512_load_ps(kptr + 16 * 15);

_sum0 = _mm512_fmadd_ps(_w0, _mm512_set1_ps(r0[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r1[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r2[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r3[sok]), _sum3);
_sum0 = _mm512_fmadd_ps(_w4, _mm512_set1_ps(r4[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w5, _mm512_set1_ps(r5[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_w6, _mm512_set1_ps(r6[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_w7, _mm512_set1_ps(r7[sok]), _sum3);
_sum0 = _mm512_fmadd_ps(_w8, _mm512_set1_ps(r8[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w9, _mm512_set1_ps(r9[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_wa, _mm512_set1_ps(ra[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_wb, _mm512_set1_ps(rb[sok]), _sum3);
_sum0 = _mm512_fmadd_ps(_wc, _mm512_set1_ps(rc[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_wd, _mm512_set1_ps(rd[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_we, _mm512_set1_ps(re[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_wf, _mm512_set1_ps(rf[sok]), _sum3);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm512_fmadd_ps(_w4, _mm512_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm512_fmadd_ps(_w5, _mm512_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm512_fmadd_ps(_w6, _mm512_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm512_fmadd_ps(_w7, _mm512_set1_ps(r0[sok + N * 7]), _sum3);
_sum0 = _mm512_fmadd_ps(_w8, _mm512_set1_ps(r0[sok + N * 8]), _sum0);
_sum1 = _mm512_fmadd_ps(_w9, _mm512_set1_ps(r0[sok + N * 9]), _sum1);
_sum2 = _mm512_fmadd_ps(_wa, _mm512_set1_ps(r0[sok + N * 10]), _sum2);
_sum3 = _mm512_fmadd_ps(_wb, _mm512_set1_ps(r0[sok + N * 11]), _sum3);
_sum0 = _mm512_fmadd_ps(_wc, _mm512_set1_ps(r0[sok + N * 12]), _sum0);
_sum1 = _mm512_fmadd_ps(_wd, _mm512_set1_ps(r0[sok + N * 13]), _sum1);
_sum2 = _mm512_fmadd_ps(_we, _mm512_set1_ps(r0[sok + N * 14]), _sum2);
_sum3 = _mm512_fmadd_ps(_wf, _mm512_set1_ps(r0[sok + N * 15]), _sum3);

kptr += 256;
}
@@ -1309,14 +1293,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1331,13 +1307,13 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m512 _w7 = _mm512_load_ps(kptr + 16 * 7);

_sum0 = _mm512_fmadd_ps(_w0, _mm512_set1_ps(r0[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r1[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r2[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r3[sok]), _sum3);
_sum0 = _mm512_fmadd_ps(_w4, _mm512_set1_ps(r4[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w5, _mm512_set1_ps(r5[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_w6, _mm512_set1_ps(r6[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_w7, _mm512_set1_ps(r7[sok]), _sum3);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm512_fmadd_ps(_w4, _mm512_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm512_fmadd_ps(_w5, _mm512_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm512_fmadd_ps(_w6, _mm512_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm512_fmadd_ps(_w7, _mm512_set1_ps(r0[sok + N * 7]), _sum3);

kptr += 128;
}
@@ -1368,10 +1344,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1382,9 +1354,9 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m512 _w3 = _mm512_load_ps(kptr + 48);

_sum0 = _mm512_fmadd_ps(_w0, _mm512_set1_ps(r0[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r1[sok]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r2[sok]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r3[sok]), _sum3);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm512_fmadd_ps(_w2, _mm512_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm512_fmadd_ps(_w3, _mm512_set1_ps(r0[sok + N * 3]), _sum3);

kptr += 64;
}
@@ -1396,8 +1368,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&

// if (elempack == 1)
{
const float* r1 = r0 + N;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1406,7 +1376,7 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m512 _w1 = _mm512_load_ps(kptr + 16);

_sum0 = _mm512_fmadd_ps(_w0, _mm512_set1_ps(r0[sok]), _sum0);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r1[sok]), _sum1);
_sum1 = _mm512_fmadd_ps(_w1, _mm512_set1_ps(r0[sok + N]), _sum1);

kptr += 32;
}
@@ -1659,22 +1629,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;
const float* r8 = r0 + N * 8;
const float* r9 = r0 + N * 9;
const float* ra = r0 + N * 10;
const float* rb = r0 + N * 11;
const float* rc = r0 + N * 12;
const float* rd = r0 + N * 13;
const float* re = r0 + N * 14;
const float* rf = r0 + N * 15;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1697,21 +1651,21 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m256 _wf = _mm256_load_ps(kptr + 8 * 15);

_sum0 = _mm256_fmadd_ps(_w0, _mm256_set1_ps(r0[sok]), _sum0);
_sum1 = _mm256_fmadd_ps(_w1, _mm256_set1_ps(r1[sok]), _sum1);
_sum2 = _mm256_fmadd_ps(_w2, _mm256_set1_ps(r2[sok]), _sum2);
_sum3 = _mm256_fmadd_ps(_w3, _mm256_set1_ps(r3[sok]), _sum3);
_sum0 = _mm256_fmadd_ps(_w4, _mm256_set1_ps(r4[sok]), _sum0);
_sum1 = _mm256_fmadd_ps(_w5, _mm256_set1_ps(r5[sok]), _sum1);
_sum2 = _mm256_fmadd_ps(_w6, _mm256_set1_ps(r6[sok]), _sum2);
_sum3 = _mm256_fmadd_ps(_w7, _mm256_set1_ps(r7[sok]), _sum3);
_sum0 = _mm256_fmadd_ps(_w8, _mm256_set1_ps(r8[sok]), _sum0);
_sum1 = _mm256_fmadd_ps(_w9, _mm256_set1_ps(r9[sok]), _sum1);
_sum2 = _mm256_fmadd_ps(_wa, _mm256_set1_ps(ra[sok]), _sum2);
_sum3 = _mm256_fmadd_ps(_wb, _mm256_set1_ps(rb[sok]), _sum3);
_sum0 = _mm256_fmadd_ps(_wc, _mm256_set1_ps(rc[sok]), _sum0);
_sum1 = _mm256_fmadd_ps(_wd, _mm256_set1_ps(rd[sok]), _sum1);
_sum2 = _mm256_fmadd_ps(_we, _mm256_set1_ps(re[sok]), _sum2);
_sum3 = _mm256_fmadd_ps(_wf, _mm256_set1_ps(rf[sok]), _sum3);
_sum1 = _mm256_fmadd_ps(_w1, _mm256_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm256_fmadd_ps(_w2, _mm256_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm256_fmadd_ps(_w3, _mm256_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm256_fmadd_ps(_w4, _mm256_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm256_fmadd_ps(_w5, _mm256_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm256_fmadd_ps(_w6, _mm256_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm256_fmadd_ps(_w7, _mm256_set1_ps(r0[sok + N * 7]), _sum3);
_sum0 = _mm256_fmadd_ps(_w8, _mm256_set1_ps(r0[sok + N * 8]), _sum0);
_sum1 = _mm256_fmadd_ps(_w9, _mm256_set1_ps(r0[sok + N * 9]), _sum1);
_sum2 = _mm256_fmadd_ps(_wa, _mm256_set1_ps(r0[sok + N * 10]), _sum2);
_sum3 = _mm256_fmadd_ps(_wb, _mm256_set1_ps(r0[sok + N * 11]), _sum3);
_sum0 = _mm256_fmadd_ps(_wc, _mm256_set1_ps(r0[sok + N * 12]), _sum0);
_sum1 = _mm256_fmadd_ps(_wd, _mm256_set1_ps(r0[sok + N * 13]), _sum1);
_sum2 = _mm256_fmadd_ps(_we, _mm256_set1_ps(r0[sok + N * 14]), _sum2);
_sum3 = _mm256_fmadd_ps(_wf, _mm256_set1_ps(r0[sok + N * 15]), _sum3);

kptr += 128;
}
@@ -1781,14 +1735,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1803,13 +1749,13 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m256 _w7 = _mm256_load_ps(kptr + 56);

_sum0 = _mm256_comp_fmadd_ps(_w0, _mm256_set1_ps(r0[sok]), _sum0);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r1[sok]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w2, _mm256_set1_ps(r2[sok]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w3, _mm256_set1_ps(r3[sok]), _sum3);
_sum0 = _mm256_comp_fmadd_ps(_w4, _mm256_set1_ps(r4[sok]), _sum0);
_sum1 = _mm256_comp_fmadd_ps(_w5, _mm256_set1_ps(r5[sok]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w6, _mm256_set1_ps(r6[sok]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w7, _mm256_set1_ps(r7[sok]), _sum3);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w2, _mm256_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w3, _mm256_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm256_comp_fmadd_ps(_w4, _mm256_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm256_comp_fmadd_ps(_w5, _mm256_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w6, _mm256_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w7, _mm256_set1_ps(r0[sok + N * 7]), _sum3);

kptr += 64;
}
@@ -1840,10 +1786,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1854,9 +1796,9 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m256 _w3 = _mm256_load_ps(kptr + 24);

_sum0 = _mm256_comp_fmadd_ps(_w0, _mm256_set1_ps(r0[sok]), _sum0);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r1[sok]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w2, _mm256_set1_ps(r2[sok]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w3, _mm256_set1_ps(r3[sok]), _sum3);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm256_comp_fmadd_ps(_w2, _mm256_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm256_comp_fmadd_ps(_w3, _mm256_set1_ps(r0[sok + N * 3]), _sum3);

kptr += 32;
}
@@ -1868,8 +1810,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&

// if (elempack == 1)
{
const float* r1 = r0 + N;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -1878,7 +1818,7 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m256 _w1 = _mm256_load_ps(kptr + 8);

_sum0 = _mm256_comp_fmadd_ps(_w0, _mm256_set1_ps(r0[sok]), _sum0);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r1[sok]), _sum1);
_sum1 = _mm256_comp_fmadd_ps(_w1, _mm256_set1_ps(r0[sok + N]), _sum1);

kptr += 16;
}
@@ -2118,22 +2058,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;
const float* r8 = r0 + N * 8;
const float* r9 = r0 + N * 9;
const float* ra = r0 + N * 10;
const float* rb = r0 + N * 11;
const float* rc = r0 + N * 12;
const float* rd = r0 + N * 13;
const float* re = r0 + N * 14;
const float* rf = r0 + N * 15;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -2156,21 +2080,21 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m128 _wf = _mm_load_ps(kptr + 4 * 15);

_sum0 = _mm_fmadd_ps(_w0, _mm_set1_ps(r0[sok]), _sum0);
_sum1 = _mm_fmadd_ps(_w1, _mm_set1_ps(r1[sok]), _sum1);
_sum2 = _mm_fmadd_ps(_w2, _mm_set1_ps(r2[sok]), _sum2);
_sum3 = _mm_fmadd_ps(_w3, _mm_set1_ps(r3[sok]), _sum3);
_sum0 = _mm_fmadd_ps(_w4, _mm_set1_ps(r4[sok]), _sum0);
_sum1 = _mm_fmadd_ps(_w5, _mm_set1_ps(r5[sok]), _sum1);
_sum2 = _mm_fmadd_ps(_w6, _mm_set1_ps(r6[sok]), _sum2);
_sum3 = _mm_fmadd_ps(_w7, _mm_set1_ps(r7[sok]), _sum3);
_sum0 = _mm_fmadd_ps(_w8, _mm_set1_ps(r8[sok]), _sum0);
_sum1 = _mm_fmadd_ps(_w9, _mm_set1_ps(r9[sok]), _sum1);
_sum2 = _mm_fmadd_ps(_wa, _mm_set1_ps(ra[sok]), _sum2);
_sum3 = _mm_fmadd_ps(_wb, _mm_set1_ps(rb[sok]), _sum3);
_sum0 = _mm_fmadd_ps(_wc, _mm_set1_ps(rc[sok]), _sum0);
_sum1 = _mm_fmadd_ps(_wd, _mm_set1_ps(rd[sok]), _sum1);
_sum2 = _mm_fmadd_ps(_we, _mm_set1_ps(re[sok]), _sum2);
_sum3 = _mm_fmadd_ps(_wf, _mm_set1_ps(rf[sok]), _sum3);
_sum1 = _mm_fmadd_ps(_w1, _mm_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm_fmadd_ps(_w2, _mm_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm_fmadd_ps(_w3, _mm_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm_fmadd_ps(_w4, _mm_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm_fmadd_ps(_w5, _mm_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm_fmadd_ps(_w6, _mm_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm_fmadd_ps(_w7, _mm_set1_ps(r0[sok + N * 7]), _sum3);
_sum0 = _mm_fmadd_ps(_w8, _mm_set1_ps(r0[sok + N * 8]), _sum0);
_sum1 = _mm_fmadd_ps(_w9, _mm_set1_ps(r0[sok + N * 9]), _sum1);
_sum2 = _mm_fmadd_ps(_wa, _mm_set1_ps(r0[sok + N * 10]), _sum2);
_sum3 = _mm_fmadd_ps(_wb, _mm_set1_ps(r0[sok + N * 11]), _sum3);
_sum0 = _mm_fmadd_ps(_wc, _mm_set1_ps(r0[sok + N * 12]), _sum0);
_sum1 = _mm_fmadd_ps(_wd, _mm_set1_ps(r0[sok + N * 13]), _sum1);
_sum2 = _mm_fmadd_ps(_we, _mm_set1_ps(r0[sok + N * 14]), _sum2);
_sum3 = _mm_fmadd_ps(_wf, _mm_set1_ps(r0[sok + N * 15]), _sum3);

kptr += 64;
}
@@ -2240,14 +2164,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -2262,13 +2178,13 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m128 _w7 = _mm_load_ps(kptr + 28);

_sum0 = _mm_comp_fmadd_ps(_w0, _mm_set1_ps(r0[sok]), _sum0);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r1[sok]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w2, _mm_set1_ps(r2[sok]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w3, _mm_set1_ps(r3[sok]), _sum3);
_sum0 = _mm_comp_fmadd_ps(_w4, _mm_set1_ps(r4[sok]), _sum0);
_sum1 = _mm_comp_fmadd_ps(_w5, _mm_set1_ps(r5[sok]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w6, _mm_set1_ps(r6[sok]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w7, _mm_set1_ps(r7[sok]), _sum3);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w2, _mm_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w3, _mm_set1_ps(r0[sok + N * 3]), _sum3);
_sum0 = _mm_comp_fmadd_ps(_w4, _mm_set1_ps(r0[sok + N * 4]), _sum0);
_sum1 = _mm_comp_fmadd_ps(_w5, _mm_set1_ps(r0[sok + N * 5]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w6, _mm_set1_ps(r0[sok + N * 6]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w7, _mm_set1_ps(r0[sok + N * 7]), _sum3);

kptr += 32;
}
@@ -2300,10 +2216,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -2314,9 +2226,9 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m128 _w3 = _mm_load_ps(kptr + 12);

_sum0 = _mm_comp_fmadd_ps(_w0, _mm_set1_ps(r0[sok]), _sum0);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r1[sok]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w2, _mm_set1_ps(r2[sok]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w3, _mm_set1_ps(r3[sok]), _sum3);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r0[sok + N]), _sum1);
_sum2 = _mm_comp_fmadd_ps(_w2, _mm_set1_ps(r0[sok + N * 2]), _sum2);
_sum3 = _mm_comp_fmadd_ps(_w3, _mm_set1_ps(r0[sok + N * 3]), _sum3);

kptr += 16;
}
@@ -2328,8 +2240,6 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&

// if (elempack == 1)
{
const float* r1 = r0 + N;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
@@ -2338,7 +2248,7 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
__m128 _w1 = _mm_load_ps(kptr + 4);

_sum0 = _mm_comp_fmadd_ps(_w0, _mm_set1_ps(r0[sok]), _sum0);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r1[sok]), _sum1);
_sum1 = _mm_comp_fmadd_ps(_w1, _mm_set1_ps(r0[sok + N]), _sum1);

kptr += 8;
}
@@ -2482,26 +2392,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;
const float* r8 = r0 + N * 8;
const float* r9 = r0 + N * 9;
const float* ra = r0 + N * 10;
const float* rb = r0 + N * 11;
const float* rc = r0 + N * 12;
const float* rd = r0 + N * 13;
const float* re = r0 + N * 14;
const float* rf = r0 + N * 15;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m512 _r0 = _mm512_set_ps(rf[sok], re[sok], rd[sok], rc[sok], rb[sok], ra[sok], r9[sok], r8[sok], r7[sok], r6[sok], r5[sok], r4[sok], r3[sok], r2[sok], r1[sok], r0[sok]);
__m512 _r0 = _mm512_set_ps(r0[sok + N * 15], r0[sok + N * 14], r0[sok + N * 13], r0[sok + N * 12], r0[sok + N * 11], r0[sok + N * 10], r0[sok + N * 9], r0[sok + N * 8], r0[sok + N * 7], r0[sok + N * 6], r0[sok + N * 5], r0[sok + N * 4], r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m512 _w0 = _mm512_load_ps(kptr);
__m512 _w1 = _mm512_load_ps(kptr + 16);
_sum0_avx512 = _mm512_fmadd_ps(_r0, _w0, _sum0_avx512);
@@ -2552,18 +2446,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m256 _r0 = _mm256_set_ps(r7[sok], r6[sok], r5[sok], r4[sok], r3[sok], r2[sok], r1[sok], r0[sok]);
__m256 _r0 = _mm256_set_ps(r0[sok + N * 7], r0[sok + N * 6], r0[sok + N * 5], r0[sok + N * 4], r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m256 _w0 = _mm256_load_ps(kptr);
__m256 _w1 = _mm256_load_ps(kptr + 8);
_sum0_avx = _mm256_comp_fmadd_ps(_r0, _w0, _sum0_avx);
@@ -2598,14 +2484,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m128 _r0 = _mm_set_ps(r3[sok], r2[sok], r1[sok], r0[sok]);
__m128 _r0 = _mm_set_ps(r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m128 _w0 = _mm_load_ps(kptr);
__m128 _w1 = _mm_load_ps(kptr + 4);
_sum0 = _mm_comp_fmadd_ps(_r0, _w0, _sum0);
@@ -2624,16 +2506,14 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&

// if (elempack == 1)
{
const float* r1 = r0 + N;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];

sum0 += r0[sok] * kptr[0];
sum1 += r0[sok] * kptr[1];
sum0 += r1[sok] * kptr[2];
sum1 += r1[sok] * kptr[3];
sum0 += r0[sok + N] * kptr[2];
sum1 += r0[sok + N] * kptr[3];

kptr += 4;
}
@@ -2745,26 +2625,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;
const float* r8 = r0 + N * 8;
const float* r9 = r0 + N * 9;
const float* ra = r0 + N * 10;
const float* rb = r0 + N * 11;
const float* rc = r0 + N * 12;
const float* rd = r0 + N * 13;
const float* re = r0 + N * 14;
const float* rf = r0 + N * 15;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m512 _r0 = _mm512_set_ps(rf[sok], re[sok], rd[sok], rc[sok], rb[sok], ra[sok], r9[sok], r8[sok], r7[sok], r6[sok], r5[sok], r4[sok], r3[sok], r2[sok], r1[sok], r0[sok]);
__m512 _r0 = _mm512_set_ps(r0[sok + N * 15], r0[sok + N * 14], r0[sok + N * 13], r0[sok + N * 12], r0[sok + N * 11], r0[sok + N * 10], r0[sok + N * 9], r0[sok + N * 8], r0[sok + N * 7], r0[sok + N * 6], r0[sok + N * 5], r0[sok + N * 4], r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m512 _w = _mm512_load_ps(kptr);
_sum_avx512 = _mm512_fmadd_ps(_r0, _w, _sum_avx512);

@@ -2807,18 +2671,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;
const float* r4 = r0 + N * 4;
const float* r5 = r0 + N * 5;
const float* r6 = r0 + N * 6;
const float* r7 = r0 + N * 7;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m256 _r0 = _mm256_set_ps(r7[sok], r6[sok], r5[sok], r4[sok], r3[sok], r2[sok], r1[sok], r0[sok]);
__m256 _r0 = _mm256_set_ps(r0[sok + N * 7], r0[sok + N * 6], r0[sok + N * 5], r0[sok + N * 4], r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m256 _w = _mm256_load_ps(kptr);
_sum_avx = _mm256_comp_fmadd_ps(_r0, _w, _sum_avx);

@@ -2847,14 +2703,10 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&
}
if (elempack == 1)
{
const float* r1 = r0 + N;
const float* r2 = r0 + N * 2;
const float* r3 = r0 + N * 3;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];
__m128 _r0 = _mm_set_ps(r3[sok], r2[sok], r1[sok], r0[sok]);
__m128 _r0 = _mm_set_ps(r0[sok + N * 3], r0[sok + N * 2], r0[sok + N], r0[sok]);
__m128 _w = _mm_load_ps(kptr);
_sum = _mm_comp_fmadd_ps(_r0, _w, _sum);

@@ -2870,14 +2722,12 @@ static void convolution_packed(const Mat& bottom_blob, Mat& top_blob, const Mat&

// if (elempack == 1)
{
const float* r1 = r0 + N;

for (int k = 0; k < maxk; k++)
{
const int sok = space_ofs[k];

sum += r0[sok] * kptr[0];
sum += r1[sok] * kptr[1];
sum += r0[sok + N] * kptr[1];

kptr += 2;
}


+ 42
- 30
tests/test_convolution1d.cpp View File

@@ -76,38 +76,50 @@ static int test_convolution1d_0()
const int d = kdsp[i][1];
const int s = kdsp[i][2];
const int p = kdsp[i][3];
const int b0 = i % 2;
const int b1 = 1 - b1;

int ret = 0
|| test_convolution1d(9, 1, 1, k, d, s, p, 1)
|| test_convolution1d(9, 4, 13, k, d, s, p, 0)
|| test_convolution1d(9, 13, 4, k, d, s, p, 1)
|| test_convolution1d(9, 12, 12, k, d, s, p, 0)
|| test_convolution1d(9, 8, 12, k, d, s, p, 1)
|| test_convolution1d(9, 8, 13, k, d, s, p, 0)
|| test_convolution1d(9, 13, 8, k, d, s, p, 1)
|| test_convolution1d(9, 12, 16, k, d, s, p, 0)
|| test_convolution1d(9, 15, 15, k, d, s, p, 0)
|| test_convolution1d(9, 16, 16, k, d, s, p, 0)
|| test_convolution1d(18, 1, 1, k, d, s, p, 1)
|| test_convolution1d(18, 4, 13, k, d, s, p, 0)
|| test_convolution1d(18, 13, 4, k, d, s, p, 1)
|| test_convolution1d(18, 12, 12, k, d, s, p, 0)
|| test_convolution1d(18, 8, 12, k, d, s, p, 1)
|| test_convolution1d(18, 8, 13, k, d, s, p, 0)
|| test_convolution1d(18, 13, 8, k, d, s, p, 1)
|| test_convolution1d(18, 12, 16, k, d, s, p, 0)
|| test_convolution1d(18, 15, 15, k, d, s, p, 0)
|| test_convolution1d(18, 16, 16, k, d, s, p, 0)
|| test_convolution1d(25, 1, 1, k, d, s, p, 1)
|| test_convolution1d(25, 4, 13, k, d, s, p, 0)
|| test_convolution1d(25, 13, 4, k, d, s, p, 1)
|| test_convolution1d(25, 12, 12, k, d, s, p, 0)
|| test_convolution1d(25, 8, 12, k, d, s, p, 1)
|| test_convolution1d(25, 8, 13, k, d, s, p, 0)
|| test_convolution1d(25, 13, 8, k, d, s, p, 1)
|| test_convolution1d(25, 12, 16, k, d, s, p, 0)
|| test_convolution1d(25, 15, 15, k, d, s, p, 0)
|| test_convolution1d(25, 16, 16, k, d, s, p, 0);
|| test_convolution1d(9, 1, 1, k, d, s, p, b0)
|| test_convolution1d(9, 1, 3, k, d, s, p, b1)
|| test_convolution1d(9, 1, 7, k, d, s, p, b0)
|| test_convolution1d(9, 1, 15, k, d, s, p, b1)
|| test_convolution1d(9, 1, 31, k, d, s, p, b0)
|| test_convolution1d(9, 3, 1, k, d, s, p, b1)
|| test_convolution1d(9, 3, 3, k, d, s, p, b0)
|| test_convolution1d(9, 3, 7, k, d, s, p, b1)
|| test_convolution1d(9, 3, 15, k, d, s, p, b0)
|| test_convolution1d(9, 3, 31, k, d, s, p, b1)
|| test_convolution1d(9, 7, 1, k, d, s, p, b0)
|| test_convolution1d(9, 7, 3, k, d, s, p, b1)
|| test_convolution1d(9, 7, 7, k, d, s, p, b0)
|| test_convolution1d(9, 7, 15, k, d, s, p, b1)
|| test_convolution1d(9, 7, 31, k, d, s, p, b0)
|| test_convolution1d(9, 15, 1, k, d, s, p, b1)
|| test_convolution1d(9, 15, 3, k, d, s, p, b0)
|| test_convolution1d(9, 15, 7, k, d, s, p, b1)
|| test_convolution1d(9, 15, 15, k, d, s, p, b0)
|| test_convolution1d(9, 15, 31, k, d, s, p, b1)
|| test_convolution1d(9, 31, 1, k, d, s, p, b0)
|| test_convolution1d(9, 31, 3, k, d, s, p, b1)
|| test_convolution1d(9, 31, 7, k, d, s, p, b0)
|| test_convolution1d(9, 31, 15, k, d, s, p, b1)
|| test_convolution1d(25, 28, 31, k, d, s, p, b0)
|| test_convolution1d(25, 31, 28, k, d, s, p, b1)
|| test_convolution1d(25, 28, 28, k, d, s, p, b0)
|| test_convolution1d(25, 24, 28, k, d, s, p, b1)
|| test_convolution1d(25, 24, 31, k, d, s, p, b0)
|| test_convolution1d(25, 28, 24, k, d, s, p, b1)
|| test_convolution1d(25, 31, 24, k, d, s, p, b0)
|| test_convolution1d(25, 24, 24, k, d, s, p, b1)
|| test_convolution1d(25, 28, 48, k, d, s, p, b0)
|| test_convolution1d(25, 31, 48, k, d, s, p, b1)
|| test_convolution1d(25, 24, 48, k, d, s, p, b0)
|| test_convolution1d(25, 48, 28, k, d, s, p, b1)
|| test_convolution1d(25, 48, 31, k, d, s, p, b0)
|| test_convolution1d(25, 48, 24, k, d, s, p, b1)
|| test_convolution1d(25, 31, 31, k, d, s, p, b0)
|| test_convolution1d(25, 48, 48, k, d, s, p, b1);

if (ret != 0)
return -1;


Write Preview
Loading…
Cancel
Save