nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

convolution flatten arm fp16sa pack8

tags/20200916
nihuini 6 years ago
parent
dbb761b9a4
commit
bc3822acc3
5 changed files with 542 additions and 227 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +321
    -150
      src/layer/arm/convolution_arm.cpp
  2. +0
    -3
      src/layer/arm/convolution_arm.h
  3. +211
    -73
      src/layer/arm/flatten_arm.cpp
  4. +1
    -1
      src/layer/arm/innerproduct_arm.cpp
  5. +9
    -0
      src/layer/arm/neon_activation.h

+ 321
- 150
src/layer/arm/convolution_arm.cpp View File

@@ -1042,169 +1042,48 @@ int Convolution_arm::create_pipeline_fp16s(const Option& opt)
const int maxk = kernel_w * kernel_h;
const int num_input = weight_data_size / maxk / num_output;

int elempack = (support_packing && opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1;
int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;
int elempack = 1;
int out_elempack = 1;

// pack4
if (elempack == 4 && out_elempack == 4)
if (opt.use_packing_layout)
{
{
// src = kw-kh-inch-outch
// dst = 4b-4a-kw-kh-inch/4a-outch/4b
Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

weight_data_pack4_fp16.create(maxk, num_input / 4, num_output / 4, (size_t)2 * 16, 16);

for (int q = 0; q + 3 < num_output; q += 4)
{
const Mat k0 = weight_data_r2.channel(q);
const Mat k1 = weight_data_r2.channel(q + 1);
const Mat k2 = weight_data_r2.channel(q + 2);
const Mat k3 = weight_data_r2.channel(q + 3);

Mat g0 = weight_data_pack4_fp16.channel(q / 4);

for (int p = 0; p + 3 < num_input; p += 4)
{
const float* k00 = k0.row(p);
const float* k01 = k0.row(p + 1);
const float* k02 = k0.row(p + 2);
const float* k03 = k0.row(p + 3);

const float* k10 = k1.row(p);
const float* k11 = k1.row(p + 1);
const float* k12 = k1.row(p + 2);
const float* k13 = k1.row(p + 3);

const float* k20 = k2.row(p);
const float* k21 = k2.row(p + 1);
const float* k22 = k2.row(p + 2);
const float* k23 = k2.row(p + 3);

const float* k30 = k3.row(p);
const float* k31 = k3.row(p + 1);
const float* k32 = k3.row(p + 2);
const float* k33 = k3.row(p + 3);

__fp16* g00 = g0.row<__fp16>(p / 4);

for (int k = 0; k < maxk; k++)
{
g00[0] = k00[k];
g00[1] = k10[k];
g00[2] = k20[k];
g00[3] = k30[k];

g00[4] = k01[k];
g00[5] = k11[k];
g00[6] = k21[k];
g00[7] = k31[k];

g00[8] = k02[k];
g00[9] = k12[k];
g00[10] = k22[k];
g00[11] = k32[k];

g00[12] = k03[k];
g00[13] = k13[k];
g00[14] = k23[k];
g00[15] = k33[k];

g00 += 16;
}
}
}
}
elempack = opt.use_fp16_arithmetic && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
}

// pack1to4
if (elempack == 1 && out_elempack == 4)
// src = kw-kh-inch-outch
// dst = pb-pa-kw-kh-inch/pa-outch/pb
{
// src = kw-kh-inch-outch
// dst = 4b-kw-kh-inch-outch/4b
{
Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

weight_data_pack1to4_fp16.create(maxk, num_input, num_output / 4, (size_t)2 * 4, 4);

for (int q = 0; q + 3 < num_output; q += 4)
{
const Mat k0 = weight_data_r2.channel(q);
const Mat k1 = weight_data_r2.channel(q + 1);
const Mat k2 = weight_data_r2.channel(q + 2);
const Mat k3 = weight_data_r2.channel(q + 3);

Mat g0 = weight_data_pack1to4_fp16.channel(q / 4);

for (int p = 0; p < num_input; p++)
{
const float* k00 = k0.row(p);
const float* k10 = k1.row(p);
const float* k20 = k2.row(p);
const float* k30 = k3.row(p);

__fp16* g00 = g0.row<__fp16>(p);

for (int k = 0; k < maxk; k++)
{
g00[0] = k00[k];
g00[1] = k10[k];
g00[2] = k20[k];
g00[3] = k30[k];
Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

g00 += 4;
}
}
}
}
}
weight_data_fp16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);

// pack4to1
if (elempack == 4 && out_elempack == 1)
{
for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
// src = kw-kh-inch-outch
// dst = 4a-kw-kh-inch/4a-outch
Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);

weight_data_pack4to1_fp16.create(maxk, num_input / 4, num_output, (size_t)2 * 4, 4);
Mat g0 = weight_data_fp16.channel(q / out_elempack);

for (int q = 0; q < num_output; q++)
for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
{
const Mat k0 = weight_data_r2.channel(q);
Mat g0 = weight_data_pack4to1_fp16.channel(q);
__fp16* g00 = g0.row<__fp16>(p / elempack);

for (int p = 0; p + 3 < num_input; p += 4)
for (int k = 0; k < maxk; k++)
{
const float* k00 = k0.row(p);
const float* k01 = k0.row(p + 1);
const float* k02 = k0.row(p + 2);
const float* k03 = k0.row(p + 3);

__fp16* g00 = g0.row<__fp16>(p / 4);

for (int k = 0; k < maxk; k++)
for (int i = 0; i < elempack; i++)
{
g00[0] = k00[k];
g00[1] = k01[k];
g00[2] = k02[k];
g00[3] = k03[k];
for (int j = 0; j < out_elempack; j++)
{
const float* k00 = weight_data_r2.channel(q + j).row(p + i);

g00 += 4;
g00[0] = (__fp16)k00[k];

g00++;
}
}
}
}
}
}

// pack1
if (elempack == 1 && out_elempack == 1)
{
{
ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt);
}
}

ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);

return 0;
@@ -1287,7 +1166,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const __fp16* kptr = weight_data_pack4_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
@@ -1344,7 +1223,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

const __fp16* kptr = weight_data_pack1to4_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
@@ -1393,7 +1272,7 @@ int Convolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const
sum = bias_data[p];
}

const __fp16* kptr = weight_data_pack4to1_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
@@ -1499,7 +1378,11 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const

int outw = (w - kernel_extent_w) / stride_w + 1;
int outh = (h - kernel_extent_h) / stride_h + 1;
int out_elempack = (support_packing && opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;
int out_elempack = 1;
if (opt.use_packing_layout)
{
out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
}
size_t out_elemsize = elemsize / elempack * out_elempack;

top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
@@ -1533,6 +1416,294 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const
}
}

if (elempack == 8 && out_elempack == 8)
{
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output / out_elempack; p++)
{
__fp16* outptr = top_blob.channel(p);

for (int i = 0; i < outh; i++)
{
for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
{
const Mat m = bottom_blob_bordered.channel(q);
const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;

for (int k = 0; k < maxk; k++)
{
float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);

float16x8_t _w0 = vld1q_f16(kptr);
float16x8_t _w1 = vld1q_f16(kptr + 8);
float16x8_t _w2 = vld1q_f16(kptr + 16);
float16x8_t _w3 = vld1q_f16(kptr + 24);
float16x8_t _w4 = vld1q_f16(kptr + 32);
float16x8_t _w5 = vld1q_f16(kptr + 40);
float16x8_t _w6 = vld1q_f16(kptr + 48);
float16x8_t _w7 = vld1q_f16(kptr + 56);

_sum = vfmaq_laneq_f16(_sum, _w0, _val, 0);
_sum = vfmaq_laneq_f16(_sum, _w1, _val, 1);
_sum = vfmaq_laneq_f16(_sum, _w2, _val, 2);
_sum = vfmaq_laneq_f16(_sum, _w3, _val, 3);
_sum = vfmaq_laneq_f16(_sum, _w4, _val, 4);
_sum = vfmaq_laneq_f16(_sum, _w5, _val, 5);
_sum = vfmaq_laneq_f16(_sum, _w6, _val, 6);
_sum = vfmaq_laneq_f16(_sum, _w7, _val, 7);

kptr += 64;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr + j * 8, _sum);
}

outptr += outw * 8;
}
}
}
}

if (elempack == 1 && out_elempack == 8)
{
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output / out_elempack; p++)
{
__fp16* outptr = top_blob.channel(p);

for (int i = 0; i < outh; i++)
{
for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
{
const Mat m = bottom_blob_bordered.channel(q);
const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w;

for (int k = 0; k < maxk; k++)
{
float16x8_t _val = vdupq_n_f16(sptr[space_ofs[k]]);
float16x8_t _w = vld1q_f16(kptr);
_sum = vfmaq_f16(_sum, _val, _w);

kptr += 8;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr + j * 8, _sum);
}

outptr += outw * 8;
}
}
}
}

if (elempack == 4 && out_elempack == 8)
{
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output / out_elempack; p++)
{
__fp16* outptr = top_blob.channel(p);

for (int i = 0; i < outh; i++)
{
for (int j = 0; j < outw; j++)
{
float16x8_t _sum = vdupq_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1q_f16(((const __fp16*)bias_data_fp16) + p * 8);
}

const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
{
const Mat m = bottom_blob_bordered.channel(q);
const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 4;

for (int k = 0; k < maxk; k++)
{
float16x4_t _val = vld1_f16(sptr + space_ofs[k] * 4);

float16x8_t _w0 = vld1q_f16(kptr);
float16x8_t _w1 = vld1q_f16(kptr + 8);
float16x8_t _w2 = vld1q_f16(kptr + 16);
float16x8_t _w3 = vld1q_f16(kptr + 24);

_sum = vfmaq_lane_f16(_sum, _w0, _val, 0);
_sum = vfmaq_lane_f16(_sum, _w1, _val, 1);
_sum = vfmaq_lane_f16(_sum, _w2, _val, 2);
_sum = vfmaq_lane_f16(_sum, _w3, _val, 3);

kptr += 32;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1q_f16(outptr + j * 8, _sum);
}

outptr += outw * 8;
}
}
}
}

if (elempack == 8 && out_elempack == 1)
{
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output; p++)
{
__fp16* outptr = top_blob.channel(p);

for (int i = 0; i < outh; i++)
{
for (int j = 0; j < outw; j++)
{
float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
{
const Mat m = bottom_blob_bordered.channel(q);
const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;

for (int k = 0; k < maxk; k++)
{
float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);
float16x8_t _w = vld1q_f16(kptr);
float16x8_t _s8 = vmulq_f16(_val, _w);

float16x4_t _s4 = vadd_f16(vget_low_f16(_s8), vget_high_f16(_s8));
sum += vaddvq_f32(vcvt_f32_f16(_s4)); // dot

kptr += 8;
}
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[j] = sum;
}

outptr += outw;
}
}
}
}

if (elempack == 8 && out_elempack == 4)
{
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output / out_elempack; p++)
{
__fp16* outptr = top_blob.channel(p);

for (int i = 0; i < outh; i++)
{
for (int j = 0; j < outw; j++)
{
float16x4_t _sum = vdup_n_f16((__fp16)0.f);

if (bias_term)
{
_sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4);
}

const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
{
const Mat m = bottom_blob_bordered.channel(q);
const __fp16* sptr = m.row<const __fp16>(i * stride_h) + j * stride_w * 8;

for (int k = 0; k < maxk; k++)
{
float16x8_t _val = vld1q_f16(sptr + space_ofs[k] * 8);

float16x4_t _w0 = vld1_f16(kptr);
float16x4_t _w1 = vld1_f16(kptr + 4);
float16x4_t _w2 = vld1_f16(kptr + 8);
float16x4_t _w3 = vld1_f16(kptr + 12);
float16x4_t _w4 = vld1_f16(kptr + 16);
float16x4_t _w5 = vld1_f16(kptr + 20);
float16x4_t _w6 = vld1_f16(kptr + 24);
float16x4_t _w7 = vld1_f16(kptr + 28);

_sum = vfma_laneq_f16(_sum, _w0, _val, 0);
_sum = vfma_laneq_f16(_sum, _w1, _val, 1);
_sum = vfma_laneq_f16(_sum, _w2, _val, 2);
_sum = vfma_laneq_f16(_sum, _w3, _val, 3);
_sum = vfma_laneq_f16(_sum, _w4, _val, 4);
_sum = vfma_laneq_f16(_sum, _w5, _val, 5);
_sum = vfma_laneq_f16(_sum, _w6, _val, 6);
_sum = vfma_laneq_f16(_sum, _w7, _val, 7);

kptr += 32;
}
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr + j * 4, _sum);
}

outptr += outw * 4;
}
}
}
}

if (elempack == 4 && out_elempack == 4)
{
{
@@ -1553,7 +1724,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const
_sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4);
}

const __fp16* kptr = weight_data_pack4_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
@@ -1610,7 +1781,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const
_sum = vld1_f16(((const __fp16*)bias_data_fp16) + p * 4);
}

const __fp16* kptr = weight_data_pack1to4_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)
@@ -1659,7 +1830,7 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const
sum = bias_data[p];
}

const __fp16* kptr = weight_data_pack4to1_fp16.channel(p);
const __fp16* kptr = weight_data_fp16.channel(p);

// channels
for (int q = 0; q < channels; q++)


+ 0
- 3
src/layer/arm/convolution_arm.h View File

@@ -59,9 +59,6 @@ public:
Mat weight_data_pack4to1;

// fp16
Mat weight_data_pack4_fp16;
Mat weight_data_pack1to4_fp16;
Mat weight_data_pack4to1_fp16;
Mat weight_data_fp16;
Mat bias_data_fp16;



+ 211
- 73
src/layer/arm/flatten_arm.cpp View File

@@ -52,38 +52,46 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
return 0;
}

#if __ARM_NEON
int w = bottom_blob.w;
int h = bottom_blob.h;
int channels = bottom_blob.c;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;
int size = w * h;

int total = size * channels * elempack;

int out_elempack = 1;
if (opt.use_packing_layout)
{
int w = bottom_blob.w;
int h = bottom_blob.h;
int channels = bottom_blob.c;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;
int size = w * h;

int total = size * channels * elempack;
out_elempack = total % 4 == 0 ? 4 : 1;
}
size_t out_elemsize = elemsize / elempack * out_elempack;

int out_elempack = total % 4 == 0 ? 4 : 1;
size_t out_elemsize = elemsize / elempack * out_elempack;
if (out_elempack == 1)
{
return Flatten::forward(bottom_blob, top_blob, opt);
}

if (dims == 2 && elempack == 1)
{
top_blob = bottom_blob;
top_blob.dims = 1;
top_blob.w = total / out_elempack;
top_blob.h = 1;
top_blob.cstep = top_blob.w;
top_blob.elemsize = out_elemsize;
top_blob.elempack = out_elempack;
return 0;
}
if (dims == 2 && elempack == 1) // out_elempack == 4
{
top_blob = bottom_blob;
top_blob.dims = 1;
top_blob.w = total / out_elempack;
top_blob.h = 1;
top_blob.cstep = top_blob.w;
top_blob.elemsize = out_elemsize;
top_blob.elempack = out_elempack;
return 0;
}

top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;
top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

if (dims == 2 && elempack == 4)
if (dims == 2)
{
if (elempack == 4) // out_elempack == 4
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int i = 0; i < h; i++)
@@ -95,6 +103,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
float* outptr3 = (float*)top_blob + w * (i * 4 + 3);

int j = 0;
#if __ARM_NEON
for (; j + 3 < w; j += 4)
{
float32x4x4_t _v4 = vld4q_f32(ptr);
@@ -109,6 +118,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
outptr2 += 4;
outptr3 += 4;
}
#endif
for (; j < w; j++)
{
*outptr0++ = ptr[0];
@@ -119,11 +129,12 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
ptr += 4;
}
}

return 0;
}
}

if (dims == 3 && elempack == 4)
if (dims == 3)
{
if (elempack == 4) // out_elempack == 4
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < channels; q++)
@@ -135,6 +146,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
float* outptr3 = (float*)top_blob + size * (q * 4 + 3);

int i = 0;
#if __ARM_NEON
for (; i + 3 < size; i += 4)
{
float32x4x4_t _v4 = vld4q_f32(ptr);
@@ -149,6 +161,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
outptr2 += 4;
outptr3 += 4;
}
#endif
for (; i < size; i++)
{
*outptr0++ = ptr[0];
@@ -159,11 +172,9 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
ptr += 4;
}
}

return 0;
}

if (dims == 3 && elempack == 1 && out_elempack == 4)
if (elempack == 1) // out_elempack == 4
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < channels; q++)
@@ -172,6 +183,7 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
float* outptr = (float*)top_blob + size * q;

int i = 0;
#if __ARM_NEON
for (; i + 3 < size; i += 4)
{
float32x4_t _v = vld1q_f32(ptr);
@@ -179,19 +191,16 @@ int Flatten_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
ptr += 4;
outptr += 4;
}
#endif
for (; i < size; i++)
{
*outptr++ = *ptr++;
}
}

return 0;
}
}

} // opt.use_packing_layout
#endif // __ARM_NEON

return Flatten::forward(bottom_blob, top_blob, opt);
return 0;
}

int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
@@ -204,38 +213,106 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
return 0;
}

#if __ARM_NEON
int w = bottom_blob.w;
int h = bottom_blob.h;
int channels = bottom_blob.c;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;
int size = w * h;

int total = size * channels * elempack;

int out_elempack = 1;
if (opt.use_packing_layout)
{
int w = bottom_blob.w;
int h = bottom_blob.h;
int channels = bottom_blob.c;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;
int size = w * h;
out_elempack = opt.use_fp16_arithmetic && total % 8 == 0 ? 8 : total % 4 == 0 ? 4 : 1;
}
size_t out_elemsize = elemsize / elempack * out_elempack;

int total = size * channels * elempack;
if (out_elempack == 1)
{
return Flatten::forward(bottom_blob, top_blob, opt);
}

int out_elempack = total % 4 == 0 ? 4 : 1;
size_t out_elemsize = elemsize / elempack * out_elempack;
if (dims == 2 && elempack == 1) // out_elempack == 4 || out_elempack == 8
{
top_blob = bottom_blob;
top_blob.dims = 1;
top_blob.w = total / out_elempack;
top_blob.h = 1;
top_blob.cstep = top_blob.w;
top_blob.elemsize = out_elemsize;
top_blob.elempack = out_elempack;
return 0;
}

if (dims == 2 && elempack == 1)
top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

if (dims == 2)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (elempack == 8) // out_elempack == 8
{
top_blob = bottom_blob;
top_blob.dims = 1;
top_blob.w = total / out_elempack;
top_blob.h = 1;
top_blob.cstep = top_blob.w;
top_blob.elemsize = out_elemsize;
top_blob.elempack = out_elempack;
return 0;
}
#pragma omp parallel for num_threads(opt.num_threads)
for (int i = 0; i < h; i++)
{
const __fp16* ptr = bottom_blob.row<const __fp16>(i);
__fp16* outptr0 = (__fp16*)top_blob + w * i * 8;
__fp16* outptr1 = (__fp16*)top_blob + w * (i * 8 + 1);
__fp16* outptr2 = (__fp16*)top_blob + w * (i * 8 + 2);
__fp16* outptr3 = (__fp16*)top_blob + w * (i * 8 + 3);
__fp16* outptr4 = (__fp16*)top_blob + w * (i * 8 + 4);
__fp16* outptr5 = (__fp16*)top_blob + w * (i * 8 + 5);
__fp16* outptr6 = (__fp16*)top_blob + w * (i * 8 + 6);
__fp16* outptr7 = (__fp16*)top_blob + w * (i * 8 + 7);

int j = 0;
for (; j + 3 < w; j += 4)
{
float16x8x4_t _v4 = vld4q_f16(ptr);
float16x8_t _v_01 = vuzp1q_f16(_v4.val[0], _v4.val[1]);
float16x8_t _v_23 = vuzp1q_f16(_v4.val[2], _v4.val[3]);
float16x8_t _v_45 = vuzp2q_f16(_v4.val[0], _v4.val[1]);
float16x8_t _v_67 = vuzp2q_f16(_v4.val[2], _v4.val[3]);
vst1_f16(outptr0, vget_low_f16(_v_01));
vst1_f16(outptr1, vget_high_f16(_v_01));
vst1_f16(outptr2, vget_low_f16(_v_23));
vst1_f16(outptr3, vget_high_f16(_v_23));
vst1_f16(outptr4, vget_low_f16(_v_45));
vst1_f16(outptr5, vget_high_f16(_v_45));
vst1_f16(outptr6, vget_low_f16(_v_67));
vst1_f16(outptr7, vget_high_f16(_v_67));

ptr += 32;
outptr0 += 4;
outptr1 += 4;
outptr2 += 4;
outptr3 += 4;
outptr4 += 4;
outptr5 += 4;
outptr6 += 4;
outptr7 += 4;
}
for (; j < w; j++)
{
*outptr0++ = ptr[0];
*outptr1++ = ptr[1];
*outptr2++ = ptr[2];
*outptr3++ = ptr[3];
*outptr4++ = ptr[4];
*outptr5++ = ptr[5];
*outptr6++ = ptr[6];
*outptr7++ = ptr[7];

top_blob.create(total / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;
ptr += 8;
}
}
}
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

if (dims == 2 && elempack == 4)
if (elempack == 4) // out_elempack == 4 || out_elempack == 8
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int i = 0; i < h; i++)
@@ -247,6 +324,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
unsigned short* outptr3 = (unsigned short*)top_blob + w * (i * 4 + 3);

int j = 0;
#if __ARM_NEON
for (; j + 3 < w; j += 4)
{
uint16x4x4_t _v4 = vld4_u16(ptr);
@@ -261,6 +339,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
outptr2 += 4;
outptr3 += 4;
}
#endif
for (; j < w; j++)
{
*outptr0++ = ptr[0];
@@ -271,11 +350,72 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
ptr += 4;
}
}
}
}

if (dims == 3)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (elempack == 8) // out_elempack == 8
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < channels; q++)
{
const __fp16* ptr = bottom_blob.channel(q);
__fp16* outptr0 = (__fp16*)top_blob + size * q * 8;
__fp16* outptr1 = (__fp16*)top_blob + size * (q * 8 + 1);
__fp16* outptr2 = (__fp16*)top_blob + size * (q * 8 + 2);
__fp16* outptr3 = (__fp16*)top_blob + size * (q * 8 + 3);
__fp16* outptr4 = (__fp16*)top_blob + size * (q * 8 + 4);
__fp16* outptr5 = (__fp16*)top_blob + size * (q * 8 + 5);
__fp16* outptr6 = (__fp16*)top_blob + size * (q * 8 + 6);
__fp16* outptr7 = (__fp16*)top_blob + size * (q * 8 + 7);

int i = 0;
for (; i + 3 < size; i += 4)
{
float16x8x4_t _v4 = vld4q_f16(ptr);
float16x8_t _v_01 = vuzp1q_f16(_v4.val[0], _v4.val[1]);
float16x8_t _v_23 = vuzp1q_f16(_v4.val[2], _v4.val[3]);
float16x8_t _v_45 = vuzp2q_f16(_v4.val[0], _v4.val[1]);
float16x8_t _v_67 = vuzp2q_f16(_v4.val[2], _v4.val[3]);
vst1_f16(outptr0, vget_low_f16(_v_01));
vst1_f16(outptr1, vget_high_f16(_v_01));
vst1_f16(outptr2, vget_low_f16(_v_23));
vst1_f16(outptr3, vget_high_f16(_v_23));
vst1_f16(outptr4, vget_low_f16(_v_45));
vst1_f16(outptr5, vget_high_f16(_v_45));
vst1_f16(outptr6, vget_low_f16(_v_67));
vst1_f16(outptr7, vget_high_f16(_v_67));

ptr += 32;
outptr0 += 4;
outptr1 += 4;
outptr2 += 4;
outptr3 += 4;
outptr4 += 4;
outptr5 += 4;
outptr6 += 4;
outptr7 += 4;
}
for (; i < size; i++)
{
*outptr0++ = ptr[0];
*outptr1++ = ptr[1];
*outptr2++ = ptr[2];
*outptr3++ = ptr[3];
*outptr4++ = ptr[4];
*outptr5++ = ptr[5];
*outptr6++ = ptr[6];
*outptr7++ = ptr[7];

return 0;
ptr += 8;
}
}
}
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

if (dims == 3 && elempack == 4)
if (elempack == 4) // out_elempack == 4 || out_elempack == 8
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < channels; q++)
@@ -287,6 +427,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
unsigned short* outptr3 = (unsigned short*)top_blob + size * (q * 4 + 3);

int i = 0;
#if __ARM_NEON
for (; i + 3 < size; i += 4)
{
uint16x4x4_t _v4 = vld4_u16(ptr);
@@ -301,6 +442,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
outptr2 += 4;
outptr3 += 4;
}
#endif
for (; i < size; i++)
{
*outptr0++ = ptr[0];
@@ -311,11 +453,9 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
ptr += 4;
}
}

return 0;
}

if (dims == 3 && elempack == 1 && out_elempack == 4)
if (elempack == 1) // out_elempack == 4 || out_elempack == 8
{
#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < channels; q++)
@@ -324,6 +464,7 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
unsigned short* outptr = (unsigned short*)top_blob + size * q;

int i = 0;
#if __ARM_NEON
for (; i + 3 < size; i += 4)
{
uint16x4_t _v = vld1_u16(ptr);
@@ -331,19 +472,16 @@ int Flatten_arm::forward_bf16s_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
ptr += 4;
outptr += 4;
}
#endif
for (; i < size; i++)
{
*outptr++ = *ptr++;
}
}

return 0;
}
}

} // opt.use_packing_layout
#endif // __ARM_NEON

return Flatten::forward(bottom_blob, top_blob, opt);
return 0;
}

} // namespace ncnn

+ 1
- 1
src/layer/arm/innerproduct_arm.cpp View File

@@ -657,7 +657,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons
int elempack = bottom_blob.elempack;
int size = w * h;

if (elempack == 4)
if (elempack == 4 || elempack == 8)
{
// flatten
Mat bottom_blob_flattened = bottom_blob;


+ 9
- 0
src/layer/arm/neon_activation.h View File

@@ -106,5 +106,14 @@ static inline float16x4_t activation_ps(float16x4_t _v, int activation_type, con
_v32 = activation_ps(_v32, activation_type, activation_params);
return vcvt_f16_f32(_v32);
}

static inline float16x8_t activation_ps(float16x8_t _v, int activation_type, const ncnn::Mat& activation_params)
{
float32x4_t _v32_low = vcvt_f32_f16(vget_low_f16(_v));
float32x4_t _v32_high = vcvt_f32_f16(vget_high_f16(_v));
_v32_low = activation_ps(_v32_low, activation_type, activation_params);
_v32_high = activation_ps(_v32_high, activation_type, activation_params);
return vcombine_f16(vcvt_f16_f32(_v32_low), vcvt_f16_f32(_v32_high));
}
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
#endif // __ARM_NEON

Write Preview
Loading…
Cancel
Save