nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

arm vfpv4 optimization for innerproduct (#3950)

tags/20220701
nihui GitHub 4 years ago
parent
440bfdd2cc
commit
706831f8a9
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1104 additions and 336 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +99
    -0
      src/layer/arm/innerproduct_arm.cpp
  2. +3
    -1
      src/layer/arm/innerproduct_arm.h
  3. +17
    -333
      src/layer/arm/innerproduct_arm_asimdhp.cpp
  4. +51
    -0
      src/layer/arm/innerproduct_arm_vfpv4.cpp
  5. +534
    -0
      src/layer/arm/innerproduct_fp16s.h
  6. +398
    -0
      src/layer/arm/innerproduct_gemm_fp16s.h
  7. +2
    -2
      tests/testutil.h

+ 99
- 0
src/layer/arm/innerproduct_arm.cpp View File

@@ -27,6 +27,11 @@

namespace ncnn {

#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__
#include "innerproduct_fp16s.h"
#include "innerproduct_gemm_fp16s.h"
#endif

InnerProduct_arm::InnerProduct_arm()
{
#if __ARM_NEON
@@ -76,6 +81,13 @@ int InnerProduct_arm::create_pipeline(const Option& opt)
}
#endif

#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__
if (cpu_support_arm_vfpv4() && opt.use_fp16_storage)
{
return create_pipeline_fp16s(opt);
}
#endif

const int num_input = weight_data_size / num_output;

int out_elempack = 1;
@@ -161,6 +173,13 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
return forward_bf16s(bottom_blob, top_blob, opt);
#endif

#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__
if (cpu_support_arm_vfpv4() && opt.use_fp16_storage)
{
return forward_fp16s(bottom_blob, top_blob, opt);
}
#endif

const int num_input = weight_data_size / num_output;

if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)
@@ -829,6 +848,86 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
return 0;
}

#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__
int InnerProduct_arm::create_pipeline_fp16s(const Option& opt)
{
const int num_input = weight_data_size / num_output;

innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt);

#if NCNN_ARM82
if (ncnn::cpu_support_arm_asimdhp() && opt.use_fp16_arithmetic)
{
ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);
}
#endif

if (opt.lightmode)
{
weight_data.release();
}

return 0;
}

int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
const int num_input = weight_data_size / num_output;

if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)
{
// gemm
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt);

return 0;
}

// flatten
Mat bottom_blob_flattened = bottom_blob;
if (bottom_blob.dims != 1)
{
Option opt_flatten = opt;
opt_flatten.blob_allocator = opt.workspace_allocator;

flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);
}

size_t elemsize = bottom_blob_flattened.elemsize;
int elempack = bottom_blob_flattened.elempack;

int out_elempack = 1;
if (opt.use_packing_layout)
{
out_elempack = num_output % 4 == 0 ? 4 : 1;
}
size_t out_elemsize = elemsize / elempack * out_elempack;

top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

if (out_elempack == 4)
{
innerproduct_fp16s_pack4_neon(bottom_blob_flattened, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt);
}

if (out_elempack == 1)
{
innerproduct_fp16s_neon(bottom_blob_flattened, top_blob, weight_data_tm, bias_data, activation_type, activation_params, opt);
}

return 0;
}
#endif // (NCNN_VFPV4 && __ARM_NEON) || __aarch64__

#if NCNN_BF16
int InnerProduct_arm::create_pipeline_bf16s(const Option& opt)
{


+ 3
- 1
src/layer/arm/innerproduct_arm.h View File

@@ -32,9 +32,11 @@ public:
virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;

protected:
#if NCNN_ARM82
#if (NCNN_VFPV4 && __ARM_NEON) || __aarch64__
int create_pipeline_fp16s(const Option& opt);
int forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
#endif
#if NCNN_ARM82
int forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
#endif
#if NCNN_BF16


+ 17
- 333
src/layer/arm/innerproduct_arm_asimdhp.cpp View File

@@ -14,7 +14,7 @@

#include "innerproduct_arm.h"

#include "layer_type.h"
#include "cpu.h"

#if __ARM_NEON
#include <arm_neon.h>
@@ -25,346 +25,30 @@

namespace ncnn {

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
int InnerProduct_arm::create_pipeline_fp16s(const Option& opt)
{
const int num_input = weight_data_size / num_output;

int out_elempack = 1;
#include "innerproduct_fp16s.h"
#include "innerproduct_gemm_fp16s.h"

if (opt.use_packing_layout)
{
out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
}

// src = inch-outch
// dst = pb-inch-outch/pb
{
Mat weight_data_r2 = weight_data.reshape(num_input, num_output);

weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
__fp16* g0 = weight_data_tm.row<__fp16>(q / out_elempack);

for (int p = 0; p < num_input; p++)
{
for (int j = 0; j < out_elempack; j++)
{
*g0++ = (__fp16)(weight_data_r2.row(q + j)[p]);
}
}
}
}

ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);

if (opt.lightmode)
{
weight_data.release();
}

return 0;
void innerproduct_fp16s_pack4_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
innerproduct_fp16s_pack4_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
void innerproduct_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
const int num_input = weight_data_size / num_output;

if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)
{
// gemm
int h = bottom_blob.h;
size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

top_blob.create(num_output, h, elemsize, elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

int num_output_elempack = 1;
if (opt.use_packing_layout)
{
num_output_elempack = num_output % 4 == 0 ? 4 : 1;
}

#pragma omp parallel for num_threads(opt.num_threads)
for (int j = 0; j < h; j++)
{
if (elempack == 4 && num_output_elempack == 4)
{
__fp16* outptr = top_blob.row<__fp16>(j);

for (int p = 0; p < num_output / num_output_elempack; p++)
{
const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4;
const __fp16* m = bottom_blob.row<const __fp16>(j);

float32x4_t _sum0 = vdupq_n_f32(0.f);
float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);

if (bias_term)
{
_sum0 = vdupq_n_f32(bias_data[p * 4 + 0]);
_sum1 = vdupq_n_f32(bias_data[p * 4 + 1]);
_sum2 = vdupq_n_f32(bias_data[p * 4 + 2]);
_sum3 = vdupq_n_f32(bias_data[p * 4 + 3]);
}

for (int i = 0; i < num_input; i++)
{
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float32x4_t _k = vcvt_f32_f16(vld1_f16(kptr));
_sum0 = vfmaq_laneq_f32(_sum0, _val, _k, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _val, _k, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _val, _k, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _val, _k, 3);

m += 4;
kptr += 4;
}

_sum0 = activation_ps(_sum0, activation_type, activation_params);
_sum1 = activation_ps(_sum1, activation_type, activation_params);
_sum2 = activation_ps(_sum2, activation_type, activation_params);
_sum3 = activation_ps(_sum3, activation_type, activation_params);

vst1_f16(outptr, vcvt_f16_f32(_sum0));
vst1_f16(outptr + 4, vcvt_f16_f32(_sum1));
vst1_f16(outptr + 8, vcvt_f16_f32(_sum2));
vst1_f16(outptr + 12, vcvt_f16_f32(_sum3));
outptr += 16;
}
}

if (elempack == 1 && num_output_elempack == 4)
{
__fp16* outptr = top_blob.row<__fp16>(j);

for (int p = 0; p < num_output / num_output_elempack; p++)
{
const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4;
const __fp16* m = bottom_blob.row<const __fp16>(j);

float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32((const float*)bias_data + p * 4);
}

for (int i = 0; i < num_input; i++)
{
float32x4_t _val = vdupq_n_f32((float)m[0]);
float32x4_t _k = vcvt_f32_f16(vld1_f16(kptr));
_sum = vfmaq_f32(_sum, _val, _k);

m += 1;
kptr += 4;
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, vcvt_f16_f32(_sum));
outptr += 4;
}
}

if (elempack == 4 && num_output_elempack == 1)
{
__fp16* outptr = top_blob.row<__fp16>(j);

for (int p = 0; p < num_output; p++)
{
const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p;
const __fp16* m = bottom_blob.row<const __fp16>(j);

float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vdupq_n_f32(bias_data[p]);
}

for (int i = 0; i < num_input; i++)
{
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float32x4_t _k = vdupq_n_f32((float)kptr[0]);
_sum = vfmaq_f32(_sum, _val, _k);

m += 4;
kptr += 1;
}

_sum = activation_ps(_sum, activation_type, activation_params);

vst1_f16(outptr, vcvt_f16_f32(_sum));
outptr += 4;
}
}

if (elempack == 1 && num_output_elempack == 1)
{
__fp16* outptr = top_blob.row<__fp16>(j);

for (int p = 0; p < num_output; p++)
{
const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p;
const __fp16* m = bottom_blob.row<const __fp16>(j);

float sum = 0.f;

if (bias_term)
{
sum = bias_data[p];
}

for (int i = 0; i < num_input; i++)
{
sum += (float)*m * (float)*kptr;

m += 1;
kptr += 1;
}

sum = activation_ss(sum, activation_type, activation_params);

outptr[0] = (__fp16)sum;
outptr += 1;
}
}
}

return 0;
}

// flatten
Mat bottom_blob_flattened = bottom_blob;
if (bottom_blob.dims != 1)
{
Option opt_flatten = opt;
opt_flatten.blob_allocator = opt.workspace_allocator;

flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);
}

size_t elemsize = bottom_blob_flattened.elemsize;
int elempack = bottom_blob_flattened.elempack;

int out_elempack = opt.use_packing_layout && num_output % 4 == 0 ? 4 : 1;
size_t out_elemsize = elemsize / elempack * out_elempack;

top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

if (out_elempack == 4)
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output / out_elempack; p++)
{
float32x4_t _sum = vdupq_n_f32(0.f);

if (bias_term)
{
_sum = vld1q_f32(((const float*)bias_data) + p * 4);
}

const __fp16* kptr = weight_data_tm.row<const __fp16>(p);

const __fp16* sptr = bottom_blob_flattened;

int i = 0;
for (; i + 3 < num_input; i += 4)
{
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));

float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr));
float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr + 4));
float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr + 8));
float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr + 12));

_sum = vfmaq_laneq_f32(_sum, _w0, _val, 0);
_sum = vfmaq_laneq_f32(_sum, _w1, _val, 1);
_sum = vfmaq_laneq_f32(_sum, _w2, _val, 2);
_sum = vfmaq_laneq_f32(_sum, _w3, _val, 3);

sptr += 4;
kptr += 16;
}
for (; i < num_input; i++)
{
float32x4_t _val = vdupq_n_f32((float)sptr[0]);

float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));

_sum = vfmaq_f32(_sum, _val, _w);

sptr += 1;
kptr += 4;
}

_sum = activation_ps(_sum, activation_type, activation_params);

__fp16* outptr = (__fp16*)top_blob;
vst1_f16(outptr + p * 4, vcvt_f16_f32(_sum));
}
}

if (out_elempack == 1)
{
// num_output
#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output; p++)
{
float sum = 0.f;

if (bias_term)
sum = bias_data[p];

const __fp16* kptr = weight_data_tm.row<__fp16>(p);

const __fp16* sptr = bottom_blob_flattened;

float32x4_t _sum = vdupq_n_f32(0.f);
int i = 0;
for (; i + 3 < num_input; i += 4)
{
float32x4_t _m = vcvt_f32_f16(vld1_f16(sptr));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));

_sum = vfmaq_f32(_sum, _m, _w);

sptr += 4;
kptr += 4;
}
for (; i < num_input; i++)
{
float v = (float)(*sptr);
float k = (float)(*kptr);

sum += v * k;

sptr++;
kptr++;
}

sum += vaddvq_f32(_sum);

sum = activation_ss(sum, activation_type, activation_params);
innerproduct_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

__fp16* outptr = (__fp16*)top_blob;
outptr[p] = (__fp16)sum;
}
}
void innerproduct_gemm_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

return 0;
void innerproduct_transform_kernel_fp16s_neon_asimdhp(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt)
{
innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt);
}

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
{
const int num_input = weight_data_size / num_output;


+ 51
- 0
src/layer/arm/innerproduct_arm_vfpv4.cpp View File

@@ -0,0 +1,51 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#include "innerproduct_arm.h"

#include "cpu.h"

#if __ARM_NEON
#include <arm_neon.h>
#endif // __ARM_NEON

#include "arm_activation.h"
#include "arm_usability.h"

namespace ncnn {

#include "innerproduct_fp16s.h"
#include "innerproduct_gemm_fp16s.h"

void innerproduct_fp16s_pack4_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
innerproduct_fp16s_pack4_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

void innerproduct_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
innerproduct_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

void innerproduct_gemm_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
innerproduct_gemm_fp16s_neon(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
}

void innerproduct_transform_kernel_fp16s_neon_vfpv4(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt)
{
innerproduct_transform_kernel_fp16s_neon(weight_data, weight_data_tm, num_input, num_output, opt);
}

} // namespace ncnn

+ 534
- 0
src/layer/arm/innerproduct_fp16s.h View File

@@ -0,0 +1,534 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
void innerproduct_fp16s_pack4_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
void innerproduct_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
void innerproduct_transform_kernel_fp16s_neon_asimdhp(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt);
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
void innerproduct_fp16s_pack4_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
void innerproduct_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
void innerproduct_transform_kernel_fp16s_neon_vfpv4(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt);
#endif

static void innerproduct_fp16s_pack4_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (ncnn::cpu_support_arm_asimdhp())
{
innerproduct_fp16s_pack4_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
if (ncnn::cpu_support_arm_vfpv4())
{
innerproduct_fp16s_pack4_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if (__ARM_FP & 2)
const int num_input = bottom_blob.w * bottom_blob.elempack;
const int num_output = top_blob.w;

const float* bias_data_ptr = bias_data;

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < num_output; p++)
{
float32x4_t _sum0 = vdupq_n_f32(0.f);

if (bias_data_ptr)
{
_sum0 = vld1q_f32(bias_data_ptr + p * 4);
}

float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* sptr = bottom_blob;
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* sptr = bottom_blob;
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

int i = 0;
for (; i + 7 < num_input; i += 8)
{
#if __aarch64__
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
asm volatile(
"prfm pldl1keep, [%0, #128] \n"
"ld1 {v1.8h}, [%0], #16 \n"
"prfm pldl1keep, [%1, #512] \n"
"ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [%1], #64 \n"
"fcvtl v0.4s, v1.4h \n"
"fcvtl2 v1.4s, v1.8h \n"
"fcvtl v2.4s, v6.4h \n"
"fcvtl2 v3.4s, v6.8h \n"
"fcvtl v4.4s, v7.4h \n"
"fcvtl2 v5.4s, v7.8h \n"
"fcvtl v6.4s, v8.4h \n"
"fcvtl2 v7.4s, v8.8h \n"
"fcvtl v8.4s, v9.4h \n"
"fcvtl2 v9.4s, v9.8h \n"
"fmla %2.4s, v2.4s, v0.s[0] \n"
"fmla %3.4s, v3.4s, v0.s[1] \n"
"fmla %4.4s, v4.4s, v0.s[2] \n"
"fmla %5.4s, v5.4s, v0.s[3] \n"
"fmla %2.4s, v6.4s, v1.s[0] \n"
"fmla %3.4s, v7.4s, v1.s[1] \n"
"fmla %4.4s, v8.4s, v1.s[2] \n"
"fmla %5.4s, v9.4s, v1.s[3] \n"
: "=r"(sptr), // %0
"=r"(kptr), // %1
"=w"(_sum0), // %2
"=w"(_sum1), // %3
"=w"(_sum2), // %4
"=w"(_sum3) // %5
: "0"(sptr),
"1"(kptr),
"2"(_sum0),
"3"(_sum1),
"4"(_sum2),
"5"(_sum3)
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9");
#else // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
asm volatile(
"prfm pldl1keep, [%0, #256] \n"
"ld1 {v0.4s, v1.4s}, [%0], #32 \n"
"prfm pldl1keep, [%1, #512] \n"
"ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [%1], #64 \n"
"fcvtl v2.4s, v6.4h \n"
"fcvtl2 v3.4s, v6.8h \n"
"fcvtl v4.4s, v7.4h \n"
"fcvtl2 v5.4s, v7.8h \n"
"fcvtl v6.4s, v8.4h \n"
"fcvtl2 v7.4s, v8.8h \n"
"fcvtl v8.4s, v9.4h \n"
"fcvtl2 v9.4s, v9.8h \n"
"fmla %2.4s, v2.4s, v0.s[0] \n"
"fmla %3.4s, v3.4s, v0.s[1] \n"
"fmla %4.4s, v4.4s, v0.s[2] \n"
"fmla %5.4s, v5.4s, v0.s[3] \n"
"fmla %2.4s, v6.4s, v1.s[0] \n"
"fmla %3.4s, v7.4s, v1.s[1] \n"
"fmla %4.4s, v8.4s, v1.s[2] \n"
"fmla %5.4s, v9.4s, v1.s[3] \n"
: "=r"(sptr), // %0
"=r"(kptr), // %1
"=w"(_sum0), // %2
"=w"(_sum1), // %3
"=w"(_sum2), // %4
"=w"(_sum3) // %5
: "0"(sptr),
"1"(kptr),
"2"(_sum0),
"3"(_sum1),
"4"(_sum2),
"5"(_sum3)
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9");
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
#else // __aarch64__
asm volatile(
"pld [%0, #256] \n"
"vld1.f32 {d0-d3}, [%0 :128]! \n"
"pld [%1, #512] \n"
"vldm %1!, {d12-d19} \n"
"vcvt.f32.f16 q2, d12 \n"
"vcvt.f32.f16 q3, d13 \n"
"vcvt.f32.f16 q4, d14 \n"
"vcvt.f32.f16 q5, d15 \n"
"vcvt.f32.f16 q6, d16 \n"
"vcvt.f32.f16 q7, d17 \n"
"vcvt.f32.f16 q8, d18 \n"
"vcvt.f32.f16 q9, d19 \n"
"vmla.f32 %q2, q2, d0[0] \n"
"vmla.f32 %q3, q3, d0[1] \n"
"vmla.f32 %q4, q4, d1[0] \n"
"vmla.f32 %q5, q5, d1[1] \n"
"vmla.f32 %q2, q6, d2[0] \n"
"vmla.f32 %q3, q7, d2[1] \n"
"vmla.f32 %q4, q8, d3[0] \n"
"vmla.f32 %q5, q9, d3[1] \n"
: "=r"(sptr), // %0
"=r"(kptr), // %1
"=w"(_sum0), // %2
"=w"(_sum1), // %3
"=w"(_sum2), // %4
"=w"(_sum3) // %5
: "0"(sptr),
"1"(kptr),
"2"(_sum0),
"3"(_sum1),
"4"(_sum2),
"5"(_sum3)
: "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9");
#endif // __aarch64__
}
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));
float16x8_t _w01 = vld1q_f16(kptr);
float16x8_t _w23 = vld1q_f16(kptr + 8);
float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));
float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));
float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));
float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));
#else
float32x4_t _val = vld1q_f32(sptr);
uint16x8_t _w01 = vld1q_u16(kptr);
uint16x8_t _w23 = vld1q_u16(kptr + 8);
float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w01)));
float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w01)));
float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w23)));
float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w23)));
#endif

#if __aarch64__
_sum0 = vfmaq_laneq_f32(_sum0, _w0, _val, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _w1, _val, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _w2, _val, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _w3, _val, 3);
#else
_sum0 = vmlaq_lane_f32(_sum0, _w0, vget_low_f32(_val), 0);
_sum1 = vmlaq_lane_f32(_sum1, _w1, vget_low_f32(_val), 1);
_sum2 = vmlaq_lane_f32(_sum2, _w2, vget_high_f32(_val), 0);
_sum3 = vmlaq_lane_f32(_sum3, _w3, vget_high_f32(_val), 1);
#endif

sptr += 4;
kptr += 16;
}
for (; i < num_input; i++)
{
float32x4_t _val = vdupq_n_f32((float)sptr[0]);
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif
_sum0 = vfmaq_f32(_sum0, _val, _w);

sptr += 1;
kptr += 4;
}

_sum0 = vaddq_f32(_sum0, _sum1);
_sum2 = vaddq_f32(_sum2, _sum3);
_sum0 = vaddq_f32(_sum0, _sum2);

_sum0 = activation_ps(_sum0, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = (__fp16*)top_blob;
vst1_f16(outptr + p * 4, vcvt_f16_f32(_sum0));
#else
float* outptr = top_blob;
vst1q_f32(outptr + p * 4, _sum0);
#endif
}
#else // (__ARM_FP & 2)
(void)bottom_blob;
(void)top_blob;
(void)weight_data_fp16;
(void)bias_data;
(void)activation_type;
(void)activation_params;
(void)opt;
#endif // (__ARM_FP & 2)
}

static void innerproduct_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (ncnn::cpu_support_arm_asimdhp())
{
innerproduct_fp16s_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
if (ncnn::cpu_support_arm_vfpv4())
{
innerproduct_fp16s_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if (__ARM_FP & 2)
const int num_input = bottom_blob.w * bottom_blob.elempack;
const int num_output = top_blob.w;

const float* bias_data_ptr = bias_data;

int nn_num_output = num_output >> 2;
int remain_num_output_start = nn_num_output << 2;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_num_output; pp++)
{
int p = pp * 4;

float sums[4] = {0.0f};
if (bias_data_ptr)
{
sums[0] = bias_data_ptr[p];
sums[1] = bias_data_ptr[p + 1];
sums[2] = bias_data_ptr[p + 2];
sums[3] = bias_data_ptr[p + 3];
}

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* sptr = bottom_blob;
const __fp16* kptr0 = weight_data_fp16.row<const __fp16>(p);
const __fp16* kptr1 = weight_data_fp16.row<const __fp16>(p + 1);
const __fp16* kptr2 = weight_data_fp16.row<const __fp16>(p + 2);
const __fp16* kptr3 = weight_data_fp16.row<const __fp16>(p + 3);
#else
const float* sptr = bottom_blob;
const unsigned short* kptr0 = weight_data_fp16.row<const unsigned short>(p);
const unsigned short* kptr1 = weight_data_fp16.row<const unsigned short>(p + 1);
const unsigned short* kptr2 = weight_data_fp16.row<const unsigned short>(p + 2);
const unsigned short* kptr3 = weight_data_fp16.row<const unsigned short>(p + 3);
#endif

int i = 0;

float32x4_t _sum0 = vdupq_n_f32(0.f);
float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));
float32x4_t _w0 = vcvt_f32_f16(vld1_f16(kptr0));
float32x4_t _w1 = vcvt_f32_f16(vld1_f16(kptr1));
float32x4_t _w2 = vcvt_f32_f16(vld1_f16(kptr2));
float32x4_t _w3 = vcvt_f32_f16(vld1_f16(kptr3));
#else
float32x4_t _val = vld1q_f32(sptr);
float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr0)));
float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr1)));
float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr2)));
float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr3)));
#endif

_sum0 = vfmaq_f32(_sum0, _val, _w0);
_sum1 = vfmaq_f32(_sum1, _val, _w1);
_sum2 = vfmaq_f32(_sum2, _val, _w2);
_sum3 = vfmaq_f32(_sum3, _val, _w3);

sptr += 4;
kptr0 += 4;
kptr1 += 4;
kptr2 += 4;
kptr3 += 4;
}

#if __aarch64__
sums[0] += vaddvq_f32(_sum0);
sums[1] += vaddvq_f32(_sum1);
sums[2] += vaddvq_f32(_sum2);
sums[3] += vaddvq_f32(_sum3);
#else
float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));
float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1));
float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2));
float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3));
float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss);
float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss);
sums[0] += vget_lane_f32(_sum01ss, 0);
sums[1] += vget_lane_f32(_sum01ss, 1);
sums[2] += vget_lane_f32(_sum23ss, 0);
sums[3] += vget_lane_f32(_sum23ss, 1);
#endif

for (; i < num_input; i++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
sums[0] += (float)(*sptr) * (float)(*kptr0);
sums[1] += (float)(*sptr) * (float)(*kptr1);
sums[2] += (float)(*sptr) * (float)(*kptr2);
sums[3] += (float)(*sptr) * (float)(*kptr3);
#else
sums[0] += *sptr * float16_to_float32(*kptr0);
sums[1] += *sptr * float16_to_float32(*kptr1);
sums[2] += *sptr * float16_to_float32(*kptr2);
sums[3] += *sptr * float16_to_float32(*kptr3);
#endif

sptr++;
kptr0++;
kptr1++;
kptr2++;
kptr3++;
}

float32x4_t _sum = vld1q_f32(sums);

_sum = activation_ps(_sum, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = (__fp16*)top_blob;
vst1_f16(outptr + p, vcvt_f16_f32(_sum));
#else
float* outptr = top_blob;
vst1q_f32(outptr + p, _sum);
#endif
}

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_num_output_start; p < num_output; p++)
{
float sum = 0.f;

if (bias_data_ptr)
sum = bias_data_ptr[p];

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* sptr = bottom_blob;
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* sptr = bottom_blob;
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

int i = 0;

float32x4_t _sum = vdupq_n_f32(0.f);
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(sptr));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _val = vld1q_f32(sptr);
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif
_sum = vfmaq_f32(_sum, _val, _w);

sptr += 4;
kptr += 4;
}
for (; i < num_input; i++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
sum += (float)(*sptr) * (float)(*kptr);
#else
sum += *sptr * float16_to_float32(*kptr);
#endif
sptr++;
kptr++;
}

#if __aarch64__
sum += vaddvq_f32(_sum);
#else
float32x2_t _sumss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));
_sumss = vpadd_f32(_sumss, _sumss);
sum += vget_lane_f32(_sumss, 0);
#endif // __aarch64__

sum = activation_ss(sum, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = (__fp16*)top_blob;
outptr[p] = (__fp16)sum;
#else
float* outptr = top_blob;
outptr[p] = sum;
#endif
}
#else // (__ARM_FP & 2)
(void)bottom_blob;
(void)top_blob;
(void)weight_data_fp16;
(void)bias_data;
(void)activation_type;
(void)activation_params;
(void)opt;
#endif // (__ARM_FP & 2)
}

static void innerproduct_transform_kernel_fp16s_neon(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, const Option& opt)
{
#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (ncnn::cpu_support_arm_asimdhp())
{
innerproduct_transform_kernel_fp16s_neon_asimdhp(weight_data, weight_data_tm, num_input, num_output, opt);
return;
}
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
if (ncnn::cpu_support_arm_vfpv4())
{
innerproduct_transform_kernel_fp16s_neon_vfpv4(weight_data, weight_data_tm, num_input, num_output, opt);
return;
}
#endif

#if (__ARM_FP & 2)
int out_elempack = 1;
if (opt.use_packing_layout)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
out_elempack = opt.use_fp16_arithmetic && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
#else
out_elempack = num_output % 4 == 0 ? 4 : 1;
#endif
}

Mat weight_data_fp16;
ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt);

// src = inch-outch
// dst = pb-inch-outch/pb
{
Mat weight_data_r2 = weight_data_fp16.reshape(num_input, num_output);

weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack);

for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
{
unsigned short* g0 = weight_data_tm.row<unsigned short>(q / out_elempack);

for (int p = 0; p < num_input; p++)
{
for (int j = 0; j < out_elempack; j++)
{
*g0++ = weight_data_r2.row<const unsigned short>(q + j)[p];
}
}
}
}
#else // (__ARM_FP & 2)
(void)weight_data;
(void)weight_data_tm;
(void)num_input;
(void)num_output;
(void)opt;
#endif // (__ARM_FP & 2)
}

+ 398
- 0
src/layer/arm/innerproduct_gemm_fp16s.h View File

@@ -0,0 +1,398 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
void innerproduct_gemm_fp16s_neon_asimdhp(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
void innerproduct_gemm_fp16s_neon_vfpv4(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt);
#endif

static void innerproduct_gemm_fp16s_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& weight_data_fp16, const Mat& bias_data, int activation_type, const Mat& activation_params, const Option& opt)
{
#if NCNN_RUNTIME_CPU && NCNN_ARM82 && __aarch64__ && !__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
if (ncnn::cpu_support_arm_asimdhp())
{
innerproduct_gemm_fp16s_neon_asimdhp(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if NCNN_RUNTIME_CPU && NCNN_VFPV4 && __ARM_NEON && !(__ARM_FP & 2)
if (ncnn::cpu_support_arm_vfpv4())
{
innerproduct_gemm_fp16s_neon_vfpv4(bottom_blob, top_blob, weight_data_fp16, bias_data, activation_type, activation_params, opt);
return;
}
#endif

#if (__ARM_FP & 2)
const int num_input = bottom_blob.w;
const int elempack = bottom_blob.elempack;
const int num_output = top_blob.w;
const int h = bottom_blob.h;

const float* bias_data_ptr = bias_data;

int num_output_elempack = 1;
if (opt.use_packing_layout)
{
num_output_elempack = num_output % 4 == 0 ? 4 : 1;
}

#pragma omp parallel for num_threads(opt.num_threads)
for (int j = 0; j < h; j++)
{
if (elempack == 4 && num_output_elempack == 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = top_blob.row<__fp16>(j);
#else
float* outptr = top_blob.row(j);
#endif

for (int p = 0; p < num_output / num_output_elempack; p++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* m = bottom_blob.row<const __fp16>(j);
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* m = bottom_blob.row(j);
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

float32x4_t _sum0 = vdupq_n_f32(0.f);
float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);

if (bias_data_ptr)
{
_sum0 = vdupq_n_f32(bias_data_ptr[p * 4 + 0]);
_sum1 = vdupq_n_f32(bias_data_ptr[p * 4 + 1]);
_sum2 = vdupq_n_f32(bias_data_ptr[p * 4 + 2]);
_sum3 = vdupq_n_f32(bias_data_ptr[p * 4 + 3]);
}

int i = 0;
for (; i < num_input; i++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _val = vld1q_f32(m);
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif

#if __aarch64__
_sum0 = vfmaq_laneq_f32(_sum0, _val, _w, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _val, _w, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _val, _w, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _val, _w, 3);
#else
_sum0 = vmlaq_lane_f32(_sum0, _val, vget_low_f32(_w), 0);
_sum1 = vmlaq_lane_f32(_sum1, _val, vget_low_f32(_w), 1);
_sum2 = vmlaq_lane_f32(_sum2, _val, vget_high_f32(_w), 0);
_sum3 = vmlaq_lane_f32(_sum3, _val, vget_high_f32(_w), 1);
#endif

m += 4;
kptr += 4;
}

_sum0 = activation_ps(_sum0, activation_type, activation_params);
_sum1 = activation_ps(_sum1, activation_type, activation_params);
_sum2 = activation_ps(_sum2, activation_type, activation_params);
_sum3 = activation_ps(_sum3, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
vst1_f16(outptr, vcvt_f16_f32(_sum0));
vst1_f16(outptr + 4, vcvt_f16_f32(_sum1));
vst1_f16(outptr + 8, vcvt_f16_f32(_sum2));
vst1_f16(outptr + 12, vcvt_f16_f32(_sum3));
#else
vst1q_f32(outptr, _sum0);
vst1q_f32(outptr + 4, _sum1);
vst1q_f32(outptr + 8, _sum2);
vst1q_f32(outptr + 12, _sum3);
#endif
outptr += 16;
}
}

if (elempack == 1 && num_output_elempack == 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = top_blob.row<__fp16>(j);
#else
float* outptr = top_blob.row(j);
#endif

for (int p = 0; p < num_output / num_output_elempack; p++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* m = bottom_blob.row<const __fp16>(j);
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* m = bottom_blob.row(j);
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

float32x4_t _sum0 = vdupq_n_f32(0.f);

if (bias_data_ptr)
{
_sum0 = vld1q_f32(bias_data_ptr + p * 4);
}

float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);

int i = 0;
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float16x8_t _w01 = vld1q_f16(kptr);
float16x8_t _w23 = vld1q_f16(kptr + 8);
float32x4_t _w0 = vcvt_f32_f16(vget_low_f16(_w01));
float32x4_t _w1 = vcvt_f32_f16(vget_high_f16(_w01));
float32x4_t _w2 = vcvt_f32_f16(vget_low_f16(_w23));
float32x4_t _w3 = vcvt_f32_f16(vget_high_f16(_w23));
#else
float32x4_t _val = vld1q_f32(m);
uint16x8_t _w01 = vld1q_u16(kptr);
uint16x8_t _w23 = vld1q_u16(kptr + 8);
float32x4_t _w0 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w01)));
float32x4_t _w1 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w01)));
float32x4_t _w2 = vcvt_f32_f16(vreinterpret_f16_u16(vget_low_u16(_w23)));
float32x4_t _w3 = vcvt_f32_f16(vreinterpret_f16_u16(vget_high_u16(_w23)));
#endif

#if __aarch64__
_sum0 = vfmaq_laneq_f32(_sum0, _w0, _val, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _w1, _val, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _w2, _val, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _w3, _val, 3);
#else
_sum0 = vmlaq_lane_f32(_sum0, _w0, vget_low_f32(_val), 0);
_sum1 = vmlaq_lane_f32(_sum1, _w1, vget_low_f32(_val), 1);
_sum2 = vmlaq_lane_f32(_sum2, _w2, vget_high_f32(_val), 0);
_sum3 = vmlaq_lane_f32(_sum3, _w3, vget_high_f32(_val), 1);
#endif

m += 4;
kptr += 16;
}
for (; i < num_input; i++)
{
float32x4_t _val = vdupq_n_f32((float)m[0]);
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif
_sum0 = vfmaq_f32(_sum0, _val, _w);

m += 1;
kptr += 4;
}

_sum0 = vaddq_f32(_sum0, _sum1);
_sum2 = vaddq_f32(_sum2, _sum3);
_sum0 = vaddq_f32(_sum0, _sum2);

_sum0 = activation_ps(_sum0, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
vst1_f16(outptr, vcvt_f16_f32(_sum0));
#else
vst1q_f32(outptr, _sum0);
#endif
outptr += 4;
}
}

if (elempack == 4 && num_output_elempack == 1)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = top_blob.row<__fp16>(j);
#else
float* outptr = top_blob.row(j);
#endif

for (int p = 0; p < num_output; p++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* m = bottom_blob.row<const __fp16>(j);
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* m = bottom_blob.row(j);
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

float32x4_t _sum0 = vdupq_n_f32(0.f);
float32x4_t _sum1 = vdupq_n_f32(0.f);
float32x4_t _sum2 = vdupq_n_f32(0.f);
float32x4_t _sum3 = vdupq_n_f32(0.f);

if (bias_data_ptr)
{
_sum0 = vdupq_n_f32(bias_data_ptr[p]);
}

int i = 0;
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val0 = vcvt_f32_f16(vld1_f16(m));
float32x4_t _val1 = vcvt_f32_f16(vld1_f16(m + 4));
float32x4_t _val2 = vcvt_f32_f16(vld1_f16(m + 8));
float32x4_t _val3 = vcvt_f32_f16(vld1_f16(m + 12));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _val0 = vld1q_f32(m);
float32x4_t _val1 = vld1q_f32(m + 4);
float32x4_t _val2 = vld1q_f32(m + 8);
float32x4_t _val3 = vld1q_f32(m + 12);
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif

#if __aarch64__
_sum0 = vfmaq_laneq_f32(_sum0, _val0, _w, 0);
_sum1 = vfmaq_laneq_f32(_sum1, _val1, _w, 1);
_sum2 = vfmaq_laneq_f32(_sum2, _val2, _w, 2);
_sum3 = vfmaq_laneq_f32(_sum3, _val3, _w, 3);
#else
_sum0 = vmlaq_lane_f32(_sum0, _val0, vget_low_f32(_w), 0);
_sum1 = vmlaq_lane_f32(_sum1, _val1, vget_low_f32(_w), 1);
_sum2 = vmlaq_lane_f32(_sum2, _val2, vget_high_f32(_w), 0);
_sum3 = vmlaq_lane_f32(_sum3, _val3, vget_high_f32(_w), 1);
#endif

m += 16;
kptr += 4;
}
for (; i < num_input; i++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float32x4_t _k = vdupq_n_f32((float)(kptr[0]));
#else
float32x4_t _val = vld1q_f32(m);
float32x4_t _k = vdupq_n_f32(float16_to_float32(kptr[0]));
#endif
_sum0 = vfmaq_f32(_sum0, _val, _k);

m += 4;
kptr += 1;
}

_sum0 = vaddq_f32(_sum0, _sum1);
_sum2 = vaddq_f32(_sum2, _sum3);
_sum0 = vaddq_f32(_sum0, _sum2);

_sum0 = activation_ps(_sum0, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
vst1_f16(outptr, vcvt_f16_f32(_sum0));
#else
vst1q_f32(outptr, _sum0);
#endif
outptr += 4;
}
}

if (elempack == 1 && num_output_elempack == 1)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
__fp16* outptr = top_blob.row<__fp16>(j);
#else
float* outptr = top_blob.row(j);
#endif

for (int p = 0; p < num_output; p++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
const __fp16* m = bottom_blob.row<const __fp16>(j);
const __fp16* kptr = weight_data_fp16.row<const __fp16>(p);
#else
const float* m = bottom_blob.row(j);
const unsigned short* kptr = weight_data_fp16.row<const unsigned short>(p);
#endif

float sum = 0.f;

if (bias_data_ptr)
{
sum = bias_data_ptr[p];
}

int i = 0;
float32x4_t _sum = vdupq_n_f32(0.f);
for (; i + 3 < num_input; i += 4)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
float32x4_t _val = vcvt_f32_f16(vld1_f16(m));
float32x4_t _w = vcvt_f32_f16(vld1_f16(kptr));
#else
float32x4_t _val = vld1q_f32(m);
float32x4_t _w = vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(kptr)));
#endif

_sum = vfmaq_f32(_sum, _val, _w);

m += 4;
kptr += 4;
}
#if __aarch64__
sum += vaddvq_f32(_sum);
#else
float32x2_t _ss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum));
_ss = vpadd_f32(_ss, _ss);
sum += vget_lane_f32(_ss, 0);
#endif
for (; i < num_input; i++)
{
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
sum += (float)(*m++) * (float)(*kptr++);
#else
sum += *m++ * float16_to_float32(*kptr++);
#endif
}

sum = activation_ss(sum, activation_type, activation_params);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
outptr[0] = (__fp16)sum;
#else
outptr[0] = sum;
#endif
outptr += 1;
}
}
}
#else // (__ARM_FP & 2)
(void)bottom_blob;
(void)top_blob;
(void)weight_data_fp16;
(void)bias_data;
(void)activation_type;
(void)activation_params;
(void)opt;
#endif // (__ARM_FP & 2)
}

+ 2
- 2
tests/testutil.h View File

@@ -1240,7 +1240,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec
opts[4].use_fp16_packed = true;
opts[4].use_fp16_storage = true;
opts[4].use_fp16_arithmetic = true;
opts[4].use_bf16_storage = true;
opts[4].use_bf16_storage = false;
opts[4].use_shader_pack8 = true;
opts[4].use_image_storage = true;

@@ -1388,7 +1388,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec
opts[4].use_fp16_packed = true;
opts[4].use_fp16_storage = true;
opts[4].use_fp16_arithmetic = true;
opts[4].use_bf16_storage = true;
opts[4].use_bf16_storage = false;
opts[4].use_shader_pack8 = true;
opts[4].use_image_storage = true;



Write Preview
Loading…
Cancel
Save