diff --git a/src/layer/riscv/convolution_1x1_packn.h b/src/layer/riscv/convolution_1x1_packn.h new file mode 100644 index 000000000..587d69b75 --- /dev/null +++ b/src/layer/riscv/convolution_1x1_packn.h @@ -0,0 +1,68 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv1x1s1_sgemm_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + const int size = w * h; + + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; + + im2col_sgemm_packn_rvv(bottom_im2col, top_blob, kernel, _bias, opt); +} + +static void conv1x1s2_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + const int packn = csrr_vlenb() / 4; + const word_type vl = vsetvl_e32m1(packn); + + int w = bottom_blob.w; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + int outw = top_blob.w; + int outh = top_blob.h; + + const int tailstep = (w - 2 * outw + w) * packn; + + Mat bottom_blob_shrinked; + bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < channels; p++) + { + const float* r0 = bottom_blob.channel(p); + float* outptr = bottom_blob_shrinked.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + vfloat32m1_t _val = vle32_v_f32m1(r0, vl); + vse32_v_f32m1(outptr, _val, vl); + + r0 += packn * 2; + outptr += packn; + } + + r0 += tailstep; + } + } + + conv1x1s1_sgemm_packn_rvv(bottom_blob_shrinked, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/riscv/convolution_1x1_packn_fp16s.h b/src/layer/riscv/convolution_1x1_packn_fp16s.h new file mode 100644 index 000000000..d4de7115f --- /dev/null +++ b/src/layer/riscv/convolution_1x1_packn_fp16s.h @@ -0,0 +1,68 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void conv1x1s1_sgemm_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + const int size = w * h; + + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; + + im2col_sgemm_packn_fp16sa_rvv(bottom_im2col, top_blob, kernel, _bias, opt); +} + +static void conv1x1s2_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + const int packn = csrr_vlenb() / 2; + const word_type vl = vsetvl_e16m1(packn); + + int w = bottom_blob.w; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + int outw = top_blob.w; + int outh = top_blob.h; + + const int tailstep = (w - 2 * outw + w) * packn; + + Mat bottom_blob_shrinked; + bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < channels; p++) + { + const __fp16* r0 = bottom_blob.channel(p); + __fp16* outptr = bottom_blob_shrinked.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + vfloat16m1_t _val = vle16_v_f16m1(r0, vl); + vse16_v_f16m1(outptr, _val, vl); + + r0 += packn * 2; + outptr += packn; + } + + r0 += tailstep; + } + } + + conv1x1s1_sgemm_packn_fp16sa_rvv(bottom_blob_shrinked, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/riscv/convolution_riscv.cpp b/src/layer/riscv/convolution_riscv.cpp index 617bb6a99..0353067b4 100644 --- a/src/layer/riscv/convolution_riscv.cpp +++ b/src/layer/riscv/convolution_riscv.cpp @@ -38,11 +38,17 @@ namespace ncnn { #include "convolution_pack1ton.h" #include "convolution_packnto1.h" +#include "convolution_sgemm_packn.h" +#include "convolution_1x1_packn.h" + #if __riscv_zfh #include "convolution_fp16s.h" #include "convolution_packn_fp16s.h" #include "convolution_pack1ton_fp16s.h" #include "convolution_packnto1_fp16s.h" + +#include "convolution_sgemm_packn_fp16s.h" +#include "convolution_1x1_packn_fp16s.h" #endif #endif // __riscv_vector @@ -54,10 +60,56 @@ Convolution_riscv::Convolution_riscv() support_fp16_storage = true; #endif #endif // __riscv_vector + + activation = 0; } int Convolution_riscv::create_pipeline(const Option& opt) { + if (activation_type == 1) + { + activation = ncnn::create_layer(ncnn::LayerType::ReLU); + + ncnn::ParamDict pd; + activation->load_param(pd); + } + else if (activation_type == 2) + { + activation = ncnn::create_layer(ncnn::LayerType::ReLU); + + ncnn::ParamDict pd; + pd.set(0, activation_params[0]); // slope + activation->load_param(pd); + } + else if (activation_type == 3) + { + activation = ncnn::create_layer(ncnn::LayerType::Clip); + + ncnn::ParamDict pd; + pd.set(0, activation_params[0]); // min + pd.set(1, activation_params[1]); // max + activation->load_param(pd); + } + else if (activation_type == 4) + { + activation = ncnn::create_layer(ncnn::LayerType::Sigmoid); + + ncnn::ParamDict pd; + activation->load_param(pd); + } + else if (activation_type == 5) + { + activation = ncnn::create_layer(ncnn::LayerType::Mish); + + ncnn::ParamDict pd; + activation->load_param(pd); + } + + if (activation) + { + activation->create_pipeline(opt); + } + #if __riscv_vector && __riscv_zfh if (opt.use_fp16_storage) { @@ -142,6 +194,13 @@ int Convolution_riscv::create_pipeline(const Option& opt) int Convolution_riscv::destroy_pipeline(const Option& opt) { + if (activation) + { + activation->destroy_pipeline(opt); + delete activation; + activation = 0; + } + return 0; } @@ -231,7 +290,6 @@ int Convolution_riscv::forward(const Mat& bottom_blob, Mat& top_blob, const Opti w = bottom_blob_bordered.w; h = bottom_blob_bordered.h; - int size = w * h; int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; @@ -248,11 +306,37 @@ int Convolution_riscv::forward(const Mat& bottom_blob, Mat& top_blob, const Opti if (top_blob.empty()) return -100; - const int num_input = channels * elempack; - #if __riscv_vector if (elempack == packn && out_elempack == packn) { + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + conv1x1s1_sgemm_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + conv1x1s2_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (opt.use_sgemm_convolution) + { + convolution_im2col_sgemm_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else { convolution_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } @@ -427,7 +511,6 @@ int Convolution_riscv::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons int w = bottom_blob.w; int h = bottom_blob.h; - int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; int elempack = bottom_blob.elempack; @@ -490,7 +573,6 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con int w = bottom_blob.w; int h = bottom_blob.h; - int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; int elempack = bottom_blob.elempack; @@ -506,7 +588,6 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con w = bottom_blob_bordered.w; h = bottom_blob_bordered.h; - int size = w * h; int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; @@ -517,10 +598,36 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con if (top_blob.empty()) return -100; - const int num_input = channels * elempack; - if (elempack == packn && out_elempack == packn) { + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + conv1x1s1_sgemm_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + conv1x1s2_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (opt.use_sgemm_convolution) + { + convolution_im2col_sgemm_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else { convolution_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } diff --git a/src/layer/riscv/convolution_riscv.h b/src/layer/riscv/convolution_riscv.h index 4608e042e..c231c7e71 100644 --- a/src/layer/riscv/convolution_riscv.h +++ b/src/layer/riscv/convolution_riscv.h @@ -37,6 +37,8 @@ protected: #endif public: + Layer* activation; + // packn Mat weight_data_packed; diff --git a/src/layer/riscv/convolution_sgemm_packn.h b/src/layer/riscv/convolution_sgemm_packn.h new file mode 100644 index 000000000..a970504c9 --- /dev/null +++ b/src/layer/riscv/convolution_sgemm_packn.h @@ -0,0 +1,373 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void im2col_sgemm_packn_rvv(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + const int packn = csrr_vlenb() / 4; + const word_type vl = vsetvl_e32m1(packn); + + // Mat bottom_im2col(size, maxk, inch, 4u * packn, packn, opt.workspace_allocator); + + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const float* bias = _bias; + + // permute + Mat tmp; + if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 4u * packn, packn, opt.workspace_allocator); + { + int remain_size_start = 0; + int nn_size = size >> 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + + float* tmpptr = tmp.channel(i / 8); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl); + vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl); + vfloat32m1_t _val2 = vle32_v_f32m1(img0 + packn * 2, vl); + vfloat32m1_t _val3 = vle32_v_f32m1(img0 + packn * 3, vl); + vfloat32m1_t _val4 = vle32_v_f32m1(img0 + packn * 4, vl); + vfloat32m1_t _val5 = vle32_v_f32m1(img0 + packn * 5, vl); + vfloat32m1_t _val6 = vle32_v_f32m1(img0 + packn * 6, vl); + vfloat32m1_t _val7 = vle32_v_f32m1(img0 + packn * 7, vl); + vsseg8e32_v_f32m1x8(tmpptr, vcreate_f32m1x8(_val0, _val1, _val2, _val3, _val4, _val5, _val6, _val7), vl); + + img0 += size * packn; + tmpptr += packn * 8; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl); + vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl); + vfloat32m1_t _val2 = vle32_v_f32m1(img0 + packn * 2, vl); + vfloat32m1_t _val3 = vle32_v_f32m1(img0 + packn * 3, vl); + vsseg4e32_v_f32m1x4(tmpptr, vcreate_f32m1x4(_val0, _val1, _val2, _val3), vl); + + img0 += size * packn; + tmpptr += packn * 4; + } + } + } + + remain_size_start += nn_size << 2; + nn_size = (size - remain_size_start) >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 2; + + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl); + vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl); + vsseg2e32_v_f32m1x2(tmpptr, vcreate_f32m1x2(_val0, _val1), vl); + + img0 += size * packn; + tmpptr += packn * 2; + } + } + } + + remain_size_start += nn_size << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { + float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + + for (int q = 0; q < inch; q++) + { + const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat32m1_t _val = vle32_v_f32m1(img0, vl); + vse32_v_f32m1(tmpptr, _val, vl); + + img0 += size * packn; + tmpptr += packn; + } + } + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + float* outptr0 = top_blob.channel(p); + + int i = 0; + for (; i + 7 < size; i += 8) + { + const float* tmpptr = tmp.channel(i / 8); + const float* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum2 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum3 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum4 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum5 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum6 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum7 = vfmv_v_f_f32m1(0.f, vl); + + if (bias) + { + _sum0 = vle32_v_f32m1(bias + p * packn, vl); + _sum1 = vle32_v_f32m1(bias + p * packn, vl); + _sum2 = vle32_v_f32m1(bias + p * packn, vl); + _sum3 = vle32_v_f32m1(bias + p * packn, vl); + _sum4 = vle32_v_f32m1(bias + p * packn, vl); + _sum5 = vle32_v_f32m1(bias + p * packn, vl); + _sum6 = vle32_v_f32m1(bias + p * packn, vl); + _sum7 = vle32_v_f32m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + float val0 = *tmpptr++; + float val1 = *tmpptr++; + float val2 = *tmpptr++; + float val3 = *tmpptr++; + float val4 = *tmpptr++; + float val5 = *tmpptr++; + float val6 = *tmpptr++; + float val7 = *tmpptr++; + vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl); + _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl); + _sum2 = vfmacc_vf_f32m1(_sum2, val2, _w0, vl); + _sum3 = vfmacc_vf_f32m1(_sum3, val3, _w0, vl); + _sum4 = vfmacc_vf_f32m1(_sum4, val4, _w0, vl); + _sum5 = vfmacc_vf_f32m1(_sum5, val5, _w0, vl); + _sum6 = vfmacc_vf_f32m1(_sum6, val6, _w0, vl); + _sum7 = vfmacc_vf_f32m1(_sum7, val7, _w0, vl); + + kptr0 += packn; + } + + vse32_v_f32m1(outptr0, _sum0, vl); + vse32_v_f32m1(outptr0 + packn, _sum1, vl); + vse32_v_f32m1(outptr0 + packn * 2, _sum2, vl); + vse32_v_f32m1(outptr0 + packn * 3, _sum3, vl); + vse32_v_f32m1(outptr0 + packn * 4, _sum4, vl); + vse32_v_f32m1(outptr0 + packn * 5, _sum5, vl); + vse32_v_f32m1(outptr0 + packn * 6, _sum6, vl); + vse32_v_f32m1(outptr0 + packn * 7, _sum7, vl); + + outptr0 += packn * 8; + } + for (; i + 3 < size; i += 4) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + const float* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum2 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum3 = vfmv_v_f_f32m1(0.f, vl); + + if (bias) + { + _sum0 = vle32_v_f32m1(bias + p * packn, vl); + _sum1 = vle32_v_f32m1(bias + p * packn, vl); + _sum2 = vle32_v_f32m1(bias + p * packn, vl); + _sum3 = vle32_v_f32m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + float val0 = *tmpptr++; + float val1 = *tmpptr++; + float val2 = *tmpptr++; + float val3 = *tmpptr++; + vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl); + _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl); + _sum2 = vfmacc_vf_f32m1(_sum2, val2, _w0, vl); + _sum3 = vfmacc_vf_f32m1(_sum3, val3, _w0, vl); + + kptr0 += packn; + } + + vse32_v_f32m1(outptr0, _sum0, vl); + vse32_v_f32m1(outptr0 + packn, _sum1, vl); + vse32_v_f32m1(outptr0 + packn * 2, _sum2, vl); + vse32_v_f32m1(outptr0 + packn * 3, _sum3, vl); + + outptr0 += packn * 4; + } + for (; i + 1 < size; i += 2) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + const float* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl); + vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl); + + if (bias) + { + _sum0 = vle32_v_f32m1(bias + p * packn, vl); + _sum1 = vle32_v_f32m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + float val0 = *tmpptr++; + float val1 = *tmpptr++; + vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl); + _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl); + + kptr0 += packn; + } + + vse32_v_f32m1(outptr0, _sum0, vl); + vse32_v_f32m1(outptr0 + packn, _sum1, vl); + + outptr0 += packn * 2; + } + for (; i < size; i++) + { + const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + const float* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat32m1_t _sum = vfmv_v_f_f32m1(0.f, vl); + + if (bias) + { + _sum = vle32_v_f32m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + float val = *tmpptr++; + vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl); + _sum = vfmacc_vf_f32m1(_sum, val, _w0, vl); + + kptr0 += packn; + } + + vse32_v_f32m1(outptr0, _sum, vl); + + outptr0 += packn; + } + } +} + +static void convolution_im2col_sgemm_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + const int packn = csrr_vlenb() / 4; + const word_type vl = vsetvl_e32m1(packn); + + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 4u * packn, packn, opt.workspace_allocator); + { + const int gap = (w * stride_h - outw * stride_w) * packn; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + float* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) + { + const float* sptr = img.row(dilation_h * u) + dilation_w * v * packn; + + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j < outw; j++) + { + vfloat32m1_t _val = vle32_v_f32m1(sptr, vl); + vse32_v_f32m1(ptr, _val, vl); + + sptr += stride_w * packn; + ptr += packn; + } + + sptr += gap; + } + } + } + } + } + + im2col_sgemm_packn_rvv(bottom_im2col, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/riscv/convolution_sgemm_packn_fp16s.h b/src/layer/riscv/convolution_sgemm_packn_fp16s.h new file mode 100644 index 000000000..6dbcf64ca --- /dev/null +++ b/src/layer/riscv/convolution_sgemm_packn_fp16s.h @@ -0,0 +1,373 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +static void im2col_sgemm_packn_fp16sa_rvv(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + const int packn = csrr_vlenb() / 2; + const word_type vl = vsetvl_e16m1(packn); + + // Mat bottom_im2col(size, maxk, inch, 2u * packn, packn, opt.workspace_allocator); + + const int size = bottom_im2col.w; + const int maxk = bottom_im2col.h; + const int inch = bottom_im2col.c; + + const int outch = top_blob.c; + + const __fp16* bias = _bias; + + // permute + Mat tmp; + if (size >= 8) + tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator); + else if (size >= 4) + tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator); + else if (size >= 2) + tmp.create(2 * maxk, inch, size / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator); + else + tmp.create(maxk, inch, size, 2u * packn, packn, opt.workspace_allocator); + { + int remain_size_start = 0; + int nn_size = size >> 3; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 8; + + __fp16* tmpptr = tmp.channel(i / 8); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl); + vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl); + vfloat16m1_t _val2 = vle16_v_f16m1(img0 + packn * 2, vl); + vfloat16m1_t _val3 = vle16_v_f16m1(img0 + packn * 3, vl); + vfloat16m1_t _val4 = vle16_v_f16m1(img0 + packn * 4, vl); + vfloat16m1_t _val5 = vle16_v_f16m1(img0 + packn * 5, vl); + vfloat16m1_t _val6 = vle16_v_f16m1(img0 + packn * 6, vl); + vfloat16m1_t _val7 = vle16_v_f16m1(img0 + packn * 7, vl); + vsseg8e16_v_f16m1x8(tmpptr, vcreate_f16m1x8(_val0, _val1, _val2, _val3, _val4, _val5, _val6, _val7), vl); + + img0 += size * packn; + tmpptr += packn * 8; + } + } + } + + remain_size_start += nn_size << 3; + nn_size = (size - remain_size_start) >> 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 4; + + __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl); + vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl); + vfloat16m1_t _val2 = vle16_v_f16m1(img0 + packn * 2, vl); + vfloat16m1_t _val3 = vle16_v_f16m1(img0 + packn * 3, vl); + vsseg4e16_v_f16m1x4(tmpptr, vcreate_f16m1x4(_val0, _val1, _val2, _val3), vl); + + img0 += size * packn; + tmpptr += packn * 4; + } + } + } + + remain_size_start += nn_size << 2; + nn_size = (size - remain_size_start) >> 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii = 0; ii < nn_size; ii++) + { + int i = remain_size_start + ii * 2; + + __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl); + vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl); + vsseg2e16_v_f16m1x2(tmpptr, vcreate_f16m1x2(_val0, _val1), vl); + + img0 += size * packn; + tmpptr += packn * 2; + } + } + } + + remain_size_start += nn_size << 1; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = remain_size_start; i < size; i++) + { + __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + + for (int q = 0; q < inch; q++) + { + const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn; + + for (int k = 0; k < maxk; k++) + { + vfloat16m1_t _val = vle16_v_f16m1(img0, vl); + vse16_v_f16m1(tmpptr, _val, vl); + + img0 += size * packn; + tmpptr += packn; + } + } + } + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < outch; p++) + { + __fp16* outptr0 = top_blob.channel(p); + + int i = 0; + for (; i + 7 < size; i += 8) + { + const __fp16* tmpptr = tmp.channel(i / 8); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum2 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum3 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum4 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum5 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum6 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum7 = vfmv_v_f_f16m1(0.f, vl); + + if (bias) + { + _sum0 = vle16_v_f16m1(bias + p * packn, vl); + _sum1 = vle16_v_f16m1(bias + p * packn, vl); + _sum2 = vle16_v_f16m1(bias + p * packn, vl); + _sum3 = vle16_v_f16m1(bias + p * packn, vl); + _sum4 = vle16_v_f16m1(bias + p * packn, vl); + _sum5 = vle16_v_f16m1(bias + p * packn, vl); + _sum6 = vle16_v_f16m1(bias + p * packn, vl); + _sum7 = vle16_v_f16m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + __fp16 val0 = *tmpptr++; + __fp16 val1 = *tmpptr++; + __fp16 val2 = *tmpptr++; + __fp16 val3 = *tmpptr++; + __fp16 val4 = *tmpptr++; + __fp16 val5 = *tmpptr++; + __fp16 val6 = *tmpptr++; + __fp16 val7 = *tmpptr++; + vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl); + _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl); + _sum2 = vfmacc_vf_f16m1(_sum2, val2, _w0, vl); + _sum3 = vfmacc_vf_f16m1(_sum3, val3, _w0, vl); + _sum4 = vfmacc_vf_f16m1(_sum4, val4, _w0, vl); + _sum5 = vfmacc_vf_f16m1(_sum5, val5, _w0, vl); + _sum6 = vfmacc_vf_f16m1(_sum6, val6, _w0, vl); + _sum7 = vfmacc_vf_f16m1(_sum7, val7, _w0, vl); + + kptr0 += packn; + } + + vse16_v_f16m1(outptr0, _sum0, vl); + vse16_v_f16m1(outptr0 + packn, _sum1, vl); + vse16_v_f16m1(outptr0 + packn * 2, _sum2, vl); + vse16_v_f16m1(outptr0 + packn * 3, _sum3, vl); + vse16_v_f16m1(outptr0 + packn * 4, _sum4, vl); + vse16_v_f16m1(outptr0 + packn * 5, _sum5, vl); + vse16_v_f16m1(outptr0 + packn * 6, _sum6, vl); + vse16_v_f16m1(outptr0 + packn * 7, _sum7, vl); + + outptr0 += packn * 8; + } + for (; i + 3 < size; i += 4) + { + const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum2 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum3 = vfmv_v_f_f16m1(0.f, vl); + + if (bias) + { + _sum0 = vle16_v_f16m1(bias + p * packn, vl); + _sum1 = vle16_v_f16m1(bias + p * packn, vl); + _sum2 = vle16_v_f16m1(bias + p * packn, vl); + _sum3 = vle16_v_f16m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + __fp16 val0 = *tmpptr++; + __fp16 val1 = *tmpptr++; + __fp16 val2 = *tmpptr++; + __fp16 val3 = *tmpptr++; + vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl); + _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl); + _sum2 = vfmacc_vf_f16m1(_sum2, val2, _w0, vl); + _sum3 = vfmacc_vf_f16m1(_sum3, val3, _w0, vl); + + kptr0 += packn; + } + + vse16_v_f16m1(outptr0, _sum0, vl); + vse16_v_f16m1(outptr0 + packn, _sum1, vl); + vse16_v_f16m1(outptr0 + packn * 2, _sum2, vl); + vse16_v_f16m1(outptr0 + packn * 3, _sum3, vl); + + outptr0 += packn * 4; + } + for (; i + 1 < size; i += 2) + { + const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl); + vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl); + + if (bias) + { + _sum0 = vle16_v_f16m1(bias + p * packn, vl); + _sum1 = vle16_v_f16m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + __fp16 val0 = *tmpptr++; + __fp16 val1 = *tmpptr++; + vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl); + _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl); + _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl); + + kptr0 += packn; + } + + vse16_v_f16m1(outptr0, _sum0, vl); + vse16_v_f16m1(outptr0 + packn, _sum1, vl); + + outptr0 += packn * 2; + } + for (; i < size; i++) + { + const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2); + const __fp16* kptr0 = kernel.channel(p); + + int nn = inch * maxk * packn; // inch always > 0 + + vfloat16m1_t _sum = vfmv_v_f_f16m1(0.f, vl); + + if (bias) + { + _sum = vle16_v_f16m1(bias + p * packn, vl); + } + + for (int j = 0; j < nn; j++) + { + __fp16 val = *tmpptr++; + vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl); + _sum = vfmacc_vf_f16m1(_sum, val, _w0, vl); + + kptr0 += packn; + } + + vse16_v_f16m1(outptr0, _sum, vl); + + outptr0 += packn; + } + } +} + +static void convolution_im2col_sgemm_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt) +{ + const int packn = csrr_vlenb() / 2; + const word_type vl = vsetvl_e16m1(packn); + + int w = bottom_blob.w; + int inch = bottom_blob.c; + + int outw = top_blob.w; + int outh = top_blob.h; + const int size = outw * outh; + + const int maxk = kernel_w * kernel_h; + + // im2col + Mat bottom_im2col(size, maxk, inch, 2u * packn, packn, opt.workspace_allocator); + { + const int gap = (w * stride_h - outw * stride_w) * packn; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < inch; p++) + { + const Mat img = bottom_blob.channel(p); + __fp16* ptr = bottom_im2col.channel(p); + + for (int u = 0; u < kernel_h; u++) + { + for (int v = 0; v < kernel_w; v++) + { + const __fp16* sptr = img.row(dilation_h * u) + dilation_w * v * packn; + + for (int i = 0; i < outh; i++) + { + int j = 0; + for (; j < outw; j++) + { + vfloat16m1_t _val = vle16_v_f16m1(sptr, vl); + vse16_v_f16m1(ptr, _val, vl); + + sptr += stride_w * packn; + ptr += packn; + } + + sptr += gap; + } + } + } + } + } + + im2col_sgemm_packn_fp16sa_rvv(bottom_im2col, top_blob, kernel, _bias, opt); +}