riscv v optimization for convolution sgemm and conv1x1 packn

src/layer/riscv/convolution_1x1_packn.h

@@ -0,0 +1,68 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void conv1x1s1_sgemm_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    const int size = w * h;
+
+    Mat bottom_im2col = bottom_blob;
+    bottom_im2col.w = size;
+    bottom_im2col.h = 1;
+
+    im2col_sgemm_packn_rvv(bottom_im2col, top_blob, kernel, _bias, opt);
+}
+
+static void conv1x1s2_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 4;
+    const word_type vl = vsetvl_e32m1(packn);
+
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+    int elempack = bottom_blob.elempack;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+
+    const int tailstep = (w - 2 * outw + w) * packn;
+
+    Mat bottom_blob_shrinked;
+    bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator);
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < channels; p++)
+    {
+        const float* r0 = bottom_blob.channel(p);
+        float* outptr = bottom_blob_shrinked.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                vfloat32m1_t _val = vle32_v_f32m1(r0, vl);
+                vse32_v_f32m1(outptr, _val, vl);
+
+                r0 += packn * 2;
+                outptr += packn;
+            }
+
+            r0 += tailstep;
+        }
+    }
+
+    conv1x1s1_sgemm_packn_rvv(bottom_blob_shrinked, top_blob, kernel, _bias, opt);
+}

src/layer/riscv/convolution_1x1_packn_fp16s.h

@@ -0,0 +1,68 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void conv1x1s1_sgemm_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    const int size = w * h;
+
+    Mat bottom_im2col = bottom_blob;
+    bottom_im2col.w = size;
+    bottom_im2col.h = 1;
+
+    im2col_sgemm_packn_fp16sa_rvv(bottom_im2col, top_blob, kernel, _bias, opt);
+}
+
+static void conv1x1s2_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 2;
+    const word_type vl = vsetvl_e16m1(packn);
+
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+    int elempack = bottom_blob.elempack;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+
+    const int tailstep = (w - 2 * outw + w) * packn;
+
+    Mat bottom_blob_shrinked;
+    bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator);
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < channels; p++)
+    {
+        const __fp16* r0 = bottom_blob.channel(p);
+        __fp16* outptr = bottom_blob_shrinked.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                vfloat16m1_t _val = vle16_v_f16m1(r0, vl);
+                vse16_v_f16m1(outptr, _val, vl);
+
+                r0 += packn * 2;
+                outptr += packn;
+            }
+
+            r0 += tailstep;
+        }
+    }
+
+    conv1x1s1_sgemm_packn_fp16sa_rvv(bottom_blob_shrinked, top_blob, kernel, _bias, opt);
+}

src/layer/riscv/convolution_riscv.cpp

@@ -38,11 +38,17 @@ namespace ncnn {
 #include "convolution_pack1ton.h"
 #include "convolution_packnto1.h"
 
+#include "convolution_sgemm_packn.h"
+#include "convolution_1x1_packn.h"
+
 #if __riscv_zfh
 #include "convolution_fp16s.h"
 #include "convolution_packn_fp16s.h"
 #include "convolution_pack1ton_fp16s.h"
 #include "convolution_packnto1_fp16s.h"
+
+#include "convolution_sgemm_packn_fp16s.h"
+#include "convolution_1x1_packn_fp16s.h"
 #endif
 #endif // __riscv_vector
 
@@ -54,10 +60,56 @@ Convolution_riscv::Convolution_riscv()
     support_fp16_storage = true;
 #endif
 #endif // __riscv_vector
+
+    activation = 0;
 }
 
 int Convolution_riscv::create_pipeline(const Option& opt)
 {
+    if (activation_type == 1)
+    {
+        activation = ncnn::create_layer(ncnn::LayerType::ReLU);
+
+        ncnn::ParamDict pd;
+        activation->load_param(pd);
+    }
+    else if (activation_type == 2)
+    {
+        activation = ncnn::create_layer(ncnn::LayerType::ReLU);
+
+        ncnn::ParamDict pd;
+        pd.set(0, activation_params[0]); // slope
+        activation->load_param(pd);
+    }
+    else if (activation_type == 3)
+    {
+        activation = ncnn::create_layer(ncnn::LayerType::Clip);
+
+        ncnn::ParamDict pd;
+        pd.set(0, activation_params[0]); // min
+        pd.set(1, activation_params[1]); // max
+        activation->load_param(pd);
+    }
+    else if (activation_type == 4)
+    {
+        activation = ncnn::create_layer(ncnn::LayerType::Sigmoid);
+
+        ncnn::ParamDict pd;
+        activation->load_param(pd);
+    }
+    else if (activation_type == 5)
+    {
+        activation = ncnn::create_layer(ncnn::LayerType::Mish);
+
+        ncnn::ParamDict pd;
+        activation->load_param(pd);
+    }
+
+    if (activation)
+    {
+        activation->create_pipeline(opt);
+    }
+
 #if __riscv_vector && __riscv_zfh
     if (opt.use_fp16_storage)
     {
@@ -142,6 +194,13 @@ int Convolution_riscv::create_pipeline(const Option& opt)
 
 int Convolution_riscv::destroy_pipeline(const Option& opt)
 {
+    if (activation)
+    {
+        activation->destroy_pipeline(opt);
+        delete activation;
+        activation = 0;
+    }
+
     return 0;
 }
 
@@ -231,7 +290,6 @@ int Convolution_riscv::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
 
     w = bottom_blob_bordered.w;
     h = bottom_blob_bordered.h;
-    int size = w * h;
 
     int outw = (w - kernel_extent_w) / stride_w + 1;
     int outh = (h - kernel_extent_h) / stride_h + 1;
@@ -248,11 +306,37 @@ int Convolution_riscv::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     if (top_blob.empty())
         return -100;
 
-    const int num_input = channels * elempack;
-
 #if __riscv_vector
     if (elempack == packn && out_elempack == packn)
     {
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        {
+            conv1x1s1_sgemm_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            conv1x1s2_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (opt.use_sgemm_convolution)
+        {
+            convolution_im2col_sgemm_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else
         {
             convolution_packn_rvv(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
@@ -427,7 +511,6 @@ int Convolution_riscv::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
 
     int w = bottom_blob.w;
     int h = bottom_blob.h;
-    int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
     int elempack = bottom_blob.elempack;
 
@@ -490,7 +573,6 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
 
     int w = bottom_blob.w;
     int h = bottom_blob.h;
-    int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
     int elempack = bottom_blob.elempack;
 
@@ -506,7 +588,6 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
 
     w = bottom_blob_bordered.w;
     h = bottom_blob_bordered.h;
-    int size = w * h;
 
     int outw = (w - kernel_extent_w) / stride_w + 1;
     int outh = (h - kernel_extent_h) / stride_h + 1;
@@ -517,10 +598,36 @@ int Convolution_riscv::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
     if (top_blob.empty())
         return -100;
 
-    const int num_input = channels * elempack;
-
     if (elempack == packn && out_elempack == packn)
     {
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        {
+            conv1x1s1_sgemm_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            conv1x1s2_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (opt.use_sgemm_convolution)
+        {
+            convolution_im2col_sgemm_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else
         {
             convolution_packn_fp16sa_rvv(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }

src/layer/riscv/convolution_riscv.h

@@ -37,6 +37,8 @@ protected:
 #endif
 
 public:
+    Layer* activation;
+
     // packn
     Mat weight_data_packed;
 

src/layer/riscv/convolution_sgemm_packn.h

@@ -0,0 +1,373 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void im2col_sgemm_packn_rvv(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 4;
+    const word_type vl = vsetvl_e32m1(packn);
+
+    // Mat bottom_im2col(size, maxk, inch, 4u * packn, packn, opt.workspace_allocator);
+
+    const int size = bottom_im2col.w;
+    const int maxk = bottom_im2col.h;
+    const int inch = bottom_im2col.c;
+
+    const int outch = top_blob.c;
+
+    const float* bias = _bias;
+
+    // permute
+    Mat tmp;
+    if (size >= 8)
+        tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator);
+    else if (size >= 4)
+        tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator);
+    else if (size >= 2)
+        tmp.create(2 * maxk, inch, size / 2 + size % 2, 4u * packn, packn, opt.workspace_allocator);
+    else
+        tmp.create(maxk, inch, size, 4u * packn, packn, opt.workspace_allocator);
+    {
+        int remain_size_start = 0;
+        int nn_size = size >> 3;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 8;
+
+            float* tmpptr = tmp.channel(i / 8);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl);
+                    vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl);
+                    vfloat32m1_t _val2 = vle32_v_f32m1(img0 + packn * 2, vl);
+                    vfloat32m1_t _val3 = vle32_v_f32m1(img0 + packn * 3, vl);
+                    vfloat32m1_t _val4 = vle32_v_f32m1(img0 + packn * 4, vl);
+                    vfloat32m1_t _val5 = vle32_v_f32m1(img0 + packn * 5, vl);
+                    vfloat32m1_t _val6 = vle32_v_f32m1(img0 + packn * 6, vl);
+                    vfloat32m1_t _val7 = vle32_v_f32m1(img0 + packn * 7, vl);
+                    vsseg8e32_v_f32m1x8(tmpptr, vcreate_f32m1x8(_val0, _val1, _val2, _val3, _val4, _val5, _val6, _val7), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 3;
+        nn_size = (size - remain_size_start) >> 2;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 4;
+
+            float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl);
+                    vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl);
+                    vfloat32m1_t _val2 = vle32_v_f32m1(img0 + packn * 2, vl);
+                    vfloat32m1_t _val3 = vle32_v_f32m1(img0 + packn * 3, vl);
+                    vsseg4e32_v_f32m1x4(tmpptr, vcreate_f32m1x4(_val0, _val1, _val2, _val3), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 4;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 2;
+        nn_size = (size - remain_size_start) >> 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 2;
+
+            float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat32m1_t _val0 = vle32_v_f32m1(img0, vl);
+                    vfloat32m1_t _val1 = vle32_v_f32m1(img0 + packn, vl);
+                    vsseg2e32_v_f32m1x2(tmpptr, vcreate_f32m1x2(_val0, _val1), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 2;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i = remain_size_start; i < size; i++)
+        {
+            float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const float* img0 = (const float*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat32m1_t _val = vle32_v_f32m1(img0, vl);
+                    vse32_v_f32m1(tmpptr, _val, vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn;
+                }
+            }
+        }
+    }
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        float* outptr0 = top_blob.channel(p);
+
+        int i = 0;
+        for (; i + 7 < size; i += 8)
+        {
+            const float* tmpptr = tmp.channel(i / 8);
+            const float* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum2 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum3 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum4 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum5 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum6 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum7 = vfmv_v_f_f32m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum1 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum2 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum3 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum4 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum5 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum6 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum7 = vle32_v_f32m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                float val0 = *tmpptr++;
+                float val1 = *tmpptr++;
+                float val2 = *tmpptr++;
+                float val3 = *tmpptr++;
+                float val4 = *tmpptr++;
+                float val5 = *tmpptr++;
+                float val6 = *tmpptr++;
+                float val7 = *tmpptr++;
+                vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl);
+                _sum2 = vfmacc_vf_f32m1(_sum2, val2, _w0, vl);
+                _sum3 = vfmacc_vf_f32m1(_sum3, val3, _w0, vl);
+                _sum4 = vfmacc_vf_f32m1(_sum4, val4, _w0, vl);
+                _sum5 = vfmacc_vf_f32m1(_sum5, val5, _w0, vl);
+                _sum6 = vfmacc_vf_f32m1(_sum6, val6, _w0, vl);
+                _sum7 = vfmacc_vf_f32m1(_sum7, val7, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse32_v_f32m1(outptr0, _sum0, vl);
+            vse32_v_f32m1(outptr0 + packn, _sum1, vl);
+            vse32_v_f32m1(outptr0 + packn * 2, _sum2, vl);
+            vse32_v_f32m1(outptr0 + packn * 3, _sum3, vl);
+            vse32_v_f32m1(outptr0 + packn * 4, _sum4, vl);
+            vse32_v_f32m1(outptr0 + packn * 5, _sum5, vl);
+            vse32_v_f32m1(outptr0 + packn * 6, _sum6, vl);
+            vse32_v_f32m1(outptr0 + packn * 7, _sum7, vl);
+
+            outptr0 += packn * 8;
+        }
+        for (; i + 3 < size; i += 4)
+        {
+            const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4);
+            const float* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum2 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum3 = vfmv_v_f_f32m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum1 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum2 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum3 = vle32_v_f32m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                float val0 = *tmpptr++;
+                float val1 = *tmpptr++;
+                float val2 = *tmpptr++;
+                float val3 = *tmpptr++;
+                vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl);
+                _sum2 = vfmacc_vf_f32m1(_sum2, val2, _w0, vl);
+                _sum3 = vfmacc_vf_f32m1(_sum3, val3, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse32_v_f32m1(outptr0, _sum0, vl);
+            vse32_v_f32m1(outptr0 + packn, _sum1, vl);
+            vse32_v_f32m1(outptr0 + packn * 2, _sum2, vl);
+            vse32_v_f32m1(outptr0 + packn * 3, _sum3, vl);
+
+            outptr0 += packn * 4;
+        }
+        for (; i + 1 < size; i += 2)
+        {
+            const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2);
+            const float* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat32m1_t _sum0 = vfmv_v_f_f32m1(0.f, vl);
+            vfloat32m1_t _sum1 = vfmv_v_f_f32m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle32_v_f32m1(bias + p * packn, vl);
+                _sum1 = vle32_v_f32m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                float val0 = *tmpptr++;
+                float val1 = *tmpptr++;
+                vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f32m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f32m1(_sum1, val1, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse32_v_f32m1(outptr0, _sum0, vl);
+            vse32_v_f32m1(outptr0 + packn, _sum1, vl);
+
+            outptr0 += packn * 2;
+        }
+        for (; i < size; i++)
+        {
+            const float* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);
+            const float* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat32m1_t _sum = vfmv_v_f_f32m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum = vle32_v_f32m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                float val = *tmpptr++;
+                vfloat32m1_t _w0 = vle32_v_f32m1(kptr0, vl);
+                _sum = vfmacc_vf_f32m1(_sum, val, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse32_v_f32m1(outptr0, _sum, vl);
+
+            outptr0 += packn;
+        }
+    }
+}
+
+static void convolution_im2col_sgemm_packn_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 4;
+    const word_type vl = vsetvl_e32m1(packn);
+
+    int w = bottom_blob.w;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    const int size = outw * outh;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // im2col
+    Mat bottom_im2col(size, maxk, inch, 4u * packn, packn, opt.workspace_allocator);
+    {
+        const int gap = (w * stride_h - outw * stride_w) * packn;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < inch; p++)
+        {
+            const Mat img = bottom_blob.channel(p);
+            float* ptr = bottom_im2col.channel(p);
+
+            for (int u = 0; u < kernel_h; u++)
+            {
+                for (int v = 0; v < kernel_w; v++)
+                {
+                    const float* sptr = img.row<const float>(dilation_h * u) + dilation_w * v * packn;
+
+                    for (int i = 0; i < outh; i++)
+                    {
+                        int j = 0;
+                        for (; j < outw; j++)
+                        {
+                            vfloat32m1_t _val = vle32_v_f32m1(sptr, vl);
+                            vse32_v_f32m1(ptr, _val, vl);
+
+                            sptr += stride_w * packn;
+                            ptr += packn;
+                        }
+
+                        sptr += gap;
+                    }
+                }
+            }
+        }
+    }
+
+    im2col_sgemm_packn_rvv(bottom_im2col, top_blob, kernel, _bias, opt);
+}

src/layer/riscv/convolution_sgemm_packn_fp16s.h

@@ -0,0 +1,373 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+static void im2col_sgemm_packn_fp16sa_rvv(const Mat& bottom_im2col, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 2;
+    const word_type vl = vsetvl_e16m1(packn);
+
+    // Mat bottom_im2col(size, maxk, inch, 2u * packn, packn, opt.workspace_allocator);
+
+    const int size = bottom_im2col.w;
+    const int maxk = bottom_im2col.h;
+    const int inch = bottom_im2col.c;
+
+    const int outch = top_blob.c;
+
+    const __fp16* bias = _bias;
+
+    // permute
+    Mat tmp;
+    if (size >= 8)
+        tmp.create(8 * maxk, inch, size / 8 + (size % 8) / 4 + (size % 4) / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator);
+    else if (size >= 4)
+        tmp.create(4 * maxk, inch, size / 4 + (size % 4) / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator);
+    else if (size >= 2)
+        tmp.create(2 * maxk, inch, size / 2 + size % 2, 2u * packn, packn, opt.workspace_allocator);
+    else
+        tmp.create(maxk, inch, size, 2u * packn, packn, opt.workspace_allocator);
+    {
+        int remain_size_start = 0;
+        int nn_size = size >> 3;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 8;
+
+            __fp16* tmpptr = tmp.channel(i / 8);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl);
+                    vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl);
+                    vfloat16m1_t _val2 = vle16_v_f16m1(img0 + packn * 2, vl);
+                    vfloat16m1_t _val3 = vle16_v_f16m1(img0 + packn * 3, vl);
+                    vfloat16m1_t _val4 = vle16_v_f16m1(img0 + packn * 4, vl);
+                    vfloat16m1_t _val5 = vle16_v_f16m1(img0 + packn * 5, vl);
+                    vfloat16m1_t _val6 = vle16_v_f16m1(img0 + packn * 6, vl);
+                    vfloat16m1_t _val7 = vle16_v_f16m1(img0 + packn * 7, vl);
+                    vsseg8e16_v_f16m1x8(tmpptr, vcreate_f16m1x8(_val0, _val1, _val2, _val3, _val4, _val5, _val6, _val7), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 8;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 3;
+        nn_size = (size - remain_size_start) >> 2;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 4;
+
+            __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl);
+                    vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl);
+                    vfloat16m1_t _val2 = vle16_v_f16m1(img0 + packn * 2, vl);
+                    vfloat16m1_t _val3 = vle16_v_f16m1(img0 + packn * 3, vl);
+                    vsseg4e16_v_f16m1x4(tmpptr, vcreate_f16m1x4(_val0, _val1, _val2, _val3), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 4;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 2;
+        nn_size = (size - remain_size_start) >> 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii = 0; ii < nn_size; ii++)
+        {
+            int i = remain_size_start + ii * 2;
+
+            __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat16m1_t _val0 = vle16_v_f16m1(img0, vl);
+                    vfloat16m1_t _val1 = vle16_v_f16m1(img0 + packn, vl);
+                    vsseg2e16_v_f16m1x2(tmpptr, vcreate_f16m1x2(_val0, _val1), vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn * 2;
+                }
+            }
+        }
+
+        remain_size_start += nn_size << 1;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i = remain_size_start; i < size; i++)
+        {
+            __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);
+
+            for (int q = 0; q < inch; q++)
+            {
+                const __fp16* img0 = (const __fp16*)bottom_im2col.channel(q) + i * packn;
+
+                for (int k = 0; k < maxk; k++)
+                {
+                    vfloat16m1_t _val = vle16_v_f16m1(img0, vl);
+                    vse16_v_f16m1(tmpptr, _val, vl);
+
+                    img0 += size * packn;
+                    tmpptr += packn;
+                }
+            }
+        }
+    }
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < outch; p++)
+    {
+        __fp16* outptr0 = top_blob.channel(p);
+
+        int i = 0;
+        for (; i + 7 < size; i += 8)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 8);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum2 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum3 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum4 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum5 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum6 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum7 = vfmv_v_f_f16m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum1 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum2 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum3 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum4 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum5 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum6 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum7 = vle16_v_f16m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                __fp16 val0 = *tmpptr++;
+                __fp16 val1 = *tmpptr++;
+                __fp16 val2 = *tmpptr++;
+                __fp16 val3 = *tmpptr++;
+                __fp16 val4 = *tmpptr++;
+                __fp16 val5 = *tmpptr++;
+                __fp16 val6 = *tmpptr++;
+                __fp16 val7 = *tmpptr++;
+                vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl);
+                _sum2 = vfmacc_vf_f16m1(_sum2, val2, _w0, vl);
+                _sum3 = vfmacc_vf_f16m1(_sum3, val3, _w0, vl);
+                _sum4 = vfmacc_vf_f16m1(_sum4, val4, _w0, vl);
+                _sum5 = vfmacc_vf_f16m1(_sum5, val5, _w0, vl);
+                _sum6 = vfmacc_vf_f16m1(_sum6, val6, _w0, vl);
+                _sum7 = vfmacc_vf_f16m1(_sum7, val7, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse16_v_f16m1(outptr0, _sum0, vl);
+            vse16_v_f16m1(outptr0 + packn, _sum1, vl);
+            vse16_v_f16m1(outptr0 + packn * 2, _sum2, vl);
+            vse16_v_f16m1(outptr0 + packn * 3, _sum3, vl);
+            vse16_v_f16m1(outptr0 + packn * 4, _sum4, vl);
+            vse16_v_f16m1(outptr0 + packn * 5, _sum5, vl);
+            vse16_v_f16m1(outptr0 + packn * 6, _sum6, vl);
+            vse16_v_f16m1(outptr0 + packn * 7, _sum7, vl);
+
+            outptr0 += packn * 8;
+        }
+        for (; i + 3 < size; i += 4)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum2 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum3 = vfmv_v_f_f16m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum1 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum2 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum3 = vle16_v_f16m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                __fp16 val0 = *tmpptr++;
+                __fp16 val1 = *tmpptr++;
+                __fp16 val2 = *tmpptr++;
+                __fp16 val3 = *tmpptr++;
+                vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl);
+                _sum2 = vfmacc_vf_f16m1(_sum2, val2, _w0, vl);
+                _sum3 = vfmacc_vf_f16m1(_sum3, val3, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse16_v_f16m1(outptr0, _sum0, vl);
+            vse16_v_f16m1(outptr0 + packn, _sum1, vl);
+            vse16_v_f16m1(outptr0 + packn * 2, _sum2, vl);
+            vse16_v_f16m1(outptr0 + packn * 3, _sum3, vl);
+
+            outptr0 += packn * 4;
+        }
+        for (; i + 1 < size; i += 2)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat16m1_t _sum0 = vfmv_v_f_f16m1(0.f, vl);
+            vfloat16m1_t _sum1 = vfmv_v_f_f16m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum0 = vle16_v_f16m1(bias + p * packn, vl);
+                _sum1 = vle16_v_f16m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                __fp16 val0 = *tmpptr++;
+                __fp16 val1 = *tmpptr++;
+                vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl);
+                _sum0 = vfmacc_vf_f16m1(_sum0, val0, _w0, vl);
+                _sum1 = vfmacc_vf_f16m1(_sum1, val1, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse16_v_f16m1(outptr0, _sum0, vl);
+            vse16_v_f16m1(outptr0 + packn, _sum1, vl);
+
+            outptr0 += packn * 2;
+        }
+        for (; i < size; i++)
+        {
+            const __fp16* tmpptr = tmp.channel(i / 8 + (i % 8) / 4 + (i % 4) / 2 + i % 2);
+            const __fp16* kptr0 = kernel.channel(p);
+
+            int nn = inch * maxk * packn; // inch always > 0
+
+            vfloat16m1_t _sum = vfmv_v_f_f16m1(0.f, vl);
+
+            if (bias)
+            {
+                _sum = vle16_v_f16m1(bias + p * packn, vl);
+            }
+
+            for (int j = 0; j < nn; j++)
+            {
+                __fp16 val = *tmpptr++;
+                vfloat16m1_t _w0 = vle16_v_f16m1(kptr0, vl);
+                _sum = vfmacc_vf_f16m1(_sum, val, _w0, vl);
+
+                kptr0 += packn;
+            }
+
+            vse16_v_f16m1(outptr0, _sum, vl);
+
+            outptr0 += packn;
+        }
+    }
+}
+
+static void convolution_im2col_sgemm_packn_fp16sa_rvv(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, int kernel_w, int kernel_h, int dilation_w, int dilation_h, int stride_w, int stride_h, const Option& opt)
+{
+    const int packn = csrr_vlenb() / 2;
+    const word_type vl = vsetvl_e16m1(packn);
+
+    int w = bottom_blob.w;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    const int size = outw * outh;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // im2col
+    Mat bottom_im2col(size, maxk, inch, 2u * packn, packn, opt.workspace_allocator);
+    {
+        const int gap = (w * stride_h - outw * stride_w) * packn;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < inch; p++)
+        {
+            const Mat img = bottom_blob.channel(p);
+            __fp16* ptr = bottom_im2col.channel(p);
+
+            for (int u = 0; u < kernel_h; u++)
+            {
+                for (int v = 0; v < kernel_w; v++)
+                {
+                    const __fp16* sptr = img.row<const __fp16>(dilation_h * u) + dilation_w * v * packn;
+
+                    for (int i = 0; i < outh; i++)
+                    {
+                        int j = 0;
+                        for (; j < outw; j++)
+                        {
+                            vfloat16m1_t _val = vle16_v_f16m1(sptr, vl);
+                            vse16_v_f16m1(ptr, _val, vl);
+
+                            sptr += stride_w * packn;
+                            ptr += packn;
+                        }
+
+                        sptr += gap;
+                    }
+                }
+            }
+        }
+    }
+
+    im2col_sgemm_packn_fp16sa_rvv(bottom_im2col, top_blob, kernel, _bias, opt);
+}