From 241524ffceedd3837d212d4b68cd5b0d879f5382 Mon Sep 17 00:00:00 2001 From: nihui Date: Wed, 1 Jun 2022 10:16:35 +0800 Subject: [PATCH] discard weight memory for x86 arm vulkan (#3865) * discard weight memory for x86 and vulkan * drop arm innerproduct weight * drop arm convolution weight * drop arm convolutiondepthwise weight * drop x86 vulkan deconvolution deconvolutiondepthwise weight * drop arm deconvolution deconvolutiondepthwise weight * arm neon assembly optimization for innerproduct pack4 --- cmake/ncnn_add_layer.cmake | 28 +- src/command.cpp | 2 +- src/layer/arm/convolution_arm.cpp | 293 +++-- src/layer/arm/convolution_arm.h | 13 +- src/layer/arm/convolutiondepthwise_arm.cpp | 120 +- src/layer/arm/convolutiondepthwise_arm.h | 15 +- src/layer/arm/deconvolution_arm.cpp | 138 +- src/layer/arm/deconvolution_arm.h | 12 +- src/layer/arm/deconvolutiondepthwise_arm.cpp | 42 +- src/layer/arm/deconvolutiondepthwise_arm.h | 10 +- src/layer/arm/innerproduct_arm.cpp | 957 ++++++++++---- src/layer/arm/innerproduct_arm.h | 13 +- src/layer/vulkan/convolution_vulkan.cpp | 1138 ++++++++--------- src/layer/vulkan/convolution_vulkan.h | 8 +- .../vulkan/convolutiondepthwise_vulkan.cpp | 122 +- .../vulkan/convolutiondepthwise_vulkan.h | 4 + src/layer/vulkan/deconvolution_vulkan.cpp | 255 ++-- src/layer/vulkan/deconvolution_vulkan.h | 3 + .../vulkan/deconvolutiondepthwise_vulkan.cpp | 186 ++- .../vulkan/deconvolutiondepthwise_vulkan.h | 3 + src/layer/vulkan/innerproduct_vulkan.cpp | 76 +- src/layer/vulkan/innerproduct_vulkan.h | 3 + src/layer/x86/convolution_1x1.h | 51 + src/layer/x86/convolution_x86.cpp | 358 +++--- src/layer/x86/convolution_x86.h | 7 +- src/layer/x86/convolutiondepthwise_x86.cpp | 75 +- src/layer/x86/convolutiondepthwise_x86.h | 8 +- src/layer/x86/deconvolution_x86.cpp | 41 +- src/layer/x86/deconvolution_x86.h | 3 +- src/layer/x86/deconvolutiondepthwise_x86.cpp | 23 +- src/layer/x86/deconvolutiondepthwise_x86.h | 3 +- src/layer/x86/innerproduct_x86.cpp | 511 ++++++-- src/layer/x86/innerproduct_x86.h | 24 +- tests/testutil.h | 2 + 34 files changed, 2672 insertions(+), 1875 deletions(-) diff --git a/cmake/ncnn_add_layer.cmake b/cmake/ncnn_add_layer.cmake index 318d9d484..bd5b77045 100644 --- a/cmake/ncnn_add_layer.cmake +++ b/cmake/ncnn_add_layer.cmake @@ -36,11 +36,6 @@ macro(ncnn_add_arch_opt_layer class NCNN_TARGET_ARCH_OPT NCNN_TARGET_ARCH_OPT_CF set(create_pipeline_content " { int ret = ${class}::create_pipeline(opt); if (ret) return ret; }\n") set(destroy_pipeline_content " { int ret = ${class}::destroy_pipeline(opt); if (ret) return ret; }\n") - set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}.h\"\n") - set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}") - set(create_pipeline_content "${create_pipeline_content} { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::create_pipeline(opt); if (ret) return ret; }\n") - set(destroy_pipeline_content " { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}") - if(WITH_LAYER_${name}_vulkan) set(layer_declaration "${layer_declaration}#include \"layer/vulkan/${name}_vulkan.h\"\n") set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_vulkan") @@ -48,6 +43,11 @@ macro(ncnn_add_arch_opt_layer class NCNN_TARGET_ARCH_OPT NCNN_TARGET_ARCH_OPT_CF set(destroy_pipeline_content " if (vkdev) { int ret = ${class}_vulkan::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}") endif() + set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}.h\"\n") + set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}") + set(create_pipeline_content "${create_pipeline_content} { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::create_pipeline(opt); if (ret) return ret; }\n") + set(destroy_pipeline_content " { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}") + set(layer_declaration "${layer_declaration}namespace ncnn {\n${layer_declaration_class}\n{\n") set(layer_declaration "${layer_declaration}public:\n") set(layer_declaration "${layer_declaration} virtual int create_pipeline(const Option& opt) {\n${create_pipeline_content} return 0;\n }\n") @@ -118,15 +118,6 @@ macro(ncnn_add_layer class) source_group ("sources\\\\layers" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${name}.cpp") endif() - if(WITH_LAYER_${name}_${NCNN_TARGET_ARCH}) - set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.h\"\n") - set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}") - set(create_pipeline_content "${create_pipeline_content} { int ret = ${class}_${NCNN_TARGET_ARCH}::create_pipeline(opt); if (ret) return ret; }\n") - set(destroy_pipeline_content " { int ret = ${class}_${NCNN_TARGET_ARCH}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}") - - source_group ("sources\\\\layers\\\\${NCNN_TARGET_ARCH}" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.cpp") - endif() - if(WITH_LAYER_${name}_vulkan) set(layer_declaration "${layer_declaration}#include \"layer/vulkan/${name}_vulkan.h\"\n") set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_vulkan") @@ -143,6 +134,15 @@ macro(ncnn_add_layer class) source_group ("sources\\\\layers\\\\vulkan" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/vulkan/${name}_vulkan.cpp") endif() + if(WITH_LAYER_${name}_${NCNN_TARGET_ARCH}) + set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.h\"\n") + set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}") + set(create_pipeline_content "${create_pipeline_content} { int ret = ${class}_${NCNN_TARGET_ARCH}::create_pipeline(opt); if (ret) return ret; }\n") + set(destroy_pipeline_content " { int ret = ${class}_${NCNN_TARGET_ARCH}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}") + + source_group ("sources\\\\layers\\\\${NCNN_TARGET_ARCH}" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.cpp") + endif() + if(WITH_LAYER_${name}) set(layer_declaration "${layer_declaration}namespace ncnn {\n${layer_declaration_class}\n{\n") set(layer_declaration "${layer_declaration}public:\n") diff --git a/src/command.cpp b/src/command.cpp index 91a22dd48..f4ec5667e 100644 --- a/src/command.cpp +++ b/src/command.cpp @@ -3175,7 +3175,7 @@ void VkTransfer::record_upload(const Mat& src, VkImageMat& dst, const Option& op // NCNN_LOGE("record_upload image src = %d | %d %d %d @ %d", src.dims, src.w, src.h, src.c, src.elempack); // NOTE keep the hack here ? - if (src.elemsize == src.elempack * 4u) + if (src.elembits() == 32) { if (opt.use_fp16_storage || (opt.use_fp16_packed && src.elempack % 4 == 0)) { diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index a395d3401..1fc08ce56 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -293,12 +293,12 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_pack4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convolution_transform_kernel_pack4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { @@ -308,7 +308,7 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_pack4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } } else if (opt.use_sgemm_convolution && prefer_sgemm) @@ -317,7 +317,7 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_pack4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } } @@ -334,15 +334,15 @@ int Convolution_arm::create_pipeline(const Option& opt) } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } else if (opt.use_sgemm_convolution && prefer_sgemm) { @@ -350,7 +350,7 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack1to4_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } } @@ -375,7 +375,7 @@ int Convolution_arm::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_pack4to1_neon(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h); + convolution_transform_kernel_pack4to1_neon(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h); } } #endif // __ARM_NEON @@ -390,6 +390,10 @@ int Convolution_arm::create_pipeline(const Option& opt) { convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else + { + weight_data_tm = weight_data; + } } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { @@ -397,6 +401,10 @@ int Convolution_arm::create_pipeline(const Option& opt) { convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else + { + weight_data_tm = weight_data; + } } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { @@ -405,6 +413,10 @@ int Convolution_arm::create_pipeline(const Option& opt) // conv3x3s1_winograd63_transform_kernel_neon(weight_data, weight_winograd63_data, num_input, num_output, opt); conv3x3s1_winograd63_transform_kernel_neon5(weight_data, weight_winograd63_data, num_input, num_output, opt); } + else + { + weight_data_tm = weight_data; + } } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { @@ -417,23 +429,37 @@ int Convolution_arm::create_pipeline(const Option& opt) } else if (kernel_w == 4 && kernel_h == 4 && dilation_w == 1 && dilation_h == 1 && stride_w == 4 && stride_h == 4) { + weight_data_tm = weight_data; } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { + weight_data_tm = weight_data; } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { + weight_data_tm = weight_data; } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { + weight_data_tm = weight_data; } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { + weight_data_tm = weight_data; } else if (opt.use_sgemm_convolution && prefer_sgemm) { convolution_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else + { + weight_data_tm = weight_data; + } + } + + if (opt.lightmode) + { + weight_data.release(); } return 0; @@ -461,15 +487,50 @@ int Convolution_arm::destroy_pipeline(const Option& opt) int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_arm(bottom_blob, top_blob, opt); } #endif - if (bottom_blob.dims != 3) + // flattened blob, implement as InnerProduct + if (bottom_blob.dims == 1 && kernel_w == 1 && kernel_h == 1) { - return Convolution::forward(bottom_blob, top_blob, opt); + Mat bottom_blob_3d; + if (bottom_blob.elemsize % 16 == 0) + { + bottom_blob_3d = bottom_blob; + bottom_blob_3d.dims = 3; + bottom_blob_3d.w = 1; + bottom_blob_3d.h = 1; + bottom_blob_3d.c = bottom_blob.w; + bottom_blob_3d.cstep = 1; + } + else + { + bottom_blob_3d = bottom_blob.reshape(1, 1, bottom_blob.w, opt.workspace_allocator); + } + + Mat top_blob_3d; + int ret = forward(bottom_blob_3d, top_blob_3d, opt); + if (ret != 0) + return ret; + + if (top_blob_3d.elemsize % 16 == 0) + { + top_blob = top_blob_3d; + top_blob.dims = 1; + top_blob.w = top_blob_3d.c; + top_blob.h = 1; + top_blob.c = 1; + bottom_blob_3d.cstep = top_blob_3d.c; + } + else + { + top_blob = top_blob_3d.reshape(top_blob_3d.c, opt.blob_allocator); + } + + return 0; } int elembits = bottom_blob.elembits(); @@ -608,7 +669,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -618,7 +679,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -633,7 +694,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -652,7 +713,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -678,7 +739,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -687,7 +748,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -696,7 +757,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv7x7s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv7x7s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -714,7 +775,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -743,7 +804,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option // TODO more proper condition conv3x3s1_winograd63_pack4to1_neon(bottom_blob_bordered, top_blob, weight_winograd63_data, bias_data, opt); - // conv3x3s1_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + // conv3x3s1_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -761,7 +822,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __ARM_NEON @@ -776,7 +837,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv1x1s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv1x1s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -792,7 +853,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -809,7 +870,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv3x3s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -831,7 +892,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 4 && kernel_h == 4 && dilation_w == 1 && dilation_h == 1 && stride_w == 4 && stride_h == 4) { - conv4x4s4_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv4x4s4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -840,7 +901,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -849,7 +910,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -858,7 +919,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv7x7s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv7x7s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -867,7 +928,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv7x7s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + conv7x7s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -923,7 +984,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option sum = bias_data[p]; } - const float* kptr = (const float*)weight_data + maxk * channels * p; + const float* kptr = (const float*)weight_data_tm + maxk * channels * p; // channels for (int q = 0; q < channels; q++) @@ -1064,7 +1125,7 @@ int Convolution_arm::forward(const std::vector& bottom_blobs, std::vector weight_3x3_winograd23_data_int8; #endif diff --git a/src/layer/arm/convolutiondepthwise_arm.cpp b/src/layer/arm/convolutiondepthwise_arm.cpp index a949c0e89..fc7fbecd3 100644 --- a/src/layer/arm/convolutiondepthwise_arm.cpp +++ b/src/layer/arm/convolutiondepthwise_arm.cpp @@ -111,7 +111,7 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) Mat weight_data_r2_packed; convert_packing(weight_data_r2, weight_data_r2_packed, 8, opt); - ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt); } if (elempack == 4) @@ -120,12 +120,12 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) Mat weight_data_r2_packed; convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt); - ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt); } if (elempack == 1) { - ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data, weight_data_tm, opt); } ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); @@ -141,15 +141,16 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_pack4, 4, opt); + Mat weight_data_r2_packed; + convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt); - ncnn::cast_float32_to_bfloat16(weight_data_pack4, weight_data_pack4_bf16, opt); + ncnn::cast_float32_to_bfloat16(weight_data_r2_packed, weight_data_tm, opt); } #endif // __ARM_NEON if (elempack == 1) { - ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt); + ncnn::cast_float32_to_bfloat16(weight_data, weight_data_tm, opt); } return 0; @@ -161,7 +162,7 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_pack4, 4, opt); + convert_packing(weight_data_r2, weight_data_tm, 4, opt); return 0; } @@ -171,18 +172,22 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { + weight_data_tm = weight_data; return 0; } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { + weight_data_tm = weight_data; return 0; } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { + weight_data_tm = weight_data; return 0; } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { + weight_data_tm = weight_data; return 0; } } @@ -191,6 +196,11 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) // group convolution create_group_ops(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -212,7 +222,7 @@ int ConvolutionDepthWise_arm::create_group_ops(const Option& opt) for (int g = 0; g < group; g++) { - Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g); + Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone(); Mat bias_data_g; if (bias_term) bias_data_g = bias_data.range(num_output_g * g, num_output_g); @@ -313,7 +323,7 @@ int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt) int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_arm(bottom_blob, top_blob, opt); } @@ -376,7 +386,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -387,7 +397,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -398,7 +408,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -409,7 +419,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); + convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -445,7 +455,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con for (int g = 0; g < channels; g++) { float* outptr = top_blob.channel(g); - const float* kptr = (const float*)weight_data_pack4 + maxk * g * 4; + const float* kptr = (const float*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -486,7 +496,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -497,7 +507,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -508,7 +518,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -519,7 +529,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -762,7 +772,7 @@ int ConvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blo for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -824,7 +834,7 @@ int ConvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blo for (int g = 0; g < group; g++) { __fp16* outptr = top_blob.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -949,7 +959,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw3x3s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -958,7 +968,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -967,7 +977,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -976,7 +986,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -1010,7 +1020,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 8; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1072,7 +1082,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1110,7 +1120,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw3x3s1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -1119,7 +1129,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt); + convdw3x3s2_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -1153,7 +1163,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl for (int g = 0; g < group; g++) { __fp16* outptr = top_blob.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1288,7 +1298,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + convdw3x3s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1297,7 +1307,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + convdw3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1306,7 +1316,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + convdw5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1315,7 +1325,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo } else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt); + convdw5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1349,7 +1359,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo for (int g = 0; g < channels; g++) { unsigned short* outptr = top_blob.channel(g); - const unsigned short* kptr = (const unsigned short*)weight_data_pack4_bf16 + maxk * g * 4; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1390,7 +1400,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo { // if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) // { - // convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); + // convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); // // if (activation) // { @@ -1401,7 +1411,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo // } // else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) // { - // convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); + // convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); // // if (activation) // { @@ -1412,7 +1422,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo // } // else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) // { - // convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); + // convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); // // if (activation) // { @@ -1423,7 +1433,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo // } // else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) // { - // convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt); + // convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); // // if (activation) // { @@ -1459,7 +1469,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo for (int g = 0; g < group; g++) { unsigned short* outptr = top_blob.channel(g); - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1573,7 +1583,12 @@ int ConvolutionDepthWise_arm::create_pipeline_int8_arm(const Option& opt) if (elempack == 8) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_int8, 8, opt); + convert_packing(weight_data_r2, weight_data_tm, 8, opt); + } + + if (elempack == 1) + { + weight_data_tm = weight_data; } return 0; @@ -1582,6 +1597,11 @@ int ConvolutionDepthWise_arm::create_pipeline_int8_arm(const Option& opt) // group convolution create_group_ops(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -1674,7 +1694,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ if (top_blob_int32.empty()) return -100; - convdw3x3s1_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_int8, opt); + convdw3x3s1_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt); Mat scale_in_data(group); for (int g = 0; g < group; g++) @@ -1710,7 +1730,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ if (top_blob_int32.empty()) return -100; - convdw3x3s2_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_int8, opt); + convdw3x3s2_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt); Mat scale_in_data(group); for (int g = 0; g < group; g++) @@ -1767,7 +1787,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ { signed char* outptr_s8 = top_blob.channel(g); float* outptr_f32 = top_blob.channel(g); - const signed char* kptr = (const signed char*)weight_data_int8 + maxk * g * 8; + const signed char* kptr = (const signed char*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1862,7 +1882,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ requantize_scales.push_back(scale_out); } - convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); + convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt); } else { @@ -1871,8 +1891,8 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ if (top_blob_int32.empty()) return -100; - convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data, opt); - // convdw3x3s1_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data, bias_data, dequantize_scales, opt); + convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt); + // convdw3x3s1_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, bias_data, dequantize_scales, opt); Mat scale_data(group); for (int g = 0; g < group; g++) @@ -1914,7 +1934,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ requantize_scales.push_back(scale_out); } - convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); + convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt); } else { @@ -1923,8 +1943,8 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ if (top_blob_int32.empty()) return -100; - convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data, opt); - // convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data, bias_data, dequantize_scales, opt); + convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt); + // convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, bias_data, dequantize_scales, opt); Mat scale_data(group); for (int g = 0; g < group; g++) @@ -1975,7 +1995,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_ { signed char* outptr_s8 = top_blob.channel(g); float* outptr_f32 = top_blob.channel(g); - const signed char* kptr = (const signed char*)weight_data + maxk * g; + const signed char* kptr = (const signed char*)weight_data_tm + maxk * g; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) diff --git a/src/layer/arm/convolutiondepthwise_arm.h b/src/layer/arm/convolutiondepthwise_arm.h index d9472cd13..2a4c5a4a6 100644 --- a/src/layer/arm/convolutiondepthwise_arm.h +++ b/src/layer/arm/convolutiondepthwise_arm.h @@ -49,23 +49,10 @@ public: Layer* activation; std::vector group_ops; - // packing - Mat weight_data_pack4; + Mat weight_data_tm; // fp16 - Mat weight_data_fp16; Mat bias_data_fp16; - -#if NCNN_BF16 - // bf16 - Mat weight_data_bf16; - Mat weight_data_pack4_bf16; -#endif - -#if NCNN_INT8 - // int8 - Mat weight_data_int8; -#endif }; } // namespace ncnn diff --git a/src/layer/arm/deconvolution_arm.cpp b/src/layer/arm/deconvolution_arm.cpp index 45653b0bb..80e8ba13c 100644 --- a/src/layer/arm/deconvolution_arm.cpp +++ b/src/layer/arm/deconvolution_arm.cpp @@ -139,7 +139,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_pack4.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16); + weight_data_tm.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16); for (int q = 0; q + 3 < num_output; q += 4) { @@ -148,7 +148,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt) const Mat k2 = weight_data_r2.channel(q + 2); const Mat k3 = weight_data_r2.channel(q + 3); - float* g00 = weight_data_pack4.channel(q / 4); + float* g00 = weight_data_tm.channel(q / 4); for (int p = 0; p + 3 < num_input; p += 4) { @@ -209,7 +209,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_pack1to4.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4); + weight_data_tm.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4); for (int q = 0; q + 3 < num_output; q += 4) { @@ -218,7 +218,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt) const Mat k2 = weight_data_r2.channel(q + 2); const Mat k3 = weight_data_r2.channel(q + 3); - float* g00 = weight_data_pack1to4.channel(q / 4); + float* g00 = weight_data_tm.channel(q / 4); for (int p = 0; p < num_input; p++) { @@ -249,12 +249,12 @@ int Deconvolution_arm::create_pipeline(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_pack4to1.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4); + weight_data_tm.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4); for (int q = 0; q < num_output; q++) { const Mat k0 = weight_data_r2.channel(q); - float* g00 = weight_data_pack4to1.channel(q); + float* g00 = weight_data_tm.channel(q); for (int p = 0; p + 3 < num_input; p += 4) { @@ -281,7 +281,31 @@ int Deconvolution_arm::create_pipeline(const Option& opt) // pack1 if (elempack == 1 && out_elempack == 1) { - weight_data_pack1 = weight_data_transposed; + if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + { + weight_data_tm = weight_data; + } + else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + { + weight_data_tm = weight_data; + } + else if (kernel_w == 4 && kernel_h == 4 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + { + weight_data_tm = weight_data; + } + else if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + { + weight_data_tm = weight_data; + } + else + { + weight_data_tm = weight_data_transposed; + } + } + + if (opt.lightmode) + { + weight_data.release(); } return 0; @@ -318,9 +342,6 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti return forward_bf16s(bottom_blob, top_blob, opt); #endif - // deconvolv with NxN kernel - // value = value + bias - int w = bottom_blob.w; int h = bottom_blob.h; int channels = bottom_blob.c; @@ -378,7 +399,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16; + const float* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -462,7 +483,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4; + const float* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -518,7 +539,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti { // num_output #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output / out_elempack; p++) + for (int p = 0; p < num_output; p++) { float* outptr = top_blob_bordered.channel(p); @@ -533,7 +554,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti sum = bias_data[p]; } - const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4; + const float* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -597,7 +618,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti { if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) { - deconv3x3s1_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt); + deconv3x3s1_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt); if (activation) { @@ -606,7 +627,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti } else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) { - deconv3x3s2_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt); + deconv3x3s2_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt); if (activation) { @@ -615,7 +636,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti } else if (kernel_w == 4 && kernel_h == 4 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) { - deconv4x4s1_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt); + deconv4x4s1_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt); if (activation) { @@ -624,7 +645,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti } else if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) { - deconv4x4s2_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt); + deconv4x4s2_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt); if (activation) { @@ -650,7 +671,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti sum = bias_data[p]; } - const float* kptr = (const float*)weight_data_pack1 + maxk * channels * p; + const float* kptr = (const float*)weight_data_tm + maxk * channels * p; // channels for (int q = 0; q < channels; q++) @@ -692,28 +713,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti kptr += maxk; } - if (activation_type == 1) - { - sum = std::max(sum, 0.f); - } - else if (activation_type == 2) - { - float slope = activation_params[0]; - sum = sum > 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; - } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } + sum = activation_ss(sum, activation_type, activation_params); outptr[j] = sum; } @@ -768,11 +768,11 @@ int Deconvolution_arm::create_pipeline_fp16s(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_fp16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack); + weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - __fp16* g00 = weight_data_fp16.channel(q / out_elempack); + __fp16* g00 = weight_data_tm.channel(q / out_elempack); for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { @@ -798,12 +798,17 @@ int Deconvolution_arm::create_pipeline_fp16s(const Option& opt) { if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) { - ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data, weight_data_tm, opt); } } ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -863,7 +868,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -942,7 +947,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1015,7 +1020,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons sum = bias_data[p]; } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1090,7 +1095,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons sum = bias_data[p]; } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1210,7 +1215,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1297,7 +1302,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1370,7 +1375,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1449,7 +1454,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con sum = bias_data[p]; } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1525,7 +1530,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1612,7 +1617,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1691,7 +1696,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4); } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1764,7 +1769,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con sum = bias_data[p]; } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1823,7 +1828,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) { - deconv4x4s2_fp16sa_neon(bottom_blob, top_blob_bordered, weight_data_fp16, bias_data_fp16, opt); + deconv4x4s2_fp16sa_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data_fp16, opt); if (activation) { @@ -1849,7 +1854,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con sum = bias_data[p]; } - const __fp16* kptr = weight_data_fp16.channel(p); + const __fp16* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -1948,11 +1953,11 @@ int Deconvolution_arm::create_pipeline_bf16s(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_bf16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack); + weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - unsigned short* g00 = weight_data_bf16.channel(q / out_elempack); + unsigned short* g00 = weight_data_tm.channel(q / out_elempack); for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { @@ -1974,6 +1979,11 @@ int Deconvolution_arm::create_pipeline_bf16s(const Option& opt) } } + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -2040,7 +2050,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const unsigned short* kptr = weight_data_bf16.channel(p); + const unsigned short* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -2126,7 +2136,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const unsigned short* kptr = weight_data_bf16.channel(p); + const unsigned short* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -2199,7 +2209,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons sum = bias_data[p]; } - const unsigned short* kptr = weight_data_bf16.channel(p); + const unsigned short* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) @@ -2280,7 +2290,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons sum = bias_data[p]; } - const unsigned short* kptr = weight_data_bf16.channel(p); + const unsigned short* kptr = weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) diff --git a/src/layer/arm/deconvolution_arm.h b/src/layer/arm/deconvolution_arm.h index 08bb004eb..964d52385 100644 --- a/src/layer/arm/deconvolution_arm.h +++ b/src/layer/arm/deconvolution_arm.h @@ -43,20 +43,10 @@ protected: public: Layer* activation; - // pack4 - Mat weight_data_pack4; - Mat weight_data_pack1to4; - Mat weight_data_pack4to1; - Mat weight_data_pack1; + Mat weight_data_tm; // fp16 - Mat weight_data_fp16; Mat bias_data_fp16; - -#if NCNN_BF16 - // bf16 - Mat weight_data_bf16; -#endif }; } // namespace ncnn diff --git a/src/layer/arm/deconvolutiondepthwise_arm.cpp b/src/layer/arm/deconvolutiondepthwise_arm.cpp index 83bb88a91..c004eaae7 100644 --- a/src/layer/arm/deconvolutiondepthwise_arm.cpp +++ b/src/layer/arm/deconvolutiondepthwise_arm.cpp @@ -86,7 +86,7 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) Mat weight_data_r2_packed; convert_packing(weight_data_r2, weight_data_r2_packed, 8, opt); - ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt); } if (elempack == 4) @@ -95,12 +95,12 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) Mat weight_data_r2_packed; convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt); - ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt); } if (elempack == 1) { - ncnn::cast_float32_to_float16(weight_data_transposed, weight_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_data_transposed, weight_data_tm, opt); } ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); @@ -116,15 +116,16 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_pack4, 4, opt); + Mat weight_data_r2_packed; + convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt); - ncnn::cast_float32_to_bfloat16(weight_data_pack4, weight_data_bf16, opt); + ncnn::cast_float32_to_bfloat16(weight_data_r2_packed, weight_data_tm, opt); } #endif // __ARM_NEON if (elempack == 1) { - ncnn::cast_float32_to_bfloat16(weight_data_transposed, weight_data_bf16, opt); + ncnn::cast_float32_to_bfloat16(weight_data_transposed, weight_data_tm, opt); } return 0; @@ -136,14 +137,14 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_pack4, 4, opt); + convert_packing(weight_data_r2, weight_data_tm, 4, opt); } #endif // __ARM_NEON // pack1 if (elempack == 1) { - weight_data_pack1 = weight_data_transposed; + weight_data_tm = weight_data_transposed; } } else @@ -161,7 +162,7 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) for (int g = 0; g < group; g++) { - Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g); + Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone(); Mat bias_data_g; if (bias_term) bias_data_g = bias_data.range(num_output_g * g, num_output_g); @@ -211,6 +212,11 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt) } } + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -293,7 +299,7 @@ int DeconvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_pack4 + maxk * g * 4; + const float* kptr = (const float*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -356,7 +362,7 @@ int DeconvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_pack1 + maxk * g; + const float* kptr = (const float*)weight_data_tm + maxk * g; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -540,7 +546,7 @@ int DeconvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_b for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob_bordered.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -604,7 +610,7 @@ int DeconvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_b for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob_bordered.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -764,7 +770,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_ for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob_bordered.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 8; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -828,7 +834,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_ for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob_bordered.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -892,7 +898,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_ for (int g = 0; g < channels; g++) { __fp16* outptr = top_blob_bordered.channel(g); - const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g; + const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -1061,7 +1067,7 @@ int DeconvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_b for (int g = 0; g < channels; g++) { unsigned short* outptr = top_blob_bordered.channel(g); - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g * 4; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -1124,7 +1130,7 @@ int DeconvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_b for (int g = 0; g < channels; g++) { unsigned short* outptr = top_blob_bordered.channel(g); - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) diff --git a/src/layer/arm/deconvolutiondepthwise_arm.h b/src/layer/arm/deconvolutiondepthwise_arm.h index 9ef46009a..406295efc 100644 --- a/src/layer/arm/deconvolutiondepthwise_arm.h +++ b/src/layer/arm/deconvolutiondepthwise_arm.h @@ -41,18 +41,10 @@ protected: public: std::vector group_ops; - // packing - Mat weight_data_pack4; - Mat weight_data_pack1; + Mat weight_data_tm; // fp16 - Mat weight_data_fp16; Mat bias_data_fp16; - -#if NCNN_BF16 - // bf16 - Mat weight_data_bf16; -#endif }; } // namespace ncnn diff --git a/src/layer/arm/innerproduct_arm.cpp b/src/layer/arm/innerproduct_arm.cpp index bb4eb66a9..c01eb1ec7 100644 --- a/src/layer/arm/innerproduct_arm.cpp +++ b/src/layer/arm/innerproduct_arm.cpp @@ -39,7 +39,6 @@ InnerProduct_arm::InnerProduct_arm() #endif flatten = 0; - activation = 0; } int InnerProduct_arm::create_pipeline(const Option& opt) @@ -75,6 +74,50 @@ int InnerProduct_arm::create_pipeline(const Option& opt) } #endif + const int num_input = weight_data_size / num_output; + + int out_elempack = 1; + +#if __ARM_NEON + if (opt.use_packing_layout) + { + out_elempack = num_output % 4 == 0 ? 4 : 1; + } +#endif // __ARM_NEON + + if (out_elempack == 4) + { + // src = inch-outch + // dst = pb-inch-outch/pb + { + Mat weight_data_r2 = weight_data.reshape(num_input, num_output); + + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + float* g0 = weight_data_tm.row(q / out_elempack); + + for (int p = 0; p < num_input; p++) + { + for (int j = 0; j < out_elempack; j++) + { + *g0++ = weight_data_r2.row(q + j)[p]; + } + } + } + } + } + else + { + weight_data_tm = weight_data; + } + + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -87,20 +130,13 @@ int InnerProduct_arm::destroy_pipeline(const Option& opt) flatten = 0; } - if (activation) - { - activation->destroy_pipeline(opt); - delete activation; - activation = 0; - } - return 0; } int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_arm(bottom_blob, top_blob, opt); } @@ -136,17 +172,138 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio if (top_blob.empty()) return -100; + int num_output_elempack = 1; +#if __ARM_NEON + if (opt.use_packing_layout) + { + num_output_elempack = num_output % 4 == 0 ? 4 : 1; + } +#endif + #pragma omp parallel for num_threads(opt.num_threads) for (int j = 0; j < h; j++) { #if __ARM_NEON - if (elempack == 4) + if (elempack == 4 && num_output_elempack == 4) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output / num_output_elempack; p++) + { + const float* kptr = weight_data_tm.row(p); + const float* m = bottom_blob.row(j); + + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + + if (bias_term) + { + _sum0 = vdupq_n_f32(bias_data[p * 4 + 0]); + _sum1 = vdupq_n_f32(bias_data[p * 4 + 1]); + _sum2 = vdupq_n_f32(bias_data[p * 4 + 2]); + _sum3 = vdupq_n_f32(bias_data[p * 4 + 3]); + } + + int i = 0; + for (; i < num_input; i++) + { + float32x4_t _val = vld1q_f32(m); + float32x4_t _w = vld1q_f32(kptr); +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _val, _w, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _val, _w, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _val, _w, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _val, _w, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _val, vget_low_f32(_w), 0); + _sum1 = vmlaq_lane_f32(_sum1, _val, vget_low_f32(_w), 1); + _sum2 = vmlaq_lane_f32(_sum2, _val, vget_high_f32(_w), 0); + _sum3 = vmlaq_lane_f32(_sum3, _val, vget_high_f32(_w), 1); +#endif + m += 4; + kptr += 4; + } + + _sum0 = activation_ps(_sum0, activation_type, activation_params); + _sum1 = activation_ps(_sum1, activation_type, activation_params); + _sum2 = activation_ps(_sum2, activation_type, activation_params); + _sum3 = activation_ps(_sum3, activation_type, activation_params); + + vst1q_f32(outptr, _sum0); + vst1q_f32(outptr + 4, _sum1); + vst1q_f32(outptr + 8, _sum2); + vst1q_f32(outptr + 12, _sum3); + outptr += 16; + } + } + + if (elempack == 1 && num_output_elempack == 4) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output / num_output_elempack; p++) + { + const float* kptr = weight_data_tm.row(p); + const float* m = bottom_blob.row(j); + + float32x4_t _sum = vdupq_n_f32(0.f); + + if (bias_term) + { + _sum = vld1q_f32((const float*)bias_data + p * 4); + } + + int i = 0; + for (; i + 3 < num_input; i += 4) + { + float32x4_t _val = vld1q_f32(m); + + float32x4_t _w0 = vld1q_f32(kptr); + float32x4_t _w1 = vld1q_f32(kptr + 4); + float32x4_t _w2 = vld1q_f32(kptr + 8); + float32x4_t _w3 = vld1q_f32(kptr + 12); + +#if __aarch64__ + _sum = vfmaq_laneq_f32(_sum, _w0, _val, 0); + _sum = vfmaq_laneq_f32(_sum, _w1, _val, 1); + _sum = vfmaq_laneq_f32(_sum, _w2, _val, 2); + _sum = vfmaq_laneq_f32(_sum, _w3, _val, 3); +#else + _sum = vmlaq_lane_f32(_sum, _w0, vget_low_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w1, vget_low_f32(_val), 1); + _sum = vmlaq_lane_f32(_sum, _w2, vget_high_f32(_val), 0); + _sum = vmlaq_lane_f32(_sum, _w3, vget_high_f32(_val), 1); +#endif + + m += 4; + kptr += 16; + } + for (; i < num_input; i++) + { + float32x4_t _val = vld1q_dup_f32(m); + float32x4_t _k = vld1q_f32(kptr); + _sum = vmlaq_f32(_sum, _val, _k); + + m += 1; + kptr += 4; + } + + _sum = activation_ps(_sum, activation_type, activation_params); + + vst1q_f32(outptr, _sum); + outptr += 4; + } + } + + if (elempack == 4 && num_output_elempack == 1) { float* outptr = top_blob.row(j); for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); float32x4_t _sum = vdupq_n_f32(0.f); @@ -174,13 +331,13 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio } #endif // __ARM_NEON - if (elempack == 1) + if (elempack == 1 && num_output_elempack == 1) { float* outptr = top_blob.row(j); for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); float sum = 0.f; @@ -229,90 +386,200 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio return 0; } - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; - int elempack = bottom_blob.elempack; - int size = w * h; + // flatten + Mat bottom_blob_flattened = bottom_blob; + if (bottom_blob.dims != 1) + { + Option opt_flatten = opt; + opt_flatten.blob_allocator = opt.workspace_allocator; + + flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); + } + size_t elemsize = bottom_blob_flattened.elemsize; + int elempack = bottom_blob_flattened.elempack; + + int out_elempack = 1; #if __ARM_NEON - if (elempack == 4) + if (opt.use_packing_layout) { - // flatten - Mat bottom_blob_flattened = bottom_blob; - if (bottom_blob.dims != 1) - { - Option opt_flatten = opt; - opt_flatten.blob_allocator = opt.workspace_allocator; + out_elempack = num_output % 4 == 0 ? 4 : 1; + } +#endif // __ARM_NEON + size_t out_elemsize = elemsize / elempack * out_elempack; - flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); - } + top_blob.create(num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; - // pack1 +#if __ARM_NEON + if (out_elempack == 4) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output / out_elempack; p++) { - bottom_blob_flattened.w *= bottom_blob_flattened.elempack; - bottom_blob_flattened.cstep = bottom_blob_flattened.w; - bottom_blob_flattened.elemsize = 4u; - bottom_blob_flattened.elempack = 1; - } + float32x4_t _sum0 = bias_term ? vld1q_f32((const float*)bias_data + p * 4) : vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); - return forward(bottom_blob_flattened, top_blob, opt); - } + const float* kptr = weight_data_tm.row(p); + + const float* sptr = bottom_blob_flattened; + + int i = 0; + for (; i + 7 < num_input; i += 8) + { +#if __aarch64__ + asm volatile( + "prfm pldl1keep, [%0, #256] \n" + "ld1 {v0.4s, v1.4s}, [%0], #32 \n" + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v2.4s, v3.4s, v4.4s, v5.4s}, [%1], #64 \n" + "prfm pldl1keep, [%1, #512] \n" + "ld1 {v6.4s, v7.4s, v8.4s, v9.4s}, [%1], #64 \n" + "fmla %2.4s, v2.4s, v0.s[0] \n" + "fmla %3.4s, v3.4s, v0.s[1] \n" + "fmla %4.4s, v4.4s, v0.s[2] \n" + "fmla %5.4s, v5.4s, v0.s[3] \n" + "fmla %2.4s, v6.4s, v1.s[0] \n" + "fmla %3.4s, v7.4s, v1.s[1] \n" + "fmla %4.4s, v8.4s, v1.s[2] \n" + "fmla %5.4s, v9.4s, v1.s[3] \n" + : "=r"(sptr), // %0 + "=r"(kptr), // %1 + "=w"(_sum0), // %2 + "=w"(_sum1), // %3 + "=w"(_sum2), // %4 + "=w"(_sum3) // %5 + : "0"(sptr), + "1"(kptr), + "2"(_sum0), + "3"(_sum1), + "4"(_sum2), + "5"(_sum3) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9"); +#else + asm volatile( + "pld [%0, #256] \n" + "vld1.f32 {d0-d3}, [%0 :128]! \n" + "pld [%1, #512] \n" + "vldm %1!, {d4-d11} \n" + "pld [%1, #512] \n" + "vldm %1!, {d12-d19} \n" + "vmla.f32 %q2, q2, d0[0] \n" + "vmla.f32 %q3, q3, d0[1] \n" + "vmla.f32 %q4, q4, d1[0] \n" + "vmla.f32 %q5, q5, d1[1] \n" + "vmla.f32 %q2, q6, d2[0] \n" + "vmla.f32 %q3, q7, d2[1] \n" + "vmla.f32 %q4, q8, d3[0] \n" + "vmla.f32 %q5, q9, d3[1] \n" + : "=r"(sptr), // %0 + "=r"(kptr), // %1 + "=w"(_sum0), // %2 + "=w"(_sum1), // %3 + "=w"(_sum2), // %4 + "=w"(_sum3) // %5 + : "0"(sptr), + "1"(kptr), + "2"(_sum0), + "3"(_sum1), + "4"(_sum2), + "5"(_sum3) + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9"); #endif + } + for (; i + 3 < num_input; i += 4) + { + float32x4_t _val = vld1q_f32(sptr); - top_blob.create(num_output, elemsize, opt.blob_allocator); - if (top_blob.empty()) - return -100; + float32x4_t _w0 = vld1q_f32(kptr); + float32x4_t _w1 = vld1q_f32(kptr + 4); + float32x4_t _w2 = vld1q_f32(kptr + 8); + float32x4_t _w3 = vld1q_f32(kptr + 12); - const float* weight_data_ptr = weight_data; +#if __aarch64__ + _sum0 = vfmaq_laneq_f32(_sum0, _w0, _val, 0); + _sum1 = vfmaq_laneq_f32(_sum1, _w1, _val, 1); + _sum2 = vfmaq_laneq_f32(_sum2, _w2, _val, 2); + _sum3 = vfmaq_laneq_f32(_sum3, _w3, _val, 3); +#else + _sum0 = vmlaq_lane_f32(_sum0, _w0, vget_low_f32(_val), 0); + _sum1 = vmlaq_lane_f32(_sum1, _w1, vget_low_f32(_val), 1); + _sum2 = vmlaq_lane_f32(_sum2, _w2, vget_high_f32(_val), 0); + _sum3 = vmlaq_lane_f32(_sum3, _w3, vget_high_f32(_val), 1); +#endif - int nn_num_output = num_output >> 2; - int remain_num_output_start = nn_num_output << 2; + sptr += 4; + kptr += 16; + } + for (; i < num_input; i++) + { + float32x4_t _val = vld1q_dup_f32(sptr); + float32x4_t _w = vld1q_f32(kptr); + _sum0 = vmlaq_f32(_sum0, _val, _w); - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < nn_num_output; pp++) - { - int p = pp * 4; + sptr += 1; + kptr += 4; + } - float sum0 = 0.f; - float sum1 = 0.f; - float sum2 = 0.f; - float sum3 = 0.f; + _sum0 = vaddq_f32(_sum0, _sum1); + _sum2 = vaddq_f32(_sum2, _sum3); + _sum0 = vaddq_f32(_sum0, _sum2); - if (bias_term) - { - sum0 = bias_data[p]; - sum1 = bias_data[p + 1]; - sum2 = bias_data[p + 2]; - sum3 = bias_data[p + 3]; + _sum0 = activation_ps(_sum0, activation_type, activation_params); + + float* outptr = top_blob; + vst1q_f32(outptr + p * 4, _sum0); } + } +#endif // __ARM_NEON - const float* w0 = weight_data_ptr + size * channels * p; - const float* w1 = weight_data_ptr + size * channels * (p + 1); - const float* w2 = weight_data_ptr + size * channels * (p + 2); - const float* w3 = weight_data_ptr + size * channels * (p + 3); + if (out_elempack == 1) + { + const float* weight_data_ptr = weight_data_tm; -#if __ARM_NEON - float32x4_t _sum0 = vdupq_n_f32(0.f); - float32x4_t _sum1 = vdupq_n_f32(0.f); - float32x4_t _sum2 = vdupq_n_f32(0.f); - float32x4_t _sum3 = vdupq_n_f32(0.f); -#endif // __ARM_NEON + int nn_num_output = num_output >> 2; + int remain_num_output_start = nn_num_output << 2; - // channels - for (int q = 0; q < channels; q++) + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_num_output; pp++) { - const float* m = bottom_blob.channel(q); + int p = pp * 4; + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + if (bias_term) + { + sum0 = bias_data[p]; + sum1 = bias_data[p + 1]; + sum2 = bias_data[p + 2]; + sum3 = bias_data[p + 3]; + } + + const float* w0 = weight_data_ptr + num_input * p; + const float* w1 = weight_data_ptr + num_input * (p + 1); + const float* w2 = weight_data_ptr + num_input * (p + 2); + const float* w3 = weight_data_ptr + num_input * (p + 3); + + const float* m = bottom_blob_flattened; #if __ARM_NEON - int nn = size >> 3; - int remain = size & 7; + int nn = num_input >> 3; + int remain = num_input & 7; #else - int remain = size; + int remain = num_input; #endif // __ARM_NEON #if __ARM_NEON + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); #if __aarch64__ if (nn > 0) { @@ -422,64 +689,57 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio w2++; w3++; } - } #if __ARM_NEON - float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); - float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1)); - float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2)); - float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3)); + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); + float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1)); + float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2)); + float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3)); - float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); - float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); + float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); + float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); - sum0 += vget_lane_f32(_sum01ss, 0); - sum1 += vget_lane_f32(_sum01ss, 1); - sum2 += vget_lane_f32(_sum23ss, 0); - sum3 += vget_lane_f32(_sum23ss, 1); + sum0 += vget_lane_f32(_sum01ss, 0); + sum1 += vget_lane_f32(_sum01ss, 1); + sum2 += vget_lane_f32(_sum23ss, 0); + sum3 += vget_lane_f32(_sum23ss, 1); #endif // __ARM_NEON - sum0 = activation_ss(sum0, activation_type, activation_params); - sum1 = activation_ss(sum1, activation_type, activation_params); - sum2 = activation_ss(sum2, activation_type, activation_params); - sum3 = activation_ss(sum3, activation_type, activation_params); - - top_blob[p] = sum0; - top_blob[p + 1] = sum1; - top_blob[p + 2] = sum2; - top_blob[p + 3] = sum3; - } + sum0 = activation_ss(sum0, activation_type, activation_params); + sum1 = activation_ss(sum1, activation_type, activation_params); + sum2 = activation_ss(sum2, activation_type, activation_params); + sum3 = activation_ss(sum3, activation_type, activation_params); - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = remain_num_output_start; p < num_output; p++) - { - float sum = 0.f; + top_blob[p] = sum0; + top_blob[p + 1] = sum1; + top_blob[p + 2] = sum2; + top_blob[p + 3] = sum3; + } - if (bias_term) - sum = bias_data[p]; + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_num_output_start; p < num_output; p++) + { + float sum = 0.f; - const float* w = weight_data_ptr + size * channels * p; + if (bias_term) + sum = bias_data[p]; -#if __ARM_NEON - float32x4_t _sum = vdupq_n_f32(0.f); - float32x4_t _sum2 = vdupq_n_f32(0.f); -#endif // __ARM_NEON + const float* w = weight_data_ptr + num_input * p; - // channels - for (int q = 0; q < channels; q++) - { - const float* m = bottom_blob.channel(q); + const float* m = bottom_blob_flattened; #if __ARM_NEON - int nn = size >> 3; - int remain = size & 7; + int nn = num_input >> 3; + int remain = num_input & 7; #else - int remain = size; + int remain = num_input; #endif // __ARM_NEON #if __ARM_NEON + float32x4_t _sum = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); #if __aarch64__ if (nn > 0) { @@ -539,22 +799,22 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio m++; w++; } - } #if __ARM_NEON - _sum = vaddq_f32(_sum, _sum2); + _sum = vaddq_f32(_sum, _sum2); #if __aarch64__ - sum += vaddvq_f32(_sum); + sum += vaddvq_f32(_sum); #else - float32x2_t _sumss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum)); - _sumss = vpadd_f32(_sumss, _sumss); - sum += vget_lane_f32(_sumss, 0); + float32x2_t _sumss = vadd_f32(vget_low_f32(_sum), vget_high_f32(_sum)); + _sumss = vpadd_f32(_sumss, _sumss); + sum += vget_lane_f32(_sumss, 0); #endif // __aarch64__ #endif // __ARM_NEON - sum = activation_ss(sum, activation_type, activation_params); + sum = activation_ss(sum, activation_type, activation_params); - top_blob[p] = sum; + top_blob[p] = sum; + } } return 0; @@ -577,11 +837,11 @@ int InnerProduct_arm::create_pipeline_fp16s(const Option& opt) { Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - weight_data_fp16.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - __fp16* g0 = weight_data_fp16.row<__fp16>(q / out_elempack); + __fp16* g0 = weight_data_tm.row<__fp16>(q / out_elempack); for (int p = 0; p < num_input; p++) { @@ -595,6 +855,11 @@ int InnerProduct_arm::create_pipeline_fp16s(const Option& opt) ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -628,7 +893,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; const __fp16* m = bottom_blob.row(j); float32x4_t _sum0 = vdupq_n_f32(0.f); @@ -676,7 +941,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; const __fp16* m = bottom_blob.row(j); float32x4_t _sum = vdupq_n_f32(0.f); @@ -709,7 +974,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; const __fp16* m = bottom_blob.row(j); float32x4_t _sum = vdupq_n_f32(0.f); @@ -742,7 +1007,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; const __fp16* m = bottom_blob.row(j); float sum = 0.f; @@ -804,7 +1069,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const _sum = vld1q_f32(((const float*)bias_data) + p * 4); } - const __fp16* kptr = weight_data_fp16.row(p); + const __fp16* kptr = weight_data_tm.row(p); const __fp16* sptr = bottom_blob_flattened; @@ -856,7 +1121,7 @@ int InnerProduct_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, const if (bias_term) sum = bias_data[p]; - const __fp16* kptr = weight_data_fp16.row<__fp16>(p); + const __fp16* kptr = weight_data_tm.row<__fp16>(p); const __fp16* sptr = bottom_blob_flattened; @@ -925,7 +1190,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 8; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8; const __fp16* m = bottom_blob.row(j); float16x8_t _sum0 = vdupq_n_f16((__fp16)0.f); @@ -993,7 +1258,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 8; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8; const __fp16* m = bottom_blob.row(j); float16x8_t _sum = vdupq_n_f16(0.f); @@ -1026,7 +1291,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 8; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 8; const __fp16* m = bottom_blob.row(j); float16x4_t _sum0 = vdup_n_f16(0.f); @@ -1094,7 +1359,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; const __fp16* m = bottom_blob.row(j); float16x8_t _sum = vdupq_n_f16((__fp16)0.f); @@ -1127,7 +1392,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; const __fp16* m = bottom_blob.row(j); float16x8_t _sum0 = vdupq_n_f16((__fp16)0.f); @@ -1175,7 +1440,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; const __fp16* m = bottom_blob.row(j); float16x4_t _sum0 = vdup_n_f16(0.f); @@ -1223,7 +1488,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output / num_output_elempack; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p * 4; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p * 4; const __fp16* m = bottom_blob.row(j); float16x4_t _sum = vdup_n_f16(0.f); @@ -1256,7 +1521,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; const __fp16* m = bottom_blob.row(j); float16x4_t _sum = vdup_n_f16(0.f); @@ -1289,7 +1554,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons for (int p = 0; p < num_output; p++) { - const __fp16* kptr = (const __fp16*)weight_data_fp16 + num_input * p; + const __fp16* kptr = (const __fp16*)weight_data_tm + num_input * p; const __fp16* m = bottom_blob.row(j); float sum = 0.f; @@ -1362,7 +1627,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons _sum0 = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8); } - const __fp16* kptr = weight_data_fp16.row(p); + const __fp16* kptr = weight_data_tm.row(p); const __fp16* sptr = bottom_blob_flattened; @@ -1485,7 +1750,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons _sum0 = vld1_f16((const __fp16*)bias_data_fp16 + p * 4); } - const __fp16* kptr = weight_data_fp16.row(p); + const __fp16* kptr = weight_data_tm.row(p); const __fp16* sptr = bottom_blob_flattened; @@ -1599,7 +1864,7 @@ int InnerProduct_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, cons if (bias_term) sum = bias_data[p]; - const __fp16* kptr = weight_data_fp16.row<__fp16>(p); + const __fp16* kptr = weight_data_tm.row<__fp16>(p); const __fp16* sptr = bottom_blob_flattened; @@ -1658,11 +1923,11 @@ int InnerProduct_arm::create_pipeline_bf16s(const Option& opt) { Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - weight_data_bf16.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)2u * out_elempack, out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - unsigned short* g0 = weight_data_bf16.row(q / out_elempack); + unsigned short* g0 = weight_data_tm.row(q / out_elempack); for (int p = 0; p < num_input; p++) { @@ -1674,6 +1939,11 @@ int InnerProduct_arm::create_pipeline_bf16s(const Option& opt) } } + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -1710,7 +1980,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output / num_output_elempack; p++) { - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + num_input * p * 4; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + num_input * p * 4; const unsigned short* m = bottom_blob.row(j); float32x4_t _sum0 = vdupq_n_f32(0.f); @@ -1765,7 +2035,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output / num_output_elempack; p++) { - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + num_input * p * 4; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + num_input * p * 4; const unsigned short* m = bottom_blob.row(j); float32x4_t _sum = vdupq_n_f32(0.f); @@ -1798,7 +2068,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output; p++) { - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + num_input * p; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + num_input * p; const unsigned short* m = bottom_blob.row(j); float32x4_t _sum = vdupq_n_f32(0.f); @@ -1832,7 +2102,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const for (int p = 0; p < num_output; p++) { - const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + num_input * p; + const unsigned short* kptr = (const unsigned short*)weight_data_tm + num_input * p; const unsigned short* m = bottom_blob.row(j); float sum = 0.f; @@ -1904,7 +2174,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const _sum0 = vld1q_f32(((const float*)bias_data) + p * 4); } - const unsigned short* kptr = weight_data_bf16.row(p); + const unsigned short* kptr = weight_data_tm.row(p); const unsigned short* sptr = bottom_blob_flattened; @@ -1968,7 +2238,7 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const if (bias_term) sum = bias_data[p]; - const unsigned short* kptr = weight_data_bf16.row(p); + const unsigned short* kptr = weight_data_tm.row(p); const unsigned short* sptr = bottom_blob_flattened; @@ -2021,8 +2291,6 @@ int InnerProduct_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const #if NCNN_INT8 int InnerProduct_arm::create_pipeline_int8_arm(const Option& opt) { - activation = create_activation_layer(activation_type, activation_params, opt); - const int num_input = weight_data_size / num_output; int out_elempack = 1; @@ -2038,11 +2306,11 @@ int InnerProduct_arm::create_pipeline_int8_arm(const Option& opt) { Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - weight_data_int8.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack); + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - signed char* g0 = weight_data_int8.row(q / out_elempack); + signed char* g0 = weight_data_tm.row(q / out_elempack); for (int p = 0; p < num_input; p++) { @@ -2054,6 +2322,24 @@ int InnerProduct_arm::create_pipeline_int8_arm(const Option& opt) } } + scale_in_data.create(num_output); + for (int p = 0; p < num_output; p++) + { + // dequantize + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); + + scale_in_data[p] = scale_in; + } + + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -2074,135 +2360,157 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co if (bottom_blob_int8.dims == 2 && bottom_blob_int8.w == num_input && bottom_blob_int8.h * bottom_blob_int8.elempack > 1) { // gemm - int h = bottom_blob_int8.h; - int elempack = bottom_blob_int8.elempack; + Mat bottom_blob_int8_unpacked; + Option opt_unpack = opt; + opt_unpack.blob_allocator = opt.workspace_allocator; + convert_packing(bottom_blob_int8, bottom_blob_int8_unpacked, 1, opt_unpack); + + int h = bottom_blob_int8_unpacked.h; int out_elempack = 1; #if __ARM_NEON if (opt.use_packing_layout) { - out_elempack = h * elempack % 4 == 0 ? 4 : 1; + out_elempack = h % 4 == 0 ? 4 : 1; } #endif - int outh = h * elempack / out_elempack; + int outh = h / out_elempack; top_blob.create(num_output, outh, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator); if (top_blob.empty()) return -100; - Mat scale_data(num_output); - for (int p = 0; p < num_output; p++) + int num_output_elempack = 1; +#if __ARM_NEON + if (opt.use_packing_layout) { - // dequantize - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_data[p] = scale_in; + num_output_elempack = num_output % 8 == 0 ? 8 : 1; } +#endif #if __ARM_NEON - if (elempack == 8) + if (num_output_elempack == 8 && out_elempack == 4) { #pragma omp parallel for num_threads(opt.num_threads) - for (int j = 0; j < h; j++) + for (int j = 0; j < outh; j++) { - float* outptr0 = top_blob.row(j * 2); - float* outptr1 = top_blob.row(j * 2 + 1); + float* outptr = top_blob.row(j); - for (int p = 0; p < num_output; p++) + for (int p = 0; p < num_output / num_output_elempack; p++) { - const signed char* kptr = (const signed char*)weight_data + num_input * p; - const signed char* m = bottom_blob_int8.row(j); - - int32x4_t _sum0 = vdupq_n_s32(0); - int32x4_t _sum1 = vdupq_n_s32(0); + const signed char* kptr = weight_data_tm.row(p); + const signed char* m0 = bottom_blob_int8_unpacked.row(j * 4); + const signed char* m1 = bottom_blob_int8_unpacked.row(j * 4 + 1); + const signed char* m2 = bottom_blob_int8_unpacked.row(j * 4 + 2); + const signed char* m3 = bottom_blob_int8_unpacked.row(j * 4 + 3); + + int32x4_t _sum00 = vdupq_n_s32(0); + int32x4_t _sum01 = vdupq_n_s32(0); + int32x4_t _sum10 = vdupq_n_s32(0); + int32x4_t _sum11 = vdupq_n_s32(0); + int32x4_t _sum20 = vdupq_n_s32(0); + int32x4_t _sum21 = vdupq_n_s32(0); + int32x4_t _sum30 = vdupq_n_s32(0); + int32x4_t _sum31 = vdupq_n_s32(0); int i = 0; - for (; i + 3 < num_input; i += 4) - { - int8x16_t _val0 = vld1q_s8(m); - int8x16_t _val1 = vld1q_s8(m + 16); - - int8x8_t _w0 = vdup_n_s8(kptr[0]); - int8x8_t _w1 = vdup_n_s8(kptr[1]); - int8x8_t _w2 = vdup_n_s8(kptr[2]); - int8x8_t _w3 = vdup_n_s8(kptr[3]); - - int16x8_t _s0 = vmull_s8(vget_low_s8(_val0), _w0); - int16x8_t _s1 = vmull_s8(vget_low_s8(_val1), _w2); - _s0 = vmlal_s8(_s0, vget_high_s8(_val0), _w1); - _s1 = vmlal_s8(_s1, vget_high_s8(_val1), _w3); - - _sum0 = vaddw_s16(_sum0, vget_low_s16(_s0)); - _sum1 = vaddw_s16(_sum1, vget_high_s16(_s0)); - _sum0 = vaddw_s16(_sum0, vget_low_s16(_s1)); - _sum1 = vaddw_s16(_sum1, vget_high_s16(_s1)); - - m += 32; - kptr += 4; - } - for (; i + 1 < num_input; i += 2) - { - int8x16_t _val0 = vld1q_s8(m); - - int8x8_t _w0 = vdup_n_s8(kptr[0]); - int8x8_t _w1 = vdup_n_s8(kptr[1]); - - int16x8_t _s0 = vmull_s8(vget_low_s8(_val0), _w0); - _s0 = vmlal_s8(_s0, vget_high_s8(_val0), _w1); - - _sum0 = vaddw_s16(_sum0, vget_low_s16(_s0)); - _sum1 = vaddw_s16(_sum1, vget_high_s16(_s0)); - - m += 16; - kptr += 2; - } for (; i < num_input; i++) { - int8x8_t _val = vld1_s8(m); - int8x8_t _w = vdup_n_s8(kptr[0]); + int8x8_t _val0 = vld1_dup_s8(m0); + int8x8_t _val1 = vld1_dup_s8(m1); + int8x8_t _val2 = vld1_dup_s8(m2); + int8x8_t _val3 = vld1_dup_s8(m3); - int16x8_t _s0 = vmull_s8(_val, _w); - _sum0 = vaddw_s16(_sum0, vget_low_s16(_s0)); - _sum1 = vaddw_s16(_sum1, vget_high_s16(_s0)); + int8x8_t _w = vld1_s8(kptr); - m += 8; - kptr += 1; + int16x8_t _s0 = vmull_s8(_val0, _w); + int16x8_t _s1 = vmull_s8(_val1, _w); + int16x8_t _s2 = vmull_s8(_val2, _w); + int16x8_t _s3 = vmull_s8(_val3, _w); + _sum00 = vaddw_s16(_sum00, vget_low_s16(_s0)); + _sum01 = vaddw_s16(_sum01, vget_high_s16(_s0)); + _sum10 = vaddw_s16(_sum10, vget_low_s16(_s1)); + _sum11 = vaddw_s16(_sum11, vget_high_s16(_s1)); + _sum20 = vaddw_s16(_sum20, vget_low_s16(_s2)); + _sum21 = vaddw_s16(_sum21, vget_high_s16(_s2)); + _sum30 = vaddw_s16(_sum30, vget_low_s16(_s3)); + _sum31 = vaddw_s16(_sum31, vget_high_s16(_s3)); + + m0++; + m1++; + m2++; + m3++; + kptr += 8; } // dequantize and relu - float32x4_t _scale_in = vdupq_n_f32(scale_data[p]); - - float32x4_t _sumfp32_0 = vcvtq_f32_s32(_sum0); - float32x4_t _sumfp32_1 = vcvtq_f32_s32(_sum1); + float32x4_t _scale_in0 = vld1q_f32((const float*)scale_in_data + p * 8); + float32x4_t _scale_in1 = vld1q_f32((const float*)scale_in_data + p * 8 + 4); + + float32x4_t _sumfp32_00 = vcvtq_f32_s32(_sum00); + float32x4_t _sumfp32_01 = vcvtq_f32_s32(_sum01); + float32x4_t _sumfp32_10 = vcvtq_f32_s32(_sum10); + float32x4_t _sumfp32_11 = vcvtq_f32_s32(_sum11); + float32x4_t _sumfp32_20 = vcvtq_f32_s32(_sum20); + float32x4_t _sumfp32_21 = vcvtq_f32_s32(_sum21); + float32x4_t _sumfp32_30 = vcvtq_f32_s32(_sum30); + float32x4_t _sumfp32_31 = vcvtq_f32_s32(_sum31); if (bias_term) { - float32x4_t _bias = vdupq_n_f32(bias_data[p]); - _sumfp32_0 = vmlaq_f32(_bias, _sumfp32_0, _scale_in); - _sumfp32_1 = vmlaq_f32(_bias, _sumfp32_1, _scale_in); + float32x4_t _bias0 = vld1q_f32((const float*)bias_data + p * 8); + float32x4_t _bias1 = vld1q_f32((const float*)bias_data + p * 8 + 4); + _sumfp32_00 = vmlaq_f32(_bias0, _sumfp32_00, _scale_in0); + _sumfp32_01 = vmlaq_f32(_bias1, _sumfp32_01, _scale_in1); + _sumfp32_10 = vmlaq_f32(_bias0, _sumfp32_10, _scale_in0); + _sumfp32_11 = vmlaq_f32(_bias1, _sumfp32_11, _scale_in1); + _sumfp32_20 = vmlaq_f32(_bias0, _sumfp32_20, _scale_in0); + _sumfp32_21 = vmlaq_f32(_bias1, _sumfp32_21, _scale_in1); + _sumfp32_30 = vmlaq_f32(_bias0, _sumfp32_30, _scale_in0); + _sumfp32_31 = vmlaq_f32(_bias1, _sumfp32_31, _scale_in1); } else { - _sumfp32_0 = vmulq_f32(_sumfp32_0, _scale_in); - _sumfp32_1 = vmulq_f32(_sumfp32_1, _scale_in); + _sumfp32_00 = vmulq_f32(_sumfp32_00, _scale_in0); + _sumfp32_01 = vmulq_f32(_sumfp32_01, _scale_in1); + _sumfp32_10 = vmulq_f32(_sumfp32_10, _scale_in0); + _sumfp32_11 = vmulq_f32(_sumfp32_11, _scale_in1); + _sumfp32_20 = vmulq_f32(_sumfp32_20, _scale_in0); + _sumfp32_21 = vmulq_f32(_sumfp32_21, _scale_in1); + _sumfp32_30 = vmulq_f32(_sumfp32_30, _scale_in0); + _sumfp32_31 = vmulq_f32(_sumfp32_31, _scale_in1); } - _sumfp32_0 = activation_ps(_sumfp32_0, activation_type, activation_params); - _sumfp32_1 = activation_ps(_sumfp32_1, activation_type, activation_params); + _sumfp32_00 = activation_ps(_sumfp32_00, activation_type, activation_params); + _sumfp32_01 = activation_ps(_sumfp32_01, activation_type, activation_params); + _sumfp32_10 = activation_ps(_sumfp32_10, activation_type, activation_params); + _sumfp32_11 = activation_ps(_sumfp32_11, activation_type, activation_params); + _sumfp32_20 = activation_ps(_sumfp32_20, activation_type, activation_params); + _sumfp32_21 = activation_ps(_sumfp32_21, activation_type, activation_params); + _sumfp32_30 = activation_ps(_sumfp32_30, activation_type, activation_params); + _sumfp32_31 = activation_ps(_sumfp32_31, activation_type, activation_params); + + // transpose 4x8 + float32x4x4_t _sumfp32_0; + _sumfp32_0.val[0] = _sumfp32_00; + _sumfp32_0.val[1] = _sumfp32_10; + _sumfp32_0.val[2] = _sumfp32_20; + _sumfp32_0.val[3] = _sumfp32_30; + float32x4x4_t _sumfp32_1; + _sumfp32_1.val[0] = _sumfp32_01; + _sumfp32_1.val[1] = _sumfp32_11; + _sumfp32_1.val[2] = _sumfp32_21; + _sumfp32_1.val[3] = _sumfp32_31; + + vst4q_f32(outptr, _sumfp32_0); + vst4q_f32(outptr + 16, _sumfp32_1); - vst1q_f32(outptr0, _sumfp32_0); - vst1q_f32(outptr1, _sumfp32_1); - outptr0 += 4; - outptr1 += 4; + outptr += 32; } } } - if (elempack == 1 && out_elempack == 4) + if (num_output_elempack == 1 && out_elempack == 4) { #pragma omp parallel for num_threads(opt.num_threads) for (int j = 0; j < outh; j++) @@ -2211,11 +2519,11 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co for (int p = 0; p < num_output; p++) { - const signed char* kptr = (const signed char*)weight_data + num_input * p; - const signed char* m0 = bottom_blob_int8.row(j * 4); - const signed char* m1 = bottom_blob_int8.row(j * 4 + 1); - const signed char* m2 = bottom_blob_int8.row(j * 4 + 2); - const signed char* m3 = bottom_blob_int8.row(j * 4 + 3); + const signed char* kptr = weight_data_tm.row(p); + const signed char* m0 = bottom_blob_int8_unpacked.row(j * 4); + const signed char* m1 = bottom_blob_int8_unpacked.row(j * 4 + 1); + const signed char* m2 = bottom_blob_int8_unpacked.row(j * 4 + 2); + const signed char* m3 = bottom_blob_int8_unpacked.row(j * 4 + 3); int sum0 = 0; int sum1 = 0; @@ -2282,10 +2590,10 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co } // dequantize and relu - float sumfp32_0 = sum0 * scale_data[p]; - float sumfp32_1 = sum1 * scale_data[p]; - float sumfp32_2 = sum2 * scale_data[p]; - float sumfp32_3 = sum3 * scale_data[p]; + float sumfp32_0 = sum0 * scale_in_data[p]; + float sumfp32_1 = sum1 * scale_in_data[p]; + float sumfp32_2 = sum2 * scale_in_data[p]; + float sumfp32_3 = sum3 * scale_in_data[p]; if (bias_term) { @@ -2303,9 +2611,91 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co } } } + + if (num_output_elempack == 8 && out_elempack == 1) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < outh; j++) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output / num_output_elempack; p++) + { + const signed char* kptr = weight_data_tm.row(p); + const signed char* m = bottom_blob_int8_unpacked.row(j); + + int32x4_t _sum0 = vdupq_n_s32(0); + int32x4_t _sum1 = vdupq_n_s32(0); + + int i = 0; + for (; i + 3 < num_input; i += 4) + { + int8x8_t _val0 = vdup_n_s8(m[0]); + int8x8_t _val1 = vdup_n_s8(m[1]); + int8x8_t _val2 = vdup_n_s8(m[2]); + int8x8_t _val3 = vdup_n_s8(m[3]); + + int8x16_t _w0 = vld1q_s8(kptr); + int8x16_t _w1 = vld1q_s8(kptr + 16); + + int16x8_t _s0 = vmull_s8(_val0, vget_low_s8(_w0)); + int16x8_t _s1 = vmull_s8(_val2, vget_low_s8(_w1)); + _s0 = vmlal_s8(_s0, _val1, vget_high_s8(_w0)); + _s1 = vmlal_s8(_s1, _val3, vget_high_s8(_w1)); + + _sum0 = vaddw_s16(_sum0, vget_low_s16(_s0)); + _sum1 = vaddw_s16(_sum1, vget_high_s16(_s0)); + _sum0 = vaddw_s16(_sum0, vget_low_s16(_s1)); + _sum1 = vaddw_s16(_sum1, vget_high_s16(_s1)); + + m += 4; + kptr += 32; + } + for (; i < num_input; i++) + { + int8x8_t _val = vld1_dup_s8(m); + int8x8_t _w = vld1_s8(kptr); + + int16x8_t _s0 = vmull_s8(_val, _w); + _sum0 = vaddw_s16(_sum0, vget_low_s16(_s0)); + _sum1 = vaddw_s16(_sum1, vget_high_s16(_s0)); + + m++; + kptr += 8; + } + + // dequantize and relu + float32x4_t _scale_in0 = vld1q_f32((const float*)scale_in_data + p * 8); + float32x4_t _scale_in1 = vld1q_f32((const float*)scale_in_data + p * 8 + 4); + + float32x4_t _sumfp32_0 = vcvtq_f32_s32(_sum0); + float32x4_t _sumfp32_1 = vcvtq_f32_s32(_sum1); + + if (bias_term) + { + float32x4_t _bias0 = vld1q_f32((const float*)bias_data + p * 8); + float32x4_t _bias1 = vld1q_f32((const float*)bias_data + p * 8 + 4); + _sumfp32_0 = vmlaq_f32(_bias0, _sumfp32_0, _scale_in0); + _sumfp32_1 = vmlaq_f32(_bias1, _sumfp32_1, _scale_in1); + } + else + { + _sumfp32_0 = vmulq_f32(_sumfp32_0, _scale_in0); + _sumfp32_1 = vmulq_f32(_sumfp32_1, _scale_in1); + } + + _sumfp32_0 = activation_ps(_sumfp32_0, activation_type, activation_params); + _sumfp32_1 = activation_ps(_sumfp32_1, activation_type, activation_params); + + vst1q_f32(outptr, _sumfp32_0); + vst1q_f32(outptr + 4, _sumfp32_1); + outptr += 8; + } + } + } #endif // __ARM_NEON - if (elempack == 1 && out_elempack == 1) + if (num_output_elempack == 1 && out_elempack == 1) { #pragma omp parallel for num_threads(opt.num_threads) for (int j = 0; j < outh; j++) @@ -2314,8 +2704,8 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co for (int p = 0; p < num_output; p++) { - const signed char* kptr = (const signed char*)weight_data + num_input * p; - const signed char* m = bottom_blob_int8.row(j); + const signed char* kptr = weight_data_tm.row(p); + const signed char* m = bottom_blob_int8_unpacked.row(j); int sum = 0; @@ -2351,7 +2741,7 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co } // dequantize and relu - float sumfp32 = sum * scale_data[p]; + float sumfp32 = sum * scale_in_data[p]; if (bias_term) sumfp32 += bias_data[p]; @@ -2387,11 +2777,6 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co if (top_blob.empty()) return -100; - Mat top_blob_int32; - top_blob_int32.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.workspace_allocator); - if (top_blob_int32.empty()) - return -100; - #if __ARM_NEON if (out_elempack == 8) { @@ -2399,7 +2784,7 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co #pragma omp parallel for num_threads(opt.num_threads) for (int p = 0; p < num_output / out_elempack; p++) { - const signed char* kptr = weight_data_int8.row(p); + const signed char* kptr = weight_data_tm.row(p); const signed char* sptr = bottom_blob_int8_flattened; int32x4_t _sum0 = vdupq_n_s32(0); @@ -2437,9 +2822,32 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co kptr += 8; } - int* outptr = (int*)top_blob_int32; - vst1q_s32(outptr + p * 8, _sum0); - vst1q_s32(outptr + p * 8 + 4, _sum1); + // dequantize and relu + float32x4_t _scale_in0 = vld1q_f32((const float*)scale_in_data + p * 8); + float32x4_t _scale_in1 = vld1q_f32((const float*)scale_in_data + p * 8 + 4); + + float32x4_t _sumfp32_0 = vcvtq_f32_s32(_sum0); + float32x4_t _sumfp32_1 = vcvtq_f32_s32(_sum1); + + if (bias_term) + { + float32x4_t _bias0 = vld1q_f32((const float*)bias_data + p * 8); + float32x4_t _bias1 = vld1q_f32((const float*)bias_data + p * 8 + 4); + _sumfp32_0 = vmlaq_f32(_bias0, _sumfp32_0, _scale_in0); + _sumfp32_1 = vmlaq_f32(_bias1, _sumfp32_1, _scale_in1); + } + else + { + _sumfp32_0 = vmulq_f32(_sumfp32_0, _scale_in0); + _sumfp32_1 = vmulq_f32(_sumfp32_1, _scale_in1); + } + + _sumfp32_0 = activation_ps(_sumfp32_0, activation_type, activation_params); + _sumfp32_1 = activation_ps(_sumfp32_1, activation_type, activation_params); + + float* outptr = (float*)top_blob + p * 8; + vst1q_f32(outptr, _sumfp32_0); + vst1q_f32(outptr + 4, _sumfp32_1); } } #endif // __ARM_NEON @@ -2450,7 +2858,7 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co #pragma omp parallel for num_threads(opt.num_threads) for (int p = 0; p < num_output / out_elempack; p++) { - const signed char* kptr = weight_data_int8.row(p); + const signed char* kptr = weight_data_tm.row(p); const signed char* sptr = bottom_blob_int8_flattened; int sum = 0; @@ -2468,29 +2876,16 @@ int InnerProduct_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, co kptr += 1; } - int* outptr = (int*)top_blob_int32; - outptr[p] = sum; - } - } - - Mat scale_data(num_output); - for (int p = 0; p < num_output; p++) - { - // dequantize - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); + // dequantize and relu + float sumfp32 = sum * scale_in_data[p]; - scale_data[p] = scale_in; - } + if (bias_term) + sumfp32 += bias_data[p]; - dequantize_from_int32(top_blob_int32, top_blob, scale_data, bias_data, opt); + sumfp32 = activation_ss(sumfp32, activation_type, activation_params); - if (activation) - { - activation->forward_inplace(top_blob, opt); + top_blob[p] = sumfp32; + } } return 0; diff --git a/src/layer/arm/innerproduct_arm.h b/src/layer/arm/innerproduct_arm.h index ba772486b..f18eb9e1e 100644 --- a/src/layer/arm/innerproduct_arm.h +++ b/src/layer/arm/innerproduct_arm.h @@ -48,21 +48,14 @@ protected: public: Layer* flatten; - Layer* activation; + + Mat weight_data_tm; // fp16 - Mat weight_data_fp16; Mat bias_data_fp16; -#if NCNN_BF16 - // bf16 - Mat weight_data_bf16; -#endif - #if NCNN_INT8 - // int8 - Mat weight_data_int8; - Mat scales_in; + Mat scale_in_data; #endif }; diff --git a/src/layer/vulkan/convolution_vulkan.cpp b/src/layer/vulkan/convolution_vulkan.cpp index f83c5024a..63701629a 100644 --- a/src/layer/vulkan/convolution_vulkan.cpp +++ b/src/layer/vulkan/convolution_vulkan.cpp @@ -39,7 +39,8 @@ Convolution_vulkan::Convolution_vulkan() pipeline_convolution_3x3s1d1_winograd43_gemm = 0; pipeline_convolution_3x3s1d1_winograd43_transform_output = 0; - innerproduct = 0; + reshape_1x1xw = 0; + reshape_w = 0; } int Convolution_vulkan::create_pipeline(const Option& _opt) @@ -58,41 +59,6 @@ int Convolution_vulkan::create_pipeline(const Option& _opt) const int maxk = kernel_w * kernel_h; int num_input = weight_data_size / maxk / num_output; - // fc - if (kernel_w == 1 && kernel_h == 1) - { - innerproduct = ncnn::create_layer(ncnn::LayerType::InnerProduct); - innerproduct->vkdev = vkdev; - - innerproduct->bottom_shapes.resize(1); - innerproduct->bottom_shapes[0] = shape; - innerproduct->top_shapes.resize(1); - innerproduct->top_shapes[0] = out_shape; - - ncnn::ParamDict pd; - pd.set(0, num_output); - pd.set(1, bias_term); - pd.set(2, weight_data_size); // TODO int8 - pd.set(9, activation_type); - pd.set(10, activation_params); - - innerproduct->load_param(pd); - - ncnn::Mat weights[2]; - weights[0] = weight_data; - weights[1] = bias_data; - ncnn::ModelBinFromMatArray mb(weights); - - innerproduct->load_model(mb); - - innerproduct->create_pipeline(opt); - - if (shape.dims == 1 && shape.w == num_input) - { - return 0; - } - } - // the shape after padding Mat shape_bordered; if (shape.dims != 0) @@ -147,6 +113,46 @@ int Convolution_vulkan::create_pipeline(const Option& _opt) Mat out_shape_packed; if (out_shape.dims == 3) out_shape_packed = Mat(out_shape.w, out_shape.h, num_output / out_elempack, (void*)0, out_elemsize, out_elempack); + // fc + if (kernel_w == 1 && kernel_h == 1) + { + { + reshape_1x1xw = ncnn::create_layer(ncnn::LayerType::Reshape); + reshape_1x1xw->vkdev = vkdev; + + reshape_1x1xw->bottom_shapes.resize(1); + reshape_1x1xw->bottom_shapes[0] = Mat(num_input, (void*)0); + reshape_1x1xw->top_shapes.resize(1); + reshape_1x1xw->top_shapes[0] = Mat(1, 1, num_input, (void*)0); + + ncnn::ParamDict pd; + pd.set(0, 1); // w + pd.set(1, 1); // h + pd.set(2, num_input); // c + + reshape_1x1xw->load_param(pd); + + reshape_1x1xw->create_pipeline(opt); + } + + { + reshape_w = ncnn::create_layer(ncnn::LayerType::Reshape); + reshape_w->vkdev = vkdev; + + reshape_w->bottom_shapes.resize(1); + reshape_w->bottom_shapes[0] = Mat(1, 1, num_output, (void*)0); + reshape_w->top_shapes.resize(1); + reshape_w->top_shapes[0] = Mat(num_output, (void*)0); + + ncnn::ParamDict pd; + pd.set(0, num_output); // w + + reshape_w->load_param(pd); + + reshape_w->create_pipeline(opt); + } + } + bool is_conv1x1s1d1 = kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1; bool is_conv3x3s1d1 = kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1; @@ -176,6 +182,85 @@ int Convolution_vulkan::create_pipeline(const Option& _opt) { bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; + // winograd43 transform kernel + { + Mat weight_data_tm; + weight_data_tm.create(6 * 6, num_input, num_output); + + const float ktm[6][3] = { + {1.0f, 0.0f, 0.0f}, + {-2.0f / 3, -2.0f / 3, -2.0f / 3}, + {-2.0f / 3, 2.0f / 3, -2.0f / 3}, + {1.0f / 6, 1.0f / 3, 2.0f / 3}, + {1.0f / 6, -1.0f / 3, 2.0f / 3}, + {0.0f, 0.0f, 4.0f} + }; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output; p++) + { + for (int q = 0; q < num_input; q++) + { + const float* kernel0 = (const float*)weight_data + p * num_input * 9 + q * 9; + float* kernel_tm0 = weight_data_tm.channel(p).row(q); + + // transform kernel + const float* k0 = kernel0; + const float* k1 = kernel0 + 3; + const float* k2 = kernel0 + 6; + + // h + float tmp[6][3]; + for (int i = 0; i < 6; i++) + { + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; + } + + // U + for (int j = 0; j < 6; j++) + { + float* tmpp = &tmp[j][0]; + + for (int i = 0; i < 6; i++) + { + kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; + } + } + } + } + + // src = 36-inch-outch + // dst = 8a-8b-inch/8a-outch/8b-36 + { + weight_winograd43_data_packed.create(num_input / elempack, num_output / out_elempack, 36, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + + for (int k = 0; k < 36; k++) + { + float* g00 = weight_winograd43_data_packed.channel(k); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) + { + for (int i = 0; i < out_elempack; i++) + { + const Mat k0 = weight_data_tm.channel(q + i); + + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + } + // winograd43 { int block_x = 0; @@ -306,6 +391,84 @@ int Convolution_vulkan::create_pipeline(const Option& _opt) } } + // winograd23 transform kernel + { + Mat weight_data_tm; + weight_data_tm.create(4 * 4, num_input, num_output); + + // G + const float ktm[4][3] = { + {1.0f, 0.0f, 0.0f}, + {1.0f / 2, 1.0f / 2, 1.0f / 2}, + {1.0f / 2, -1.0f / 2, 1.0f / 2}, + {0.0f, 0.0f, 1.0f} + }; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < num_output; p++) + { + for (int q = 0; q < num_input; q++) + { + const float* kernel0 = (const float*)weight_data + p * num_input * 9 + q * 9; + float* kernel_tm0 = weight_data_tm.channel(p).row(q); + + // transform kernel + const float* k0 = kernel0; + const float* k1 = kernel0 + 3; + const float* k2 = kernel0 + 6; + + // h + float tmp[4][3]; + for (int i = 0; i < 4; i++) + { + tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; + tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; + tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; + } + + // U + for (int j = 0; j < 4; j++) + { + float* tmpp = &tmp[j][0]; + + for (int i = 0; i < 4; i++) + { + kernel_tm0[j * 4 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; + } + } + } + } + + // src = 16-inch-outch + // dst = 8a-8b-inch/8a-outch/8b-16 + { + weight_winograd23_data_packed.create(num_input / elempack, num_output / out_elempack, 16, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + + for (int k = 0; k < 16; k++) + { + float* g00 = weight_winograd23_data_packed.channel(k); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) + { + for (int i = 0; i < out_elempack; i++) + { + const Mat k0 = weight_data_tm.channel(q + i); + + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + } + // winograd23 { int block_x = 0; @@ -436,275 +599,66 @@ int Convolution_vulkan::create_pipeline(const Option& _opt) } } } - if (opt.use_sgemm_convolution && !is_conv1x1s1d1 && num_input >= 16 && num_output >= 16) + else { - bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - - // check blob shape - if (!vkdev->shape_support_image_storage(shape_bordered_packed) || !vkdev->shape_support_image_storage(out_shape_packed)) + // src = kw-kh-inch-outch + // dst = pa-pb-kw-kh-inch/pa-outch/pb + if (opt.use_sgemm_convolution && !is_conv1x1s1d1 && num_input >= 16 && num_output >= 16) { - support_image_storage = false; - opt.use_image_storage = false; - } + bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; + if (use_cooperative_matrix) + { + // dst = 8b-8a-maxk-inch/8a-outch/8b + // dst = 16b-16a-maxk-inch/16a-outch/16b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - // check weight shape - Mat weight_data_packed(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - if (!vkdev->shape_support_image_storage(weight_data_packed)) - { - support_image_storage = false; - opt.use_image_storage = false; - } + weight_data_packed.create(maxk * num_input / 8, num_output / 8, (size_t)4 * 8 * 8, 8 * 8); - std::vector specializations(10 + 8); - specializations[0].i = kernel_w; - specializations[1].i = kernel_h; - specializations[2].i = dilation_w; - specializations[3].i = dilation_h; - specializations[4].i = stride_w; - specializations[5].i = stride_h; - specializations[6].i = bias_term; - specializations[7].i = activation_type; - specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f; - specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f; - specializations[10 + 0].i = shape_bordered_packed.w; - specializations[10 + 1].i = shape_bordered_packed.h; - specializations[10 + 2].i = shape_bordered_packed.c; - specializations[10 + 3].i = shape_bordered_packed.cstep; - specializations[10 + 4].i = out_shape_packed.w; - specializations[10 + 5].i = out_shape_packed.h; - specializations[10 + 6].i = out_shape_packed.c; - specializations[10 + 7].i = out_shape_packed.cstep; + for (int q = 0; q + 7 < num_output; q += 8) + { + float* g00 = weight_data_packed.row(q / 8); - Mat local_size_xyz(16, std::min(4, num_output / out_elempack), 1, (void*)0); - if (out_shape_packed.dims != 0) - { - local_size_xyz.w = std::min(16, out_shape_packed.w * out_shape_packed.h); - local_size_xyz.h = std::min(4, out_shape_packed.c); - } + for (int p = 0; p + 7 < num_input; p += 8) + { + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < 8; i++) + { + for (int j = 0; j < 8; j++) + { + const float* k00 = weight_data_r2.channel(q + j).row(p + i); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + else + { + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - int shader_type_index = -1; - if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_gemm; - if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4_gemm; - if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4_gemm; - if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1_gemm; - if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8_gemm; - if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8_gemm; - if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1_gemm; - if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8_gemm; - if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4_gemm; + weight_data_packed.create(maxk * num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - if (use_cooperative_matrix) - { - shader_type_index = LayerShaderType::convolution_pack4_gemm_cm_16_8_8; - } - - pipeline_convolution_gemm = new Pipeline(vkdev); - if (use_cooperative_matrix) - { - // TODO proper unroll y - pipeline_convolution_gemm->set_local_size_xyz(32, 4, 1); // 16_8_8 ly*4 - } - else if (opt.use_shader_local_memory) - { - pipeline_convolution_gemm->set_local_size_xyz(8, 8, 1); - } - else - { - pipeline_convolution_gemm->set_optimal_local_size_xyz(local_size_xyz); - } - pipeline_convolution_gemm->create(shader_type_index, opt, specializations); - } - if (is_conv1x1s1d1) - { - bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - - std::vector specializations(4 + 8); - specializations[0].i = bias_term; - specializations[1].i = activation_type; - specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f; - specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f; - specializations[4 + 0].i = shape_bordered_packed.w; - specializations[4 + 1].i = shape_bordered_packed.h; - specializations[4 + 2].i = shape_bordered_packed.c; - specializations[4 + 3].i = shape_bordered_packed.cstep; - specializations[4 + 4].i = out_shape_packed.w; - specializations[4 + 5].i = out_shape_packed.h; - specializations[4 + 6].i = out_shape_packed.c; - specializations[4 + 7].i = out_shape_packed.cstep; - - int shader_type_index = -1; - if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_1x1s1d1; - if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4_1x1s1d1; - if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4_1x1s1d1; - if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1_1x1s1d1; - if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8_1x1s1d1; - if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8_1x1s1d1; - if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1_1x1s1d1; - if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8_1x1s1d1; - if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4_1x1s1d1; - - if (use_cooperative_matrix) - { - shader_type_index = LayerShaderType::convolution_pack4_1x1s1d1_cm_16_8_8; - } - - pipeline_convolution_1x1s1d1 = new Pipeline(vkdev); - if (use_cooperative_matrix) - { - // TODO proper unroll y - pipeline_convolution_1x1s1d1->set_local_size_xyz(32, 4, 1); // 16_8_8 ly*4 - } - else if (opt.use_shader_local_memory) - { - pipeline_convolution_1x1s1d1->set_local_size_xyz(8, 8, 1); - } - else - { - pipeline_convolution_1x1s1d1->set_local_size_xyz(8, std::min(8, num_output / out_elempack), 1); - } - pipeline_convolution_1x1s1d1->create(shader_type_index, opt, specializations); - } - else - { - std::vector specializations(10 + 10); - specializations[0].i = kernel_w; - specializations[1].i = kernel_h; - specializations[2].i = dilation_w; - specializations[3].i = dilation_h; - specializations[4].i = stride_w; - specializations[5].i = stride_h; - specializations[6].i = bias_term; - specializations[7].i = activation_type; - specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f; - specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f; - specializations[10 + 0].i = shape_bordered_packed.dims; - specializations[10 + 1].i = shape_bordered_packed.w; - specializations[10 + 2].i = shape_bordered_packed.h; - specializations[10 + 3].i = shape_bordered_packed.c; - specializations[10 + 4].i = shape_bordered_packed.cstep; - specializations[10 + 5].i = out_shape_packed.dims; - specializations[10 + 6].i = out_shape_packed.w; - specializations[10 + 7].i = out_shape_packed.h; - specializations[10 + 8].i = out_shape_packed.c; - specializations[10 + 9].i = out_shape_packed.cstep; - - Mat local_size_xyz(8, 8, std::min(4, (num_output / out_elempack + 1) / 2), (void*)0); - if (out_shape_packed.dims != 0) - { - local_size_xyz.w = std::min(8, out_shape_packed.w); - local_size_xyz.h = std::min(8, out_shape_packed.h); - local_size_xyz.c = std::min(4, (out_shape_packed.c + 1) / 2); - } - - int shader_type_index = -1; - if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution; - if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4; - if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4; - if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1; - if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8; - if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8; - if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1; - if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8; - if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4; - - pipeline_convolution = new Pipeline(vkdev); - pipeline_convolution->set_optimal_local_size_xyz(local_size_xyz); - pipeline_convolution->create(shader_type_index, opt, specializations); - } - - return 0; -} - -int Convolution_vulkan::destroy_pipeline(const Option& opt) -{ - if (padding) - { - padding->destroy_pipeline(opt); - delete padding; - padding = 0; - } - - delete pipeline_convolution; - pipeline_convolution = 0; - - delete pipeline_convolution_1x1s1d1; - pipeline_convolution_1x1s1d1 = 0; - - delete pipeline_convolution_gemm; - pipeline_convolution_gemm = 0; - - delete pipeline_convolution_3x3s1d1_winograd23_transform_input; - delete pipeline_convolution_3x3s1d1_winograd23_gemm; - delete pipeline_convolution_3x3s1d1_winograd23_transform_output; - pipeline_convolution_3x3s1d1_winograd23_transform_input = 0; - pipeline_convolution_3x3s1d1_winograd23_gemm = 0; - pipeline_convolution_3x3s1d1_winograd23_transform_output = 0; - - delete pipeline_convolution_3x3s1d1_winograd43_transform_input; - delete pipeline_convolution_3x3s1d1_winograd43_gemm; - delete pipeline_convolution_3x3s1d1_winograd43_transform_output; - pipeline_convolution_3x3s1d1_winograd43_transform_input = 0; - pipeline_convolution_3x3s1d1_winograd43_gemm = 0; - pipeline_convolution_3x3s1d1_winograd43_transform_output = 0; - - // fc - if (innerproduct) - { - innerproduct->destroy_pipeline(opt); - delete innerproduct; - innerproduct = 0; - } - - return 0; -} - -int Convolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) -{ - if (padding) - { - padding->upload_model(cmd, opt); - } - - const int maxk = kernel_w * kernel_h; - int num_input = weight_data_size / maxk / num_output; - - int elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1; - int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - - bool is_conv1x1s1d1 = kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1; - bool is_conv3x3s1d1 = kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1; - - // src = kw-kh-inch-outch - // dst = pa-pb-kw-kh-inch/pa-outch/pb - Mat weight_data_packed; - if (opt.use_sgemm_convolution && !is_conv1x1s1d1 && num_input >= 16 && num_output >= 16) - { - bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - if (use_cooperative_matrix) - { - // dst = 8b-8a-maxk-inch/8a-outch/8b - // dst = 16b-16a-maxk-inch/16a-outch/16b - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_packed.create(maxk * num_input / 8, num_output / 8, (size_t)4 * 8 * 8, 8 * 8); - - for (int q = 0; q + 7 < num_output; q += 8) - { - float* g00 = weight_data_packed.row(q / 8); - - for (int p = 0; p + 7 < num_input; p += 8) + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - for (int k = 0; k < maxk; k++) + float* g00 = weight_data_packed.row(q / out_elempack); + + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { - for (int i = 0; i < 8; i++) + for (int k = 0; k < maxk; k++) { - for (int j = 0; j < 8; j++) + for (int i = 0; i < out_elempack; i++) { - const float* k00 = weight_data_r2.channel(q + j).row(p + i); - - g00[0] = k00[k]; + const Mat k0 = weight_data_r2.channel(q + i); - g00++; + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } } } } @@ -713,95 +667,60 @@ int Convolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) } else { - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_packed.create(maxk * num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && is_conv1x1s1d1 && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; + if (use_cooperative_matrix) { - float* g00 = weight_data_packed.row(q / out_elempack); - - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) - { - for (int k = 0; k < maxk; k++) - { - for (int i = 0; i < out_elempack; i++) - { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack; j++) - { - const float* k00 = k0.row(p + j); - - g00[0] = k00[k]; + // dst = 8b-8a-inch/8a-outch/8b + // dst = 16b-16a-inch/16a-outch/16b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - g00++; - } - } - } - } - } - } - } - else - { - bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && is_conv1x1s1d1 && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - if (use_cooperative_matrix) - { - // dst = 8b-8a-inch/8a-outch/8b - // dst = 16b-16a-inch/16a-outch/16b - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_packed.create(maxk, num_input / 8, num_output / 8, (size_t)4 * 8 * 8, 8 * 8); - - for (int q = 0; q + 7 < num_output; q += 8) - { - float* g00 = weight_data_packed.channel(q / 8); + weight_data_packed.create(maxk, num_input / 8, num_output / 8, (size_t)4 * 8 * 8, 8 * 8); - for (int p = 0; p + 7 < num_input; p += 8) + for (int q = 0; q + 7 < num_output; q += 8) { - for (int k = 0; k < maxk; k++) + float* g00 = weight_data_packed.channel(q / 8); + + for (int p = 0; p + 7 < num_input; p += 8) { - for (int i = 0; i < 8; i++) + for (int k = 0; k < maxk; k++) { - for (int j = 0; j < 8; j++) + for (int i = 0; i < 8; i++) { - const float* k00 = weight_data_r2.channel(q + j).row(p + i); - - g00[0] = k00[k]; - - g00++; + for (int j = 0; j < 8; j++) + { + const float* k00 = weight_data_r2.channel(q + j).row(p + i); + g00[0] = k00[k]; + g00++; + } } } } } } - } - else - { - Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - - weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + else { - float* g00 = weight_data_packed.channel(q / out_elempack); + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - for (int k = 0; k < maxk; k++) + float* g00 = weight_data_packed.channel(q / out_elempack); + + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { - for (int i = 0; i < out_elempack; i++) + for (int k = 0; k < maxk; k++) { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack; j++) + for (int i = 0; i < out_elempack; i++) { - const float* k00 = k0.row(p + j); + const Mat k0 = weight_data_r2.channel(q + i); - g00[0] = k00[k]; - - g00++; + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } } } } @@ -810,266 +729,298 @@ int Convolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) } } - if (support_image_storage && opt.use_image_storage) - { - cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); - } - else + if (bias_term) { - cmd.record_upload(weight_data_packed, weight_data_gpu, opt); + convert_packing(bias_data, bias_data_packed, out_elempack, opt); } - if (opt.use_winograd_convolution && is_conv3x3s1d1 && num_input >= 16 && num_output >= 16) + if (opt.use_sgemm_convolution && !is_conv1x1s1d1 && num_input >= 16 && num_output >= 16) { bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - // winograd43 transform kernel + // check blob shape + if (!vkdev->shape_support_image_storage(shape_bordered_packed) || !vkdev->shape_support_image_storage(out_shape_packed)) { - Mat weight_data_tm; - weight_data_tm.create(6 * 6, num_input, num_output); - - const float ktm[6][3] = { - {1.0f, 0.0f, 0.0f}, - {-2.0f / 3, -2.0f / 3, -2.0f / 3}, - {-2.0f / 3, 2.0f / 3, -2.0f / 3}, - {1.0f / 6, 1.0f / 3, 2.0f / 3}, - {1.0f / 6, -1.0f / 3, 2.0f / 3}, - {0.0f, 0.0f, 4.0f} - }; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - for (int q = 0; q < num_input; q++) - { - const float* kernel0 = (const float*)weight_data + p * num_input * 9 + q * 9; - float* kernel_tm0 = weight_data_tm.channel(p).row(q); - - // transform kernel - const float* k0 = kernel0; - const float* k1 = kernel0 + 3; - const float* k2 = kernel0 + 6; - - // h - float tmp[6][3]; - for (int i = 0; i < 6; i++) - { - tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; - tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; - tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; - } - - // U - for (int j = 0; j < 6; j++) - { - float* tmpp = &tmp[j][0]; - - for (int i = 0; i < 6; i++) - { - kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; - } - } - } - } - - // src = 36-inch-outch - // dst = 8a-8b-inch/8a-outch/8b-36 - Mat weight_data_tm_packed; - if (use_cooperative_matrix) - { - // dst = 8b-8a-inch/8a-outch/8b-36 - // dst = 16b-16a-inch/16a-outch/16b-36 - weight_data_tm_packed.create(num_input / 8, num_output / 8, 36, (size_t)4 * 8 * 8, 8 * 8); + support_image_storage = false; + opt.use_image_storage = false; + } - for (int k = 0; k < 36; k++) - { - float* g00 = weight_data_tm_packed.channel(k); + // check weight shape + Mat weight_data_packed_shape(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + if (!vkdev->shape_support_image_storage(weight_data_packed_shape)) + { + support_image_storage = false; + opt.use_image_storage = false; + } - for (int q = 0; q + (8 - 1) < num_output; q += 8) - { - for (int p = 0; p + (8 - 1) < num_input; p += 8) - { - for (int i = 0; i < 8; i++) - { - for (int j = 0; j < 8; j++) - { - const float* k00 = weight_data_tm.channel(q + j).row(p + i); + std::vector specializations(10 + 8); + specializations[0].i = kernel_w; + specializations[1].i = kernel_h; + specializations[2].i = dilation_w; + specializations[3].i = dilation_h; + specializations[4].i = stride_w; + specializations[5].i = stride_h; + specializations[6].i = bias_term; + specializations[7].i = activation_type; + specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f; + specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f; + specializations[10 + 0].i = shape_bordered_packed.w; + specializations[10 + 1].i = shape_bordered_packed.h; + specializations[10 + 2].i = shape_bordered_packed.c; + specializations[10 + 3].i = shape_bordered_packed.cstep; + specializations[10 + 4].i = out_shape_packed.w; + specializations[10 + 5].i = out_shape_packed.h; + specializations[10 + 6].i = out_shape_packed.c; + specializations[10 + 7].i = out_shape_packed.cstep; - g00[0] = k00[k]; + Mat local_size_xyz(16, std::min(4, num_output / out_elempack), 1, (void*)0); + if (out_shape_packed.dims != 0) + { + local_size_xyz.w = std::min(16, out_shape_packed.w * out_shape_packed.h); + local_size_xyz.h = std::min(4, out_shape_packed.c); + } - g00++; - } - } - } - } - } - } - else - { - weight_data_tm_packed.create(num_input / elempack, num_output / out_elempack, 36, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + int shader_type_index = -1; + if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_gemm; + if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4_gemm; + if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4_gemm; + if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1_gemm; + if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8_gemm; + if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8_gemm; + if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1_gemm; + if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8_gemm; + if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4_gemm; - for (int k = 0; k < 36; k++) - { - float* g00 = weight_data_tm_packed.channel(k); + if (use_cooperative_matrix) + { + shader_type_index = LayerShaderType::convolution_pack4_gemm_cm_16_8_8; + } - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) - { - for (int i = 0; i < out_elempack; i++) - { - const Mat k0 = weight_data_tm.channel(q + i); + pipeline_convolution_gemm = new Pipeline(vkdev); + if (use_cooperative_matrix) + { + // TODO proper unroll y + pipeline_convolution_gemm->set_local_size_xyz(32, 4, 1); // 16_8_8 ly*4 + } + else if (opt.use_shader_local_memory) + { + pipeline_convolution_gemm->set_local_size_xyz(8, 8, 1); + } + else + { + pipeline_convolution_gemm->set_optimal_local_size_xyz(local_size_xyz); + } + pipeline_convolution_gemm->create(shader_type_index, opt, specializations); + } + if (is_conv1x1s1d1) + { + bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - for (int j = 0; j < elempack; j++) - { - const float* k00 = k0.row(p + j); + std::vector specializations(4 + 8); + specializations[0].i = bias_term; + specializations[1].i = activation_type; + specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f; + specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f; + specializations[4 + 0].i = shape_bordered_packed.w; + specializations[4 + 1].i = shape_bordered_packed.h; + specializations[4 + 2].i = shape_bordered_packed.c; + specializations[4 + 3].i = shape_bordered_packed.cstep; + specializations[4 + 4].i = out_shape_packed.w; + specializations[4 + 5].i = out_shape_packed.h; + specializations[4 + 6].i = out_shape_packed.c; + specializations[4 + 7].i = out_shape_packed.cstep; - g00[0] = k00[k]; + int shader_type_index = -1; + if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_1x1s1d1; + if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4_1x1s1d1; + if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4_1x1s1d1; + if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1_1x1s1d1; + if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8_1x1s1d1; + if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8_1x1s1d1; + if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1_1x1s1d1; + if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8_1x1s1d1; + if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4_1x1s1d1; - g00++; - } - } - } - } - } - } + if (use_cooperative_matrix) + { + shader_type_index = LayerShaderType::convolution_pack4_1x1s1d1_cm_16_8_8; + } - if (support_image_storage && opt.use_image_storage) - { - cmd.record_upload(weight_data_tm_packed, weight_data_gpu_tm_winograd43_image, opt); - } - else - { - cmd.record_upload(weight_data_tm_packed, weight_data_gpu_tm_winograd43, opt); - } + pipeline_convolution_1x1s1d1 = new Pipeline(vkdev); + if (use_cooperative_matrix) + { + // TODO proper unroll y + pipeline_convolution_1x1s1d1->set_local_size_xyz(32, 4, 1); // 16_8_8 ly*4 + } + else if (opt.use_shader_local_memory) + { + pipeline_convolution_1x1s1d1->set_local_size_xyz(8, 8, 1); + } + else + { + pipeline_convolution_1x1s1d1->set_local_size_xyz(8, std::min(8, num_output / out_elempack), 1); } + pipeline_convolution_1x1s1d1->create(shader_type_index, opt, specializations); + } + else + { + std::vector specializations(10 + 10); + specializations[0].i = kernel_w; + specializations[1].i = kernel_h; + specializations[2].i = dilation_w; + specializations[3].i = dilation_h; + specializations[4].i = stride_w; + specializations[5].i = stride_h; + specializations[6].i = bias_term; + specializations[7].i = activation_type; + specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f; + specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f; + specializations[10 + 0].i = shape_bordered_packed.dims; + specializations[10 + 1].i = shape_bordered_packed.w; + specializations[10 + 2].i = shape_bordered_packed.h; + specializations[10 + 3].i = shape_bordered_packed.c; + specializations[10 + 4].i = shape_bordered_packed.cstep; + specializations[10 + 5].i = out_shape_packed.dims; + specializations[10 + 6].i = out_shape_packed.w; + specializations[10 + 7].i = out_shape_packed.h; + specializations[10 + 8].i = out_shape_packed.c; + specializations[10 + 9].i = out_shape_packed.cstep; - // winograd23 transform kernel + Mat local_size_xyz(8, 8, std::min(4, (num_output / out_elempack + 1) / 2), (void*)0); + if (out_shape_packed.dims != 0) { - Mat weight_data_tm; - weight_data_tm.create(4 * 4, num_input, num_output); + local_size_xyz.w = std::min(8, out_shape_packed.w); + local_size_xyz.h = std::min(8, out_shape_packed.h); + local_size_xyz.c = std::min(4, (out_shape_packed.c + 1) / 2); + } - // G - const float ktm[4][3] = { - {1.0f, 0.0f, 0.0f}, - {1.0f / 2, 1.0f / 2, 1.0f / 2}, - {1.0f / 2, -1.0f / 2, 1.0f / 2}, - {0.0f, 0.0f, 1.0f} - }; + int shader_type_index = -1; + if (elempack == 1 && out_elempack == 1) shader_type_index = LayerShaderType::convolution; + if (elempack == 4 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack4; + if (elempack == 1 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack1to4; + if (elempack == 4 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack4to1; + if (elempack == 8 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack8; + if (elempack == 1 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack1to8; + if (elempack == 8 && out_elempack == 1) shader_type_index = LayerShaderType::convolution_pack8to1; + if (elempack == 4 && out_elempack == 8) shader_type_index = LayerShaderType::convolution_pack4to8; + if (elempack == 8 && out_elempack == 4) shader_type_index = LayerShaderType::convolution_pack8to4; - #pragma omp parallel for num_threads(opt.num_threads) - for (int p = 0; p < num_output; p++) - { - for (int q = 0; q < num_input; q++) - { - const float* kernel0 = (const float*)weight_data + p * num_input * 9 + q * 9; - float* kernel_tm0 = weight_data_tm.channel(p).row(q); + pipeline_convolution = new Pipeline(vkdev); + pipeline_convolution->set_optimal_local_size_xyz(local_size_xyz); + pipeline_convolution->create(shader_type_index, opt, specializations); + } - // transform kernel - const float* k0 = kernel0; - const float* k1 = kernel0 + 3; - const float* k2 = kernel0 + 6; + return 0; +} - // h - float tmp[4][3]; - for (int i = 0; i < 4; i++) - { - tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2]; - tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2]; - tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2]; - } +int Convolution_vulkan::destroy_pipeline(const Option& opt) +{ + if (padding) + { + padding->destroy_pipeline(opt); + delete padding; + padding = 0; + } - // U - for (int j = 0; j < 4; j++) - { - float* tmpp = &tmp[j][0]; + delete pipeline_convolution; + pipeline_convolution = 0; - for (int i = 0; i < 4; i++) - { - kernel_tm0[j * 4 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2]; - } - } - } - } + delete pipeline_convolution_1x1s1d1; + pipeline_convolution_1x1s1d1 = 0; - // src = 16-inch-outch - // dst = 8a-8b-inch/8a-outch/8b-16 - Mat weight_data_tm_packed; - if (use_cooperative_matrix) - { - // dst = 8b-8a-inch/8a-outch/8b-16 - // dst = 16b-16a-inch/16a-outch/16b-36 - weight_data_tm_packed.create(num_input / 8, num_output / 8, 16, (size_t)4 * 8 * 8, 8 * 8); + delete pipeline_convolution_gemm; + pipeline_convolution_gemm = 0; - for (int k = 0; k < 16; k++) - { - float* g00 = weight_data_tm_packed.channel(k); + delete pipeline_convolution_3x3s1d1_winograd23_transform_input; + delete pipeline_convolution_3x3s1d1_winograd23_gemm; + delete pipeline_convolution_3x3s1d1_winograd23_transform_output; + pipeline_convolution_3x3s1d1_winograd23_transform_input = 0; + pipeline_convolution_3x3s1d1_winograd23_gemm = 0; + pipeline_convolution_3x3s1d1_winograd23_transform_output = 0; - for (int q = 0; q + (8 - 1) < num_output; q += 8) - { - for (int p = 0; p + (8 - 1) < num_input; p += 8) - { - for (int i = 0; i < 8; i++) - { - for (int j = 0; j < 8; j++) - { - const float* k00 = weight_data_tm.channel(q + j).row(p + i); + delete pipeline_convolution_3x3s1d1_winograd43_transform_input; + delete pipeline_convolution_3x3s1d1_winograd43_gemm; + delete pipeline_convolution_3x3s1d1_winograd43_transform_output; + pipeline_convolution_3x3s1d1_winograd43_transform_input = 0; + pipeline_convolution_3x3s1d1_winograd43_gemm = 0; + pipeline_convolution_3x3s1d1_winograd43_transform_output = 0; - g00[0] = k00[k]; + // fc + if (reshape_1x1xw) + { + reshape_1x1xw->destroy_pipeline(opt); + delete reshape_1x1xw; + reshape_1x1xw = 0; + } - g00++; - } - } - } - } - } - } - else - { - weight_data_tm_packed.create(num_input / elempack, num_output / out_elempack, 16, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + if (reshape_w) + { + reshape_w->destroy_pipeline(opt); + delete reshape_w; + reshape_w = 0; + } - for (int k = 0; k < 16; k++) - { - float* g00 = weight_data_tm_packed.channel(k); + return 0; +} - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) - { - for (int i = 0; i < out_elempack; i++) - { - const Mat k0 = weight_data_tm.channel(q + i); +int Convolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) +{ + if (padding) + { + padding->upload_model(cmd, opt); + } - for (int j = 0; j < elempack; j++) - { - const float* k00 = k0.row(p + j); + const int maxk = kernel_w * kernel_h; + int num_input = weight_data_size / maxk / num_output; - g00[0] = k00[k]; + bool is_conv3x3s1d1 = kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1; - g00++; - } - } - } - } - } + if (opt.use_winograd_convolution && is_conv3x3s1d1 && num_input >= 16 && num_output >= 16) + { + // winograd43 + { + if (support_image_storage && opt.use_image_storage) + { + cmd.record_upload(weight_winograd43_data_packed, weight_data_gpu_tm_winograd43_image, opt); } + else + { + cmd.record_upload(weight_winograd43_data_packed, weight_data_gpu_tm_winograd43, opt); + } + + weight_winograd43_data_packed.release(); + } + // winograd23 + { if (support_image_storage && opt.use_image_storage) { - cmd.record_upload(weight_data_tm_packed, weight_data_gpu_tm_winograd23_image, opt); + cmd.record_upload(weight_winograd23_data_packed, weight_data_gpu_tm_winograd23_image, opt); } else { - cmd.record_upload(weight_data_tm_packed, weight_data_gpu_tm_winograd23, opt); + cmd.record_upload(weight_winograd23_data_packed, weight_data_gpu_tm_winograd23, opt); } + + weight_winograd23_data_packed.release(); + } + } + else + { + if (support_image_storage && opt.use_image_storage) + { + cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); + } + else + { + cmd.record_upload(weight_data_packed, weight_data_gpu, opt); } + + weight_data_packed.release(); } if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -1078,11 +1029,8 @@ int Convolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } - } - if (innerproduct) - { - innerproduct->upload_model(cmd, opt); + bias_data_packed.release(); } return 0; @@ -1102,7 +1050,22 @@ int Convolution_vulkan::forward(const VkMat& bottom_blob, VkMat& top_blob, VkCom int num_input = weight_data_size / num_output; if (bottom_blob.w * bottom_blob.elempack == num_input) { - return innerproduct->forward(bottom_blob, top_blob, cmd, opt); + VkMat bottom_blob_1x1xw; + { + Option opt_reshape = opt; + opt_reshape.blob_vkallocator = opt.workspace_vkallocator; + reshape_1x1xw->forward(bottom_blob, bottom_blob_1x1xw, cmd, opt_reshape); + } + + if (bottom_blob_1x1xw.empty()) + return -100; + + VkMat top_blob_1x1xw; + int ret = forward(bottom_blob_1x1xw, top_blob_1x1xw, cmd, opt); + if (ret != 0) + return ret; + + return reshape_w->forward(top_blob_1x1xw, top_blob, cmd, opt); } } @@ -1527,7 +1490,22 @@ int Convolution_vulkan::forward(const VkImageMat& bottom_blob, VkImageMat& top_b int num_input = weight_data_size / num_output; if (bottom_blob.w * bottom_blob.elempack == num_input) { - return innerproduct->forward(bottom_blob, top_blob, cmd, opt); + VkImageMat bottom_blob_1x1xw; + { + Option opt_reshape = opt; + opt_reshape.blob_vkallocator = opt.workspace_vkallocator; + reshape_1x1xw->forward(bottom_blob, bottom_blob_1x1xw, cmd, opt_reshape); + } + + if (bottom_blob_1x1xw.empty()) + return -100; + + VkImageMat top_blob_1x1xw; + int ret = forward(bottom_blob_1x1xw, top_blob_1x1xw, cmd, opt); + if (ret != 0) + return ret; + + return reshape_w->forward(top_blob_1x1xw, top_blob, cmd, opt); } } diff --git a/src/layer/vulkan/convolution_vulkan.h b/src/layer/vulkan/convolution_vulkan.h index 5ebfd0e23..0efa76fec 100644 --- a/src/layer/vulkan/convolution_vulkan.h +++ b/src/layer/vulkan/convolution_vulkan.h @@ -36,6 +36,11 @@ public: public: ncnn::Layer* padding; + Mat weight_data_packed; + Mat weight_winograd23_data_packed; + Mat weight_winograd43_data_packed; + Mat bias_data_packed; + VkMat weight_data_gpu; VkMat bias_data_gpu; @@ -61,7 +66,8 @@ public: Pipeline* pipeline_convolution_3x3s1d1_winograd43_transform_output; // convolution as fc - ncnn::Layer* innerproduct; + ncnn::Layer* reshape_1x1xw; + ncnn::Layer* reshape_w; }; } // namespace ncnn diff --git a/src/layer/vulkan/convolutiondepthwise_vulkan.cpp b/src/layer/vulkan/convolutiondepthwise_vulkan.cpp index 556fc48a3..57069074c 100644 --- a/src/layer/vulkan/convolutiondepthwise_vulkan.cpp +++ b/src/layer/vulkan/convolutiondepthwise_vulkan.cpp @@ -214,6 +214,14 @@ int ConvolutionDepthWise_vulkan::create_pipeline(const Option& _opt) // depth-wise if (channels == group && group == num_output) { + Mat weight_data_r2 = weight_data.reshape(maxk, group); + convert_packing(weight_data_r2, weight_data_packed, elempack, opt); + + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack, opt); + } + specializations[11 + 0].i = shape_bordered_packed.dims; specializations[11 + 1].i = shape_bordered_packed.w; specializations[11 + 2].i = shape_bordered_packed.h; @@ -260,6 +268,51 @@ int ConvolutionDepthWise_vulkan::create_pipeline(const Option& _opt) return 0; } + // src = kw-kh-inch-outch + // dst = pa-pb-kw-kh-inch/pa-outch/pb + { + Mat weight_data_r2_groups = weight_data.reshape(maxk, channels_g, num_output_g * group); + + weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g); + + for (int g = 0; g < group; g++) + { + const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g); + + Mat weight_data_packed = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g); + + for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g) + { + float* g00 = weight_data_packed.channel(q / out_elempack_g); + + for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g) + { + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < out_elempack_g; i++) + { + const Mat k0 = weight_data_r2.channel(q + i); + + for (int j = 0; j < elempack_g; j++) + { + const float* k00 = k0.row(p + j); + + g00[0] = k00[k]; + + g00++; + } + } + } + } + } + } + } + + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack_g, opt); + } + specializations[11 + 0].i = shape_bordered_g_packed.dims; specializations[11 + 1].i = shape_bordered_g_packed.w; specializations[11 + 2].i = shape_bordered_g_packed.h; @@ -412,16 +465,9 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt const int maxk = kernel_w * kernel_h; int channels = (weight_data_size / group) / maxk / (num_output / group) * group; - int elempack = opt.use_shader_pack8 && channels % 8 == 0 ? 8 : channels % 4 == 0 ? 4 : 1; - int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - // depth-wise if (channels == group && group == num_output) { - Mat weight_data_packed; - Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, elempack, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); @@ -431,11 +477,10 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt cmd.record_upload(weight_data_packed, weight_data_gpu, opt); } + weight_data_packed.release(); + if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -444,59 +489,13 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } + + bias_data_packed.release(); } return 0; } - // group convolution - const int channels_g = channels / group; - const int num_output_g = num_output / group; - - int elempack_g = opt.use_shader_pack8 && channels_g % 8 == 0 ? 8 : channels_g % 4 == 0 ? 4 : 1; - int out_elempack_g = opt.use_shader_pack8 && num_output_g % 8 == 0 ? 8 : num_output_g % 4 == 0 ? 4 : 1; - - // src = kw-kh-inch-outch - // dst = pa-pb-kw-kh-inch/pa-outch/pb - Mat weight_data_packed_groups; - { - Mat weight_data_r2_groups = weight_data.reshape(maxk, channels_g, num_output_g * group); - - weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g); - - for (int g = 0; g < group; g++) - { - const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g); - - Mat weight_data_packed = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g); - - for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g) - { - float* g00 = weight_data_packed.channel(q / out_elempack_g); - - for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g) - { - for (int k = 0; k < maxk; k++) - { - for (int i = 0; i < out_elempack_g; i++) - { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack_g; j++) - { - const float* k00 = k0.row(p + j); - - g00[0] = k00[k]; - - g00++; - } - } - } - } - } - } - } - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(weight_data_packed_groups, weight_data_gpu_image, opt); @@ -506,11 +505,10 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt cmd.record_upload(weight_data_packed_groups, weight_data_gpu, opt); } + weight_data_packed_groups.release(); + if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack_g, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -519,6 +517,8 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } + + bias_data_packed.release(); } return 0; diff --git a/src/layer/vulkan/convolutiondepthwise_vulkan.h b/src/layer/vulkan/convolutiondepthwise_vulkan.h index 77a793207..3689e369c 100644 --- a/src/layer/vulkan/convolutiondepthwise_vulkan.h +++ b/src/layer/vulkan/convolutiondepthwise_vulkan.h @@ -34,6 +34,10 @@ public: virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const; public: + Mat weight_data_packed; + Mat weight_data_packed_groups; + Mat bias_data_packed; + VkMat weight_data_gpu; VkMat bias_data_gpu; diff --git a/src/layer/vulkan/deconvolution_vulkan.cpp b/src/layer/vulkan/deconvolution_vulkan.cpp index d391af647..2480c854b 100644 --- a/src/layer/vulkan/deconvolution_vulkan.cpp +++ b/src/layer/vulkan/deconvolution_vulkan.cpp @@ -94,8 +94,8 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt) } // check weight shape - Mat weight_data_packed(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - if (!vkdev->shape_support_image_storage(weight_data_packed)) + Mat weight_data_packed_shape(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + if (!vkdev->shape_support_image_storage(weight_data_packed_shape)) { support_image_storage = false; opt.use_image_storage = false; @@ -139,10 +139,76 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt) output_crop->create_pipeline(opt); } + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack, opt); + } + if (opt.use_sgemm_convolution) { bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; + // src = kw-kh-inch-outch + // dst = pa-pb-inch/pa-kw-kh-outch/pb (sgemm) + if (use_cooperative_matrix) + { + // dst = 8a-8b-inch/8a-maxk-outch/8b + // dst = 16a-16b-inch/16a-maxk-outch/16b + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_packed.create(num_input / 8, maxk * num_output / 8, (size_t)4 * 8 * 8, 8 * 8); + + for (int q = 0; q + 7 < num_output; q += 8) + { + for (int k = 0; k < maxk; k++) + { + float* g00 = weight_data_packed.row(q / 8 * maxk + k); + + for (int p = 0; p + 7 < num_input; p += 8) + { + for (int i = 0; i < 8; i++) + { + for (int j = 0; j < 8; j++) + { + const float* k00 = weight_data_r2.channel(q + j).row(p + i); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + else + { + Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); + + weight_data_packed.create(num_input / elempack, maxk * num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + for (int k = 0; k < maxk; k++) + { + float* g00 = weight_data_packed.row(q / out_elempack * maxk + k); + + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) + { + for (int i = 0; i < out_elempack; i++) + { + const Mat k0 = weight_data_r2.channel(q + i); + + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + Mat out_shape_col; if (shape.dims != 0 && out_shape.dims != 0) { @@ -249,6 +315,54 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt) return 0; } + Mat weight_data_transposed(weight_data.w); + { + float* pt = weight_data_transposed; + const float* p = weight_data; + + for (int i = 0; i < num_input * num_output; i++) + { + for (int k = 0; k < maxk; k++) + { + pt[maxk - 1 - k] = p[k]; + } + + p += maxk; + pt += maxk; + } + } + + // src = kw-kh-inch-outch + // dst = pa-pb-kw-kh-inch/pa-outch/pb + { + Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); + + weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + float* g00 = weight_data_packed.channel(q / out_elempack); + + for (int p = 0; p + (elempack - 1) < num_input; p += elempack) + { + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < out_elempack; i++) + { + const Mat k0 = weight_data_r2.channel(q + i); + + for (int j = 0; j < elempack; j++) + { + const float* k00 = k0.row(p + j); + g00[0] = k00[k]; + g00++; + } + } + } + } + } + } + std::vector specializations(10 + 10); specializations[0].i = kernel_w; specializations[1].i = kernel_h; @@ -337,136 +451,6 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) output_crop->upload_model(cmd, opt); } - const int maxk = kernel_w * kernel_h; - int num_input = weight_data_size / maxk / num_output; - - int elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1; - int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - - Mat weight_data_transposed(weight_data.w); - if (opt.use_sgemm_convolution) - { - weight_data_transposed = weight_data; - } - else - { - float* pt = weight_data_transposed; - const float* p = weight_data; - - for (int i = 0; i < num_input * num_output; i++) - { - for (int k = 0; k < maxk; k++) - { - pt[maxk - 1 - k] = p[k]; - } - - p += maxk; - pt += maxk; - } - } - - // src = kw-kh-inch-outch - // dst = pa-pb-kw-kh-inch/pa-outch/pb - // dst = pa-pb-inch/pa-kw-kh-outch/pb (sgemm) - Mat weight_data_packed; - if (opt.use_sgemm_convolution) - { - bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0; - if (use_cooperative_matrix) - { - // dst = 8a-8b-inch/8a-maxk-outch/8b - // dst = 16a-16b-inch/16a-maxk-outch/16b - Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - - weight_data_packed.create(num_input / 8, maxk * num_output / 8, (size_t)4 * 8 * 8, 8 * 8); - - for (int q = 0; q + 7 < num_output; q += 8) - { - for (int k = 0; k < maxk; k++) - { - float* g00 = weight_data_packed.row(q / 8 * maxk + k); - - for (int p = 0; p + 7 < num_input; p += 8) - { - for (int i = 0; i < 8; i++) - { - for (int j = 0; j < 8; j++) - { - const float* k00 = weight_data_r2.channel(q + j).row(p + i); - - g00[0] = k00[k]; - - g00++; - } - } - } - } - } - } - else - { - Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - - weight_data_packed.create(num_input / elempack, maxk * num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - for (int k = 0; k < maxk; k++) - { - float* g00 = weight_data_packed.row(q / out_elempack * maxk + k); - - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) - { - for (int i = 0; i < out_elempack; i++) - { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack; j++) - { - const float* k00 = k0.row(p + j); - - g00[0] = k00[k]; - - g00++; - } - } - } - } - } - } - } - else - { - Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - - weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - float* g00 = weight_data_packed.channel(q / out_elempack); - - for (int p = 0; p + (elempack - 1) < num_input; p += elempack) - { - for (int k = 0; k < maxk; k++) - { - for (int i = 0; i < out_elempack; i++) - { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack; j++) - { - const float* k00 = k0.row(p + j); - - g00[0] = k00[k]; - - g00++; - } - } - } - } - } - } - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); @@ -476,11 +460,10 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) cmd.record_upload(weight_data_packed, weight_data_gpu, opt); } + weight_data_packed.release(); + if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -489,6 +472,8 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt) { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } + + bias_data_packed.release(); } return 0; diff --git a/src/layer/vulkan/deconvolution_vulkan.h b/src/layer/vulkan/deconvolution_vulkan.h index e8cbaa28f..578bdc967 100644 --- a/src/layer/vulkan/deconvolution_vulkan.h +++ b/src/layer/vulkan/deconvolution_vulkan.h @@ -34,6 +34,9 @@ public: virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const; public: + Mat weight_data_packed; + Mat bias_data_packed; + VkMat weight_data_gpu; VkMat bias_data_gpu; diff --git a/src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp b/src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp index 3821e521f..ee9d949d3 100644 --- a/src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp +++ b/src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp @@ -198,6 +198,23 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt) output_crop->create_pipeline(opt); } + Mat weight_data_transposed(weight_data.w); + { + float* pt = weight_data_transposed; + const float* p = weight_data; + + for (int i = 0; i < (channels / group) * (num_output / group) * group; i++) + { + for (int k = 0; k < maxk; k++) + { + pt[maxk - 1 - k] = p[k]; + } + + p += maxk; + pt += maxk; + } + } + std::vector specializations(11 + 10); specializations[0].i = kernel_w; specializations[1].i = kernel_h; @@ -214,6 +231,14 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt) // depth-wise if (channels == group && group == num_output) { + Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); + convert_packing(weight_data_r2, weight_data_packed, elempack, opt); + + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack, opt); + } + specializations[11 + 0].i = shape_packed.dims; specializations[11 + 1].i = shape_packed.w; specializations[11 + 2].i = shape_packed.h; @@ -260,6 +285,51 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt) return 0; } + // src = kw-kh-inch-outch + // dst = pa-pb-kw-kh-inch/pa-outch/pb + { + Mat weight_data_r2_groups = weight_data_transposed.reshape(maxk, channels_g, num_output_g * group); + + weight_data_packed.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g); + + for (int g = 0; g < group; g++) + { + const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g); + + Mat weight_data_pack4 = weight_data_packed.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g); + + for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g) + { + float* g00 = weight_data_pack4.channel(q / out_elempack_g); + + for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g) + { + for (int k = 0; k < maxk; k++) + { + for (int i = 0; i < out_elempack_g; i++) + { + const Mat k0 = weight_data_r2.channel(q + i); + + for (int j = 0; j < elempack_g; j++) + { + const float* k00 = k0.row(p + j); + + g00[0] = k00[k]; + + g00++; + } + } + } + } + } + } + } + + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack_g, opt); + } + specializations[11 + 0].i = shape_g_packed.dims; specializations[11 + 1].i = shape_g_packed.w; specializations[11 + 2].i = shape_g_packed.h; @@ -421,125 +491,19 @@ int DeconvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& o output_crop->upload_model(cmd, opt); } - const int maxk = kernel_w * kernel_h; - int channels = (weight_data_size / group) / maxk / (num_output / group) * group; - - int elempack = opt.use_shader_pack8 && channels % 8 == 0 ? 8 : channels % 4 == 0 ? 4 : 1; - int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - - Mat weight_data_transposed(weight_data.w); - { - float* pt = weight_data_transposed; - const float* p = weight_data; - - for (int i = 0; i < (channels / group) * (num_output / group) * group; i++) - { - for (int k = 0; k < maxk; k++) - { - pt[maxk - 1 - k] = p[k]; - } - - p += maxk; - pt += maxk; - } - } - - // depth-wise - if (channels == group && group == num_output) - { - Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - Mat weight_data_r2_packed; - convert_packing(weight_data_r2, weight_data_r2_packed, elempack, opt); - - if (support_image_storage && opt.use_image_storage) - { - cmd.record_upload(weight_data_r2_packed, weight_data_gpu_image, opt); - } - else - { - cmd.record_upload(weight_data_r2_packed, weight_data_gpu, opt); - } - - if (bias_term) - { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack, opt); - - if (support_image_storage && opt.use_image_storage) - { - cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); - } - else - { - cmd.record_upload(bias_data_packed, bias_data_gpu, opt); - } - } - - return 0; - } - - // group deconvolution - const int channels_g = channels / group; - const int num_output_g = num_output / group; - - int elempack_g = opt.use_shader_pack8 && channels_g % 8 == 0 ? 8 : channels_g % 4 == 0 ? 4 : 1; - int out_elempack_g = opt.use_shader_pack8 && num_output_g % 8 == 0 ? 8 : num_output_g % 4 == 0 ? 4 : 1; - - // src = kw-kh-inch-outch - // dst = pa-pb-kw-kh-inch/pa-outch/pb - Mat weight_data_packed_groups; - { - Mat weight_data_r2_groups = weight_data_transposed.reshape(maxk, channels_g, num_output_g * group); - - weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g); - - for (int g = 0; g < group; g++) - { - const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g); - - Mat weight_data_pack4 = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g); - - for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g) - { - float* g00 = weight_data_pack4.channel(q / out_elempack_g); - - for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g) - { - for (int k = 0; k < maxk; k++) - { - for (int i = 0; i < out_elempack_g; i++) - { - const Mat k0 = weight_data_r2.channel(q + i); - - for (int j = 0; j < elempack_g; j++) - { - const float* k00 = k0.row(p + j); - - g00[0] = k00[k]; - - g00++; - } - } - } - } - } - } - } - if (support_image_storage && opt.use_image_storage) { - cmd.record_upload(weight_data_packed_groups, weight_data_gpu_image, opt); + cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); } else { - cmd.record_upload(weight_data_packed_groups, weight_data_gpu, opt); + cmd.record_upload(weight_data_packed, weight_data_gpu, opt); } + weight_data_packed.release(); + if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack_g, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -548,6 +512,8 @@ int DeconvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& o { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } + + bias_data_packed.release(); } return 0; diff --git a/src/layer/vulkan/deconvolutiondepthwise_vulkan.h b/src/layer/vulkan/deconvolutiondepthwise_vulkan.h index 1adad8601..bf38f254e 100644 --- a/src/layer/vulkan/deconvolutiondepthwise_vulkan.h +++ b/src/layer/vulkan/deconvolutiondepthwise_vulkan.h @@ -34,6 +34,9 @@ public: virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const; public: + Mat weight_data_packed; + Mat bias_data_packed; + VkMat weight_data_gpu; VkMat bias_data_gpu; diff --git a/src/layer/vulkan/innerproduct_vulkan.cpp b/src/layer/vulkan/innerproduct_vulkan.cpp index 92d4ffd7b..9cb57e5d8 100644 --- a/src/layer/vulkan/innerproduct_vulkan.cpp +++ b/src/layer/vulkan/innerproduct_vulkan.cpp @@ -45,6 +45,40 @@ int InnerProduct_vulkan::create_pipeline(const Option& _opt) int in_elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1; int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; + // src = inch-outch + // dst = pa-pb-inch/pa-outch/pb + { + Mat weight_data_r2 = weight_data.reshape(num_input, num_output); + + weight_data_packed.create(num_input / in_elempack, num_output / out_elempack, (size_t)4 * in_elempack * out_elempack, in_elempack * out_elempack); + + for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) + { + float* g00 = weight_data_packed.row(q / out_elempack); + + for (int p = 0; p + (in_elempack - 1) < num_input; p += in_elempack) + { + for (int i = 0; i < out_elempack; i++) + { + const float* k0 = weight_data_r2.row(q + i); + k0 += p; + + for (int j = 0; j < in_elempack; j++) + { + g00[0] = k0[j]; + + g00++; + } + } + } + } + } + + if (bias_term) + { + convert_packing(bias_data, bias_data_packed, out_elempack, opt); + } + if (shape.dims == 2 && shape.w == num_input && shape.h > 1) { // gemm @@ -358,41 +392,6 @@ int InnerProduct_vulkan::destroy_pipeline(const Option& opt) int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt) { - const int num_input = weight_data_size / num_output; - - int in_elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1; - int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1; - - // src = inch-outch - // dst = pa-pb-inch/pa-outch/pb - Mat weight_data_packed; - { - Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - - weight_data_packed.create(num_input / in_elempack, num_output / out_elempack, (size_t)4 * in_elempack * out_elempack, in_elempack * out_elempack); - - for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) - { - float* g00 = weight_data_packed.row(q / out_elempack); - - for (int p = 0; p + (in_elempack - 1) < num_input; p += in_elempack) - { - for (int i = 0; i < out_elempack; i++) - { - const float* k0 = weight_data_r2.row(q + i); - k0 += p; - - for (int j = 0; j < in_elempack; j++) - { - g00[0] = k0[j]; - - g00++; - } - } - } - } - } - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt); @@ -402,11 +401,10 @@ int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt) cmd.record_upload(weight_data_packed, weight_data_gpu, opt); } + weight_data_packed.release(); + if (bias_term) { - Mat bias_data_packed; - convert_packing(bias_data, bias_data_packed, out_elempack, opt); - if (support_image_storage && opt.use_image_storage) { cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt); @@ -415,6 +413,8 @@ int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt) { cmd.record_upload(bias_data_packed, bias_data_gpu, opt); } + + bias_data_packed.release(); } return 0; diff --git a/src/layer/vulkan/innerproduct_vulkan.h b/src/layer/vulkan/innerproduct_vulkan.h index c54a28467..4fe138d48 100644 --- a/src/layer/vulkan/innerproduct_vulkan.h +++ b/src/layer/vulkan/innerproduct_vulkan.h @@ -36,6 +36,9 @@ public: public: ncnn::Layer* flatten; + Mat weight_data_packed; + Mat bias_data_packed; + VkMat weight_data_gpu; VkMat bias_data_gpu; diff --git a/src/layer/x86/convolution_1x1.h b/src/layer/x86/convolution_1x1.h index acf0f8c0c..0f286ff41 100644 --- a/src/layer/x86/convolution_1x1.h +++ b/src/layer/x86/convolution_1x1.h @@ -205,3 +205,54 @@ static void conv1x1s2_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& _ker } } } + +static void conv1x1s1_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + const int size = w * h; + + Mat bottom_im2col = bottom_blob; + bottom_im2col.w = size; + bottom_im2col.h = 1; + + im2col_sgemm_sse(bottom_im2col, top_blob, kernel, _bias, opt); +} + +static void conv1x1s2_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt) +{ + int w = bottom_blob.w; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + + int outw = top_blob.w; + int outh = top_blob.h; + + const int tailstep = w - 2 * outw + w; + + Mat bottom_blob_shrinked; + bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < channels; p++) + { + const float* r0 = bottom_blob.channel(p); + float* outptr = bottom_blob_shrinked.channel(p); + + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + outptr[0] = r0[0]; + + r0 += 2; + outptr += 1; + } + + r0 += tailstep; + } + } + + conv1x1s1_sgemm_sse(bottom_blob_shrinked, top_blob, kernel, _bias, opt); +} diff --git a/src/layer/x86/convolution_x86.cpp b/src/layer/x86/convolution_x86.cpp index 58078641c..c10236ee3 100644 --- a/src/layer/x86/convolution_x86.cpp +++ b/src/layer/x86/convolution_x86.cpp @@ -140,7 +140,7 @@ Convolution_x86::Convolution_x86() convolution_dilation1 = 0; } -static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat& weight_data_packed, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack) +static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack) { const int maxk = kernel_w * kernel_h; @@ -149,11 +149,11 @@ static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat& { Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output); - weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack); + weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - float* g00 = weight_data_packed.channel(q / out_elempack); + float* g00 = weight_data_tm.channel(q / out_elempack); for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { @@ -232,6 +232,11 @@ int Convolution_x86::create_pipeline(const Option& opt) convolution_dilation1->create_pipeline(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -278,7 +283,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -298,7 +303,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -318,7 +323,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -338,7 +343,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -358,7 +363,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -378,7 +383,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -400,7 +405,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } else if (opt.use_sgemm_convolution) @@ -409,7 +414,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -435,12 +440,12 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } else if (kernel_w == 2 && kernel_h == 2 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } else if (opt.use_sgemm_convolution) { @@ -448,7 +453,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -469,7 +474,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -486,11 +491,11 @@ int Convolution_x86::create_pipeline(const Option& opt) } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } else if (opt.use_sgemm_convolution) { @@ -498,7 +503,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -519,7 +524,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } @@ -542,7 +547,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } else if (opt.use_sgemm_convolution) @@ -551,7 +556,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } #endif // __AVX__ @@ -584,7 +589,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } } @@ -602,11 +607,11 @@ int Convolution_x86::create_pipeline(const Option& opt) } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } else { @@ -620,7 +625,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } } @@ -652,7 +657,7 @@ int Convolution_x86::create_pipeline(const Option& opt) } else { - convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); + convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack); } } } @@ -661,7 +666,15 @@ int Convolution_x86::create_pipeline(const Option& opt) // pack1 if (elempack == 1 && out_elempack == 1) { - if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); + } + else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { if (num_input >= 16 && num_output >= 16) { @@ -676,6 +689,15 @@ int Convolution_x86::create_pipeline(const Option& opt) { convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else + { + weight_data_tm = weight_data; + } + } + + if (opt.lightmode) + { + weight_data.release(); } return 0; @@ -702,19 +724,51 @@ int Convolution_x86::destroy_pipeline(const Option& opt) int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { - // convolv with NxN kernel - // value = value + bias - #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_x86(bottom_blob, top_blob, opt); } #endif - if (bottom_blob.dims != 3) + // flattened blob, implement as InnerProduct + if (bottom_blob.dims == 1 && kernel_w == 1 && kernel_h == 1) { - return Convolution::forward(bottom_blob, top_blob, opt); + Mat bottom_blob_3d; + if (bottom_blob.elemsize % 16 == 0) + { + bottom_blob_3d = bottom_blob; + bottom_blob_3d.dims = 3; + bottom_blob_3d.w = 1; + bottom_blob_3d.h = 1; + bottom_blob_3d.c = bottom_blob.w; + bottom_blob_3d.cstep = 1; + } + else + { + bottom_blob_3d = bottom_blob.reshape(1, 1, bottom_blob.w, opt.workspace_allocator); + } + + Mat top_blob_3d; + int ret = forward(bottom_blob_3d, top_blob_3d, opt); + if (ret != 0) + return ret; + + if (top_blob_3d.elemsize % 16 == 0) + { + top_blob = top_blob_3d; + top_blob.dims = 1; + top_blob.w = top_blob_3d.c; + top_blob.h = 1; + top_blob.c = 1; + bottom_blob_3d.cstep = top_blob_3d.c; + } + else + { + top_blob = top_blob_3d.reshape(top_blob_3d.c, opt.blob_allocator); + } + + return 0; } int w = bottom_blob.w; @@ -816,7 +870,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -851,7 +905,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack8to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack8to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -886,7 +940,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack16to8_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack16to8_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -921,7 +975,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -956,7 +1010,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack16to4_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack16to4_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -991,7 +1045,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack1to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack1to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1024,7 +1078,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv3x3s1_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -1043,7 +1097,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1085,7 +1139,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -1095,7 +1149,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 2 && kernel_h == 2 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv2x2s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv2x2s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1113,7 +1167,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1139,7 +1193,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1148,7 +1202,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s2_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1166,7 +1220,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1201,7 +1255,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1234,7 +1288,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - conv3x3s1_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); } if (activation) @@ -1253,7 +1307,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } @@ -1288,7 +1342,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack8to4_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack8to4_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __AVX__ @@ -1347,7 +1401,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } } @@ -1374,7 +1428,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - conv3x3s1_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s1_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1383,7 +1437,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + conv3x3s2_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1407,7 +1461,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } } @@ -1437,7 +1491,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option // TODO more proper condition conv3x3s1_winograd63_pack4to1_sse(bottom_blob_bordered, top_blob, weight_winograd63_data, bias_data, opt); - // conv3x3s1_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + // conv3x3s1_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -1461,7 +1515,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else { - convolution_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + convolution_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } } @@ -1469,7 +1523,25 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option if (elempack == 1 && out_elempack == 1) { - if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + conv1x1s1_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + conv1x1s2_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { if (num_input >= 16 && num_output >= 16) { @@ -1533,7 +1605,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option sum = bias_data[p]; } - const float* kptr = (const float*)weight_data + maxk * channels * p; + const float* kptr = (const float*)weight_data_tm + maxk * channels * p; // channels for (int q = 0; q < channels; q++) @@ -1640,7 +1712,7 @@ int Convolution_x86::forward(const std::vector& bottom_blobs, std::vector= 16 && num_output >= 16) - { - conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_winograd23_data, num_input, num_output, opt); - // conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_winograd43_data, num_input, num_output, opt); - } - if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); @@ -1789,12 +1855,37 @@ int Convolution_x86::create_pipeline_int8_x86(const Option& opt) { convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) + { + conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_winograd23_data, num_input, num_output, opt); + // conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_winograd43_data, num_input, num_output, opt); + } else if (opt.use_sgemm_convolution) { convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h); } + else + { + weight_data_tm = weight_data; + } + } - return 0; + scale_in_data.create(num_output); + for (int p = 0; p < num_output; p++) + { + // requantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); + + scale_in_data[p] = scale_in; + } + + if (opt.lightmode) + { + weight_data.release(); } return 0; @@ -1885,34 +1976,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con } else { - convolution_pack8to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + convolution_pack8to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } } @@ -1944,34 +2008,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con } else { - convolution_pack1to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + convolution_pack1to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } } @@ -1995,34 +2032,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con } else { - convolution_pack8to1_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - } - - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } - - if (use_int8_requantize) - { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + convolution_pack8to1_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } } #endif // __SSE2__ @@ -2048,35 +2058,21 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con } else { - // convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); - convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); + convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt); } + } - Mat scale_in_data(num_output); - for (int p = 0; p < num_output; p++) - { - // requantize and relu - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_in_data[p] = scale_in; - } + if (use_int8_requantize) + { + requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); + } + else + { + dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - if (use_int8_requantize) + if (activation) { - requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt); - } - else - { - dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + activation->forward_inplace(top_blob, opt); } } diff --git a/src/layer/x86/convolution_x86.h b/src/layer/x86/convolution_x86.h index e5763dde9..0ec675a8d 100644 --- a/src/layer/x86/convolution_x86.h +++ b/src/layer/x86/convolution_x86.h @@ -41,6 +41,7 @@ protected: public: Layer* activation; + Mat weight_data_tm; Mat weight_sgemm_data; Mat weight_winograd23_data; Mat weight_winograd43_data; @@ -49,12 +50,8 @@ public: // forwardDilation Layer* convolution_dilation1; - // pack4/8 - Mat weight_data_packed; - #if NCNN_INT8 - // int8 - Mat weight_data_int8; + Mat scale_in_data; #endif }; diff --git a/src/layer/x86/convolutiondepthwise_x86.cpp b/src/layer/x86/convolutiondepthwise_x86.cpp index bd5ff1ffe..b0c6eb37d 100644 --- a/src/layer/x86/convolutiondepthwise_x86.cpp +++ b/src/layer/x86/convolutiondepthwise_x86.cpp @@ -95,7 +95,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 16) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 16, opt); + convert_packing(weight_data_r2, weight_data_tm, 16, opt); return 0; } @@ -105,7 +105,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 8) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 8, opt); + convert_packing(weight_data_r2, weight_data_tm, 8, opt); return 0; } @@ -115,7 +115,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 4, opt); + convert_packing(weight_data_r2, weight_data_tm, 4, opt); return 0; } @@ -126,10 +126,12 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) // depth-wise specific if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { + weight_data_tm = weight_data; return 0; } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { + weight_data_tm = weight_data; return 0; } } @@ -138,6 +140,11 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) // group convolution create_group_ops(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -159,7 +166,7 @@ int ConvolutionDepthWise_x86::create_group_ops(const Option& opt) for (int g = 0; g < group; g++) { - Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g); + Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone(); Mat bias_data_g; if (bias_term) bias_data_g = bias_data.range(num_output_g * g, num_output_g); @@ -260,7 +267,7 @@ int ConvolutionDepthWise_x86::destroy_pipeline(const Option& opt) int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_x86(bottom_blob, top_blob, opt); } @@ -314,7 +321,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -325,7 +332,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -336,7 +343,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -347,7 +354,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -383,7 +390,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con for (int g = 0; g < channels; g++) { float* outptr = top_blob.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 16; + const float* kptr = (const float*)weight_data_tm + maxk * g * 16; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -425,7 +432,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -436,7 +443,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -447,7 +454,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -458,7 +465,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -494,7 +501,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con for (int g = 0; g < channels; g++) { float* outptr = top_blob.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 8; + const float* kptr = (const float*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -538,7 +545,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -549,7 +556,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw3x3s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -560,7 +567,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw5x5s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -571,7 +578,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw5x5s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); + convdw5x5s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -606,7 +613,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con for (int g = 0; g < channels; g++) { float* outptr = top_blob.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 4; + const float* kptr = (const float*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -647,7 +654,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -658,7 +665,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt); if (activation) { @@ -831,7 +838,12 @@ int ConvolutionDepthWise_x86::create_pipeline_int8_x86(const Option& opt) if (elempack == 8) { Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_int8, 8, opt); + convert_packing(weight_data_r2, weight_data_tm, 8, opt); + } + + if (elempack == 1) + { + weight_data_tm = weight_data; } return 0; @@ -840,6 +852,11 @@ int ConvolutionDepthWise_x86::create_pipeline_int8_x86(const Option& opt) // group convolution create_group_ops(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -938,7 +955,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ { signed char* outptr_s8 = top_blob.channel(g); float* outptr_f32 = top_blob.channel(g); - const signed char* kptr = (const signed char*)weight_data_int8 + maxk * g * 8; + const signed char* kptr = (const signed char*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) @@ -1045,7 +1062,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ requantize_scales.push_back(scale_out); } - convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); + convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt); } else { @@ -1057,7 +1074,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ dequantize_scales.push_back(top_rescale); } - convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); + convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, dequantize_scales, opt); } if (activation) @@ -1084,7 +1101,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ requantize_scales.push_back(scale_out); } - convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); + convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt); } else { @@ -1096,7 +1113,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ dequantize_scales.push_back(top_rescale); } - convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); + convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, dequantize_scales, opt); } if (activation) @@ -1132,7 +1149,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_ { signed char* outptr_s8 = top_blob.channel(g); float* outptr_f32 = top_blob.channel(g); - const signed char* kptr = (const signed char*)weight_data + maxk * g; + const signed char* kptr = (const signed char*)weight_data_tm + maxk * g; const Mat m = bottom_blob_bordered.channel(g); for (int i = 0; i < outh; i++) diff --git a/src/layer/x86/convolutiondepthwise_x86.h b/src/layer/x86/convolutiondepthwise_x86.h index db5e208ac..6fe066e5b 100644 --- a/src/layer/x86/convolutiondepthwise_x86.h +++ b/src/layer/x86/convolutiondepthwise_x86.h @@ -42,13 +42,7 @@ public: Layer* activation; std::vector group_ops; - // packing - Mat weight_data_packed; - -#if NCNN_INT8 - // int8 - Mat weight_data_int8; -#endif + Mat weight_data_tm; }; } // namespace ncnn diff --git a/src/layer/x86/deconvolution_x86.cpp b/src/layer/x86/deconvolution_x86.cpp index 295ba4a93..b253fa2e3 100644 --- a/src/layer/x86/deconvolution_x86.cpp +++ b/src/layer/x86/deconvolution_x86.cpp @@ -104,11 +104,11 @@ int Deconvolution_x86::create_pipeline(const Option& opt) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output); - weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack); + weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - float* g00 = weight_data_packed.channel(q / out_elempack); + float* g00 = weight_data_tm.channel(q / out_elempack); for (int p = 0; p + (elempack - 1) < num_input; p += elempack) { @@ -130,6 +130,11 @@ int Deconvolution_x86::create_pipeline(const Option& opt) } } + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -192,49 +197,49 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti if (elempack == 16 && out_elempack == 16) { { - deconvolution_pack16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 8 && out_elempack == 16) { { - deconvolution_pack8to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack8to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 16 && out_elempack == 8) { { - deconvolution_pack16to8_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack16to8_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 4 && out_elempack == 16) { { - deconvolution_pack4to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack4to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 16 && out_elempack == 4) { { - deconvolution_pack16to4_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack16to4_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 1 && out_elempack == 16) { { - deconvolution_pack1to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack1to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 16 && out_elempack == 1) { { - deconvolution_pack16to1_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack16to1_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __AVX512F__ @@ -242,35 +247,35 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti if (elempack == 8 && out_elempack == 8) { { - deconvolution_pack8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 4 && out_elempack == 8) { { - deconvolution_pack4to8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack4to8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 8 && out_elempack == 4) { { - deconvolution_pack8to4_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack8to4_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 1 && out_elempack == 8) { { - deconvolution_pack1to8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack1to8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 8 && out_elempack == 1) { { - deconvolution_pack8to1_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack8to1_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __AVX__ @@ -278,21 +283,21 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti if (elempack == 4 && out_elempack == 4) { { - deconvolution_pack4_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack4_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 1 && out_elempack == 4) { { - deconvolution_pack1to4_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack1to4_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } if (elempack == 4 && out_elempack == 1) { { - deconvolution_pack4to1_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); + deconvolution_pack4to1_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt); } } #endif // __SSE2__ @@ -317,7 +322,7 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti sum = bias_data[p]; } - const float* kptr = (const float*)weight_data_packed.channel(p); + const float* kptr = (const float*)weight_data_tm.channel(p); // channels for (int q = 0; q < channels; q++) diff --git a/src/layer/x86/deconvolution_x86.h b/src/layer/x86/deconvolution_x86.h index ba5ecc920..6f9176bf4 100644 --- a/src/layer/x86/deconvolution_x86.h +++ b/src/layer/x86/deconvolution_x86.h @@ -30,8 +30,7 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; public: - // packn - Mat weight_data_packed; + Mat weight_data_tm; }; } // namespace ncnn diff --git a/src/layer/x86/deconvolutiondepthwise_x86.cpp b/src/layer/x86/deconvolutiondepthwise_x86.cpp index 2cfc019a9..03a249991 100644 --- a/src/layer/x86/deconvolutiondepthwise_x86.cpp +++ b/src/layer/x86/deconvolutiondepthwise_x86.cpp @@ -81,7 +81,7 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 16) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 16, opt); + convert_packing(weight_data_r2, weight_data_tm, 16, opt); } #endif // __AVX512F__ @@ -89,7 +89,7 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 8) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 8, opt); + convert_packing(weight_data_r2, weight_data_tm, 8, opt); } #endif // __AVX__ @@ -97,13 +97,13 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt) if (elempack == 4) { Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_packed, 4, opt); + convert_packing(weight_data_r2, weight_data_tm, 4, opt); } #endif // __SSE2__ if (elempack == 1) { - weight_data_packed = weight_data_transposed; + weight_data_tm = weight_data_transposed; } return 0; @@ -112,6 +112,11 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt) // group convolution create_group_ops(opt); + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -133,7 +138,7 @@ int DeconvolutionDepthWise_x86::create_group_ops(const Option& opt) for (int g = 0; g < group; g++) { - Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g); + Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone(); Mat bias_data_g; if (bias_term) bias_data_g = bias_data.range(num_output_g * g, num_output_g); @@ -256,7 +261,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 16; + const float* kptr = (const float*)weight_data_tm + maxk * g * 16; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -318,7 +323,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 8; + const float* kptr = (const float*)weight_data_tm + maxk * g * 8; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -380,7 +385,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g * 4; + const float* kptr = (const float*)weight_data_tm + maxk * g * 4; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) @@ -441,7 +446,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c for (int g = 0; g < channels; g++) { float* outptr = top_blob_bordered.channel(g); - const float* kptr = (const float*)weight_data_packed + maxk * g; + const float* kptr = (const float*)weight_data_tm + maxk * g; const Mat m = bottom_blob.channel(g); for (int i = 0; i < outh; i++) diff --git a/src/layer/x86/deconvolutiondepthwise_x86.h b/src/layer/x86/deconvolutiondepthwise_x86.h index 039f622fb..33139cfb5 100644 --- a/src/layer/x86/deconvolutiondepthwise_x86.h +++ b/src/layer/x86/deconvolutiondepthwise_x86.h @@ -35,8 +35,7 @@ protected: public: std::vector group_ops; - // packing - Mat weight_data_packed; + Mat weight_data_tm; }; } // namespace ncnn diff --git a/src/layer/x86/innerproduct_x86.cpp b/src/layer/x86/innerproduct_x86.cpp index 03fe13741..807569b74 100644 --- a/src/layer/x86/innerproduct_x86.cpp +++ b/src/layer/x86/innerproduct_x86.cpp @@ -35,7 +35,6 @@ InnerProduct_x86::InnerProduct_x86() #endif // __SSE2__ flatten = 0; - activation = 0; } int InnerProduct_x86::create_pipeline(const Option& opt) @@ -82,11 +81,11 @@ int InnerProduct_x86::create_pipeline(const Option& opt) { Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - weight_data_packed.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack); + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - float* g0 = weight_data_packed.row(q / out_elempack); + float* g0 = weight_data_tm.row(q / out_elempack); for (int p = 0; p < num_input; p++) { @@ -98,6 +97,15 @@ int InnerProduct_x86::create_pipeline(const Option& opt) } } } + else + { + weight_data_tm = weight_data; + } + + if (opt.lightmode) + { + weight_data.release(); + } return 0; } @@ -111,20 +119,13 @@ int InnerProduct_x86::destroy_pipeline(const Option& opt) flatten = 0; } - if (activation) - { - activation->destroy_pipeline(opt); - delete activation; - activation = 0; - } - return 0; } int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { #if NCNN_INT8 - if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + if (opt.use_int8_inference && int8_scale_term) { return forward_int8_x86(bottom_blob, top_blob, opt); } @@ -169,7 +170,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 16; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m512 _sum0 = _mm512_set1_ps(0.f); @@ -276,7 +277,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 16; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m512 _sum = _mm512_set1_ps(0.f); @@ -310,7 +311,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 16; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m128 _sum0 = _mm_set1_ps(0.f); @@ -418,7 +419,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 16; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m256 _sum0 = _mm256_set1_ps(0.f); @@ -526,7 +527,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); __m512 _sum0 = _mm512_set1_ps(0.f); @@ -560,7 +561,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 4; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m512 _sum0 = _mm512_set1_ps(0.f); @@ -608,7 +609,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 8; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m512 _sum0 = _mm512_set1_ps(0.f); @@ -678,7 +679,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 8; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m256 _sum0 = _mm256_set1_ps(0.f); @@ -753,7 +754,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 8; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m256 _sum = _mm256_set1_ps(0.f); @@ -837,7 +838,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 8; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m128 _sum0 = _mm_set1_ps(0.f); @@ -905,7 +906,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); __m256 _sum0 = _mm256_set1_ps(0.f); @@ -982,7 +983,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 4; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m256 _sum0 = _mm256_set1_ps(0.f); @@ -1059,7 +1060,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 4; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m128 _sum0 = _mm_set1_ps(0.f); @@ -1133,7 +1134,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output / num_output_elempack; p++) { - const float* kptr = (const float*)weight_data_packed + num_input * p * 4; + const float* kptr = weight_data_tm.row(p); const float* m = bottom_blob.row(j); __m128 _sum = _mm_set1_ps(0.f); @@ -1219,7 +1220,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); __m128 _sum0 = _mm_set1_ps(0.f); @@ -1297,7 +1298,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (int p = 0; p < num_output; p++) { - const float* kptr = (const float*)weight_data + num_input * p; + const float* kptr = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob.row(j); float sum = 0.f; @@ -1409,7 +1410,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio _sum0 = _mm512_loadu_ps((const float*)bias_data + p * 16); } - const float* kptr = weight_data_packed.row(p); + const float* kptr = weight_data_tm.row(p); const float* sptr = bottom_blob_flattened; @@ -1511,7 +1512,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio _sum0 = _mm256_loadu_ps((const float*)bias_data + p * 8); } - const float* kptr = weight_data_packed.row(p); + const float* kptr = weight_data_tm.row(p); const float* sptr = bottom_blob_flattened; @@ -1613,7 +1614,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio _sum0 = _mm_loadu_ps((const float*)bias_data + p * 4); } - const float* kptr = weight_data_packed.row(p); + const float* kptr = weight_data_tm.row(p); const float* sptr = bottom_blob_flattened; @@ -1725,14 +1726,14 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio sums[7] = bias_data[p + 7]; } - const float* w0 = (const float*)weight_data + num_input * p; - const float* w1 = (const float*)weight_data + num_input * (p + 1); - const float* w2 = (const float*)weight_data + num_input * (p + 2); - const float* w3 = (const float*)weight_data + num_input * (p + 3); - const float* w4 = (const float*)weight_data + num_input * (p + 4); - const float* w5 = (const float*)weight_data + num_input * (p + 5); - const float* w6 = (const float*)weight_data + num_input * (p + 6); - const float* w7 = (const float*)weight_data + num_input * (p + 7); + const float* w0 = (const float*)weight_data_tm + num_input * p; + const float* w1 = (const float*)weight_data_tm + num_input * (p + 1); + const float* w2 = (const float*)weight_data_tm + num_input * (p + 2); + const float* w3 = (const float*)weight_data_tm + num_input * (p + 3); + const float* w4 = (const float*)weight_data_tm + num_input * (p + 4); + const float* w5 = (const float*)weight_data_tm + num_input * (p + 5); + const float* w6 = (const float*)weight_data_tm + num_input * (p + 6); + const float* w7 = (const float*)weight_data_tm + num_input * (p + 7); const float* m = bottom_blob_flattened; @@ -1829,10 +1830,10 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio sums[3] = bias_data[p + 3]; } - const float* w0 = (const float*)weight_data + num_input * p; - const float* w1 = (const float*)weight_data + num_input * (p + 1); - const float* w2 = (const float*)weight_data + num_input * (p + 2); - const float* w3 = (const float*)weight_data + num_input * (p + 3); + const float* w0 = (const float*)weight_data_tm + num_input * p; + const float* w1 = (const float*)weight_data_tm + num_input * (p + 1); + const float* w2 = (const float*)weight_data_tm + num_input * (p + 2); + const float* w3 = (const float*)weight_data_tm + num_input * (p + 3); const float* m = bottom_blob_flattened; @@ -1919,7 +1920,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio int remain_num_output_start = 0; #endif // __SSE2__ -// num_output #pragma omp parallel for num_threads(opt.num_threads) for (int p = remain_num_output_start; p < num_output; p++) { @@ -1928,7 +1928,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio if (bias_term) sum = bias_data[p]; - const float* w = (const float*)weight_data + num_input * p; + const float* w = (const float*)weight_data_tm + num_input * p; const float* m = bottom_blob_flattened; @@ -1986,8 +1986,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio #if NCNN_INT8 int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt) { - activation = create_activation_layer(activation_type, activation_params, opt); - const int num_input = weight_data_size / num_output; int out_elempack = 1; @@ -2003,11 +2001,11 @@ int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt) { Mat weight_data_r2 = weight_data.reshape(num_input, num_output); - weight_data_int8.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack); + weight_data_tm.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack); for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack) { - signed char* g0 = weight_data_int8.row(q / out_elempack); + signed char* g0 = weight_data_tm.row(q / out_elempack); for (int p = 0; p < num_input; p++) { @@ -2019,6 +2017,24 @@ int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt) } } + scale_in_data.create(num_output); + for (int p = 0; p < num_output; p++) + { + // dequantize + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); + + scale_in_data[p] = scale_in; + } + + if (opt.lightmode) + { + weight_data.release(); + } + return 0; } @@ -2026,17 +2042,6 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co { const int num_input = weight_data_size / num_output; - if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1) - { - // gemm - Mat bottom_blob_unpacked; - Option opt_unpack = opt; - opt_unpack.blob_allocator = opt.workspace_allocator; - convert_packing(bottom_blob, bottom_blob_unpacked, 1, opt_unpack); - - return forward_int8(bottom_blob_unpacked, top_blob, opt); - } - int elembits = bottom_blob.elembits(); Mat bottom_blob_int8 = bottom_blob; @@ -2047,6 +2052,327 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co quantize_to_int8(bottom_blob, bottom_blob_int8, bottom_blob_int8_scales, opt_q); } + if (bottom_blob_int8.dims == 2 && bottom_blob_int8.w == num_input && bottom_blob_int8.h * bottom_blob_int8.elempack > 1) + { + // gemm + Mat bottom_blob_int8_unpacked; + Option opt_unpack = opt; + opt_unpack.blob_allocator = opt.workspace_allocator; + convert_packing(bottom_blob_int8, bottom_blob_int8_unpacked, 1, opt_unpack); + + int h = bottom_blob_int8_unpacked.h; + + int out_elempack = 1; +#if __SSE2__ + if (opt.use_packing_layout) + { + out_elempack = h % 4 == 0 ? 4 : 1; + } +#endif + + int outh = h / out_elempack; + + top_blob.create(num_output, outh, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + int num_output_elempack = 1; +#if __SSE2__ + if (opt.use_packing_layout) + { + num_output_elempack = num_output % 8 == 0 ? 8 : 1; + } +#endif + +#if __SSE2__ + if (num_output_elempack == 8 && out_elempack == 4) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < outh; j++) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output / num_output_elempack; p++) + { + const signed char* kptr = weight_data_tm.row(p); + const signed char* m0 = bottom_blob_int8_unpacked.row(j * 4); + const signed char* m1 = bottom_blob_int8_unpacked.row(j * 4 + 1); + const signed char* m2 = bottom_blob_int8_unpacked.row(j * 4 + 2); + const signed char* m3 = bottom_blob_int8_unpacked.row(j * 4 + 3); + + __m128i _sum00 = _mm_setzero_si128(); + __m128i _sum01 = _mm_setzero_si128(); + __m128i _sum10 = _mm_setzero_si128(); + __m128i _sum11 = _mm_setzero_si128(); + __m128i _sum20 = _mm_setzero_si128(); + __m128i _sum21 = _mm_setzero_si128(); + __m128i _sum30 = _mm_setzero_si128(); + __m128i _sum31 = _mm_setzero_si128(); + + int i = 0; + for (; i < num_input; i++) + { + // TODO use _mm_cvtepi8_epi16 on sse4.1 + __m128i _w = _mm_loadl_epi64((const __m128i*)kptr); + _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w)); + + __m128i _val0 = _mm_set1_epi16((short)m0[0]); + __m128i _val1 = _mm_set1_epi16((short)m1[0]); + __m128i _val2 = _mm_set1_epi16((short)m2[0]); + __m128i _val3 = _mm_set1_epi16((short)m3[0]); + + __m128i _s0l = _mm_mullo_epi16(_val0, _w); + __m128i _s0h = _mm_mulhi_epi16(_val0, _w); + __m128i _s1l = _mm_mullo_epi16(_val1, _w); + __m128i _s1h = _mm_mulhi_epi16(_val1, _w); + __m128i _s2l = _mm_mullo_epi16(_val2, _w); + __m128i _s2h = _mm_mulhi_epi16(_val2, _w); + __m128i _s3l = _mm_mullo_epi16(_val3, _w); + __m128i _s3h = _mm_mulhi_epi16(_val3, _w); + __m128i _s00 = _mm_unpacklo_epi16(_s0l, _s0h); + __m128i _s01 = _mm_unpackhi_epi16(_s0l, _s0h); + __m128i _s10 = _mm_unpacklo_epi16(_s1l, _s1h); + __m128i _s11 = _mm_unpackhi_epi16(_s1l, _s1h); + __m128i _s20 = _mm_unpacklo_epi16(_s2l, _s2h); + __m128i _s21 = _mm_unpackhi_epi16(_s2l, _s2h); + __m128i _s30 = _mm_unpacklo_epi16(_s3l, _s3h); + __m128i _s31 = _mm_unpackhi_epi16(_s3l, _s3h); + + _sum00 = _mm_add_epi32(_sum00, _s00); + _sum01 = _mm_add_epi32(_sum01, _s01); + _sum10 = _mm_add_epi32(_sum10, _s10); + _sum11 = _mm_add_epi32(_sum11, _s11); + _sum20 = _mm_add_epi32(_sum20, _s20); + _sum21 = _mm_add_epi32(_sum21, _s21); + _sum30 = _mm_add_epi32(_sum30, _s30); + _sum31 = _mm_add_epi32(_sum31, _s31); + + m0++; + m1++; + m2++; + m3++; + kptr += 8; + } + + // dequantize and relu + __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8); + __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4); + + __m128 _sumfp32_00 = _mm_cvtepi32_ps(_sum00); + __m128 _sumfp32_01 = _mm_cvtepi32_ps(_sum01); + __m128 _sumfp32_10 = _mm_cvtepi32_ps(_sum10); + __m128 _sumfp32_11 = _mm_cvtepi32_ps(_sum11); + __m128 _sumfp32_20 = _mm_cvtepi32_ps(_sum20); + __m128 _sumfp32_21 = _mm_cvtepi32_ps(_sum21); + __m128 _sumfp32_30 = _mm_cvtepi32_ps(_sum30); + __m128 _sumfp32_31 = _mm_cvtepi32_ps(_sum31); + if (bias_term) + { + __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8); + __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4); + _sumfp32_00 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_00, _scale_in0)); + _sumfp32_01 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_01, _scale_in1)); + _sumfp32_10 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_10, _scale_in0)); + _sumfp32_11 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_11, _scale_in1)); + _sumfp32_20 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_20, _scale_in0)); + _sumfp32_21 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_21, _scale_in1)); + _sumfp32_30 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_30, _scale_in0)); + _sumfp32_31 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_31, _scale_in1)); + } + else + { + _sumfp32_00 = _mm_mul_ps(_sumfp32_00, _scale_in0); + _sumfp32_01 = _mm_mul_ps(_sumfp32_01, _scale_in1); + _sumfp32_10 = _mm_mul_ps(_sumfp32_10, _scale_in0); + _sumfp32_11 = _mm_mul_ps(_sumfp32_11, _scale_in1); + _sumfp32_20 = _mm_mul_ps(_sumfp32_20, _scale_in0); + _sumfp32_21 = _mm_mul_ps(_sumfp32_21, _scale_in1); + _sumfp32_30 = _mm_mul_ps(_sumfp32_30, _scale_in0); + _sumfp32_31 = _mm_mul_ps(_sumfp32_31, _scale_in1); + } + + _sumfp32_00 = activation_sse(_sumfp32_00, activation_type, activation_params); + _sumfp32_01 = activation_sse(_sumfp32_01, activation_type, activation_params); + _sumfp32_10 = activation_sse(_sumfp32_10, activation_type, activation_params); + _sumfp32_11 = activation_sse(_sumfp32_11, activation_type, activation_params); + _sumfp32_20 = activation_sse(_sumfp32_20, activation_type, activation_params); + _sumfp32_21 = activation_sse(_sumfp32_21, activation_type, activation_params); + _sumfp32_30 = activation_sse(_sumfp32_30, activation_type, activation_params); + _sumfp32_31 = activation_sse(_sumfp32_31, activation_type, activation_params); + + // transpose 4x8 + _MM_TRANSPOSE4_PS(_sumfp32_00, _sumfp32_10, _sumfp32_20, _sumfp32_30); + _MM_TRANSPOSE4_PS(_sumfp32_01, _sumfp32_11, _sumfp32_21, _sumfp32_31); + + _mm_storeu_ps(outptr, _sumfp32_00); + _mm_storeu_ps(outptr + 4, _sumfp32_10); + _mm_storeu_ps(outptr + 8, _sumfp32_20); + _mm_storeu_ps(outptr + 12, _sumfp32_30); + _mm_storeu_ps(outptr + 16, _sumfp32_01); + _mm_storeu_ps(outptr + 20, _sumfp32_11); + _mm_storeu_ps(outptr + 24, _sumfp32_21); + _mm_storeu_ps(outptr + 28, _sumfp32_31); + + outptr += 32; + } + } + } + + if (num_output_elempack == 1 && out_elempack == 4) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < outh; j++) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output; p++) + { + const signed char* kptr = weight_data_tm.row(p); + const signed char* m0 = bottom_blob_int8_unpacked.row(j * 4); + const signed char* m1 = bottom_blob_int8_unpacked.row(j * 4 + 1); + const signed char* m2 = bottom_blob_int8_unpacked.row(j * 4 + 2); + const signed char* m3 = bottom_blob_int8_unpacked.row(j * 4 + 3); + + int sum0 = 0; + int sum1 = 0; + int sum2 = 0; + int sum3 = 0; + + int i = 0; + for (; i < num_input; i++) + { + sum0 += *m0++ * kptr[0]; + sum1 += *m1++ * kptr[0]; + sum2 += *m2++ * kptr[0]; + sum3 += *m3++ * kptr[0]; + kptr += 1; + } + + // dequantize and relu + float sumfp32_0 = sum0 * scale_in_data[p]; + float sumfp32_1 = sum1 * scale_in_data[p]; + float sumfp32_2 = sum2 * scale_in_data[p]; + float sumfp32_3 = sum3 * scale_in_data[p]; + + if (bias_term) + { + sumfp32_0 += bias_data[p]; + sumfp32_1 += bias_data[p]; + sumfp32_2 += bias_data[p]; + sumfp32_3 += bias_data[p]; + } + + outptr[0] = activation_ss(sumfp32_0, activation_type, activation_params); + outptr[1] = activation_ss(sumfp32_1, activation_type, activation_params); + outptr[2] = activation_ss(sumfp32_2, activation_type, activation_params); + outptr[3] = activation_ss(sumfp32_3, activation_type, activation_params); + outptr += 4; + } + } + } + + if (num_output_elempack == 8 && out_elempack == 1) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < outh; j++) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output / num_output_elempack; p++) + { + const signed char* kptr = weight_data_tm.row(p); + const signed char* m = bottom_blob_int8_unpacked.row(j); + + __m128i _sum0 = _mm_setzero_si128(); + __m128i _sum1 = _mm_setzero_si128(); + + int i = 0; + for (; i < num_input; i++) + { + __m128i _val = _mm_set1_epi16((short)m[0]); + + // TODO use _mm_cvtepi8_epi16 on sse4.1 + __m128i _w = _mm_loadl_epi64((const __m128i*)kptr); + _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w)); + + __m128i _sl = _mm_mullo_epi16(_val, _w); + __m128i _sh = _mm_mulhi_epi16(_val, _w); + __m128i _s0 = _mm_unpacklo_epi16(_sl, _sh); + __m128i _s1 = _mm_unpackhi_epi16(_sl, _sh); + + _sum0 = _mm_add_epi32(_sum0, _s0); + _sum1 = _mm_add_epi32(_sum1, _s1); + + m++; + kptr += 8; + } + + // dequantize and relu + __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8); + __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4); + + __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0); + __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1); + + if (bias_term) + { + __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8); + __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4); + _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0)); + _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1)); + } + else + { + _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0); + _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1); + } + + _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params); + _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params); + + _mm_storeu_ps(outptr, _sumfp32_0); + _mm_storeu_ps(outptr + 4, _sumfp32_1); + outptr += 8; + } + } + } +#endif // __SSE2__ + + if (num_output_elempack == 1 && out_elempack == 1) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < outh; j++) + { + float* outptr = top_blob.row(j); + + for (int p = 0; p < num_output; p++) + { + const signed char* kptr = weight_data_tm.row(p); + const signed char* m = bottom_blob_int8_unpacked.row(j); + + int sum = 0; + + int i = 0; + for (; i < num_input; i++) + { + sum += *m++ * *kptr++; + } + + // dequantize and relu + float sumfp32 = sum * scale_in_data[p]; + + if (bias_term) + sumfp32 += bias_data[p]; + + outptr[0] = activation_ss(sumfp32, activation_type, activation_params); + outptr += 1; + } + } + } + + return 0; + } + Mat bottom_blob_int8_flattened = bottom_blob_int8; if (bottom_blob_int8.dims != 1) { @@ -2070,22 +2396,16 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co if (top_blob.empty()) return -100; - Mat top_blob_int32; - top_blob_int32.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.workspace_allocator); - if (top_blob_int32.empty()) - return -100; - #if __SSE2__ if (out_elempack == 8) { -// num_output #pragma omp parallel for num_threads(opt.num_threads) for (int p = 0; p < num_output / out_elempack; p++) { __m128i _sum0 = _mm_setzero_si128(); __m128i _sum1 = _mm_setzero_si128(); - const signed char* kptr = weight_data_int8.row(p); + const signed char* kptr = weight_data_tm.row(p); const signed char* sptr = bottom_blob_int8_flattened; int i = 0; @@ -2109,22 +2429,44 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co kptr += 8; } - int* outptr = (int*)top_blob_int32; - _mm_storeu_si128((__m128i*)(outptr + p * 8), _sum0); - _mm_storeu_si128((__m128i*)(outptr + p * 8 + 4), _sum1); + // dequantize and relu + __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8); + __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4); + + __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0); + __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1); + + if (bias_term) + { + __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8); + __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4); + _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0)); + _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1)); + } + else + { + _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0); + _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1); + } + + _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params); + _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params); + + float* outptr = (float*)top_blob + p * 8; + _mm_storeu_ps(outptr, _sumfp32_0); + _mm_storeu_ps(outptr + 4, _sumfp32_1); } } #endif // __SSE2__ if (out_elempack == 1) { -// num_output #pragma omp parallel for num_threads(opt.num_threads) for (int p = 0; p < num_output / out_elempack; p++) { int sum = 0; - const signed char* kptr = weight_data_int8.row(p); + const signed char* kptr = weight_data_tm.row(p); const signed char* sptr = bottom_blob_int8_flattened; int i = 0; @@ -2140,29 +2482,16 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co kptr += 1; } - int* outptr = (int*)top_blob_int32; - outptr[p] = sum; - } - } + // dequantize and relu + float sumfp32 = sum * scale_in_data[p]; - Mat scale_data(num_output); - for (int p = 0; p < num_output; p++) - { - // dequantize - float scale_in; - if (weight_data_int8_scales[p] == 0) - scale_in = 0; - else - scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]); - - scale_data[p] = scale_in; - } + if (bias_term) + sumfp32 += bias_data[p]; - dequantize_from_int32(top_blob_int32, top_blob, scale_data, bias_data, opt); + sumfp32 = activation_ss(sumfp32, activation_type, activation_params); - if (activation) - { - activation->forward_inplace(top_blob, opt); + top_blob[p] = sumfp32; + } } return 0; diff --git a/src/layer/x86/innerproduct_x86.h b/src/layer/x86/innerproduct_x86.h index fa8abbfa8..5484c02cb 100644 --- a/src/layer/x86/innerproduct_x86.h +++ b/src/layer/x86/innerproduct_x86.h @@ -1,19 +1,16 @@ -// Tencent is pleased to support the open source community by making ncnn -// available. +// Tencent is pleased to support the open source community by making ncnn available. // // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. // -// Licensed under the BSD 3-Clause License (the "License"); you may not use this -// file except in compliance with the License. You may obtain a copy of the -// License at +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at // // https://opensource.org/licenses/BSD-3-Clause // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. #ifndef LAYER_INNERPRODUCT_X86_H #define LAYER_INNERPRODUCT_X86_H @@ -40,14 +37,11 @@ protected: public: Layer* flatten; - Layer* activation; - Mat weight_data_packed; + Mat weight_data_tm; #if NCNN_INT8 - // int8 - Mat weight_data_int8; - Mat scales_in; + Mat scale_in_data; #endif }; diff --git a/tests/testutil.h b/tests/testutil.h index 9c32d16dd..fdae5d08e 100644 --- a/tests/testutil.h +++ b/tests/testutil.h @@ -363,6 +363,7 @@ int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector ncnn::Option opt; opt.num_threads = 1; + opt.lightmode = false; opt.use_packing_layout = false; opt.use_fp16_packed = false; opt.use_fp16_storage = false; @@ -808,6 +809,7 @@ int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector ncnn::Option opt; opt.num_threads = 1; + opt.lightmode = false; opt.use_packing_layout = false; opt.use_fp16_packed = false; opt.use_fp16_storage = false;