discard weight memory for x86 arm vulkan (#3865)

* discard weight memory for x86 and vulkan
* drop arm innerproduct weight
* drop arm convolution weight
* drop arm convolutiondepthwise weight
* drop x86 vulkan deconvolution deconvolutiondepthwise weight
* drop arm deconvolution deconvolutiondepthwise weight
* arm neon assembly optimization for innerproduct pack4

cmake/ncnn_add_layer.cmake

@@ -36,11 +36,6 @@ macro(ncnn_add_arch_opt_layer class NCNN_TARGET_ARCH_OPT NCNN_TARGET_ARCH_OPT_CF
         set(create_pipeline_content "        { int ret = ${class}::create_pipeline(opt); if (ret) return ret; }\n")
         set(destroy_pipeline_content "        { int ret = ${class}::destroy_pipeline(opt); if (ret) return ret; }\n")
 
-        set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}.h\"\n")
-        set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}")
-        set(create_pipeline_content "${create_pipeline_content}        { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::create_pipeline(opt); if (ret) return ret; }\n")
-        set(destroy_pipeline_content "        { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}")
-
         if(WITH_LAYER_${name}_vulkan)
             set(layer_declaration "${layer_declaration}#include \"layer/vulkan/${name}_vulkan.h\"\n")
             set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_vulkan")
@@ -48,6 +43,11 @@ macro(ncnn_add_arch_opt_layer class NCNN_TARGET_ARCH_OPT NCNN_TARGET_ARCH_OPT_CF
             set(destroy_pipeline_content "        if (vkdev) { int ret = ${class}_vulkan::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}")
         endif()
 
+        set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}.h\"\n")
+        set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}")
+        set(create_pipeline_content "${create_pipeline_content}        { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::create_pipeline(opt); if (ret) return ret; }\n")
+        set(destroy_pipeline_content "        { int ret = ${class}_${NCNN_TARGET_ARCH}_${NCNN_TARGET_ARCH_OPT}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}")
+
         set(layer_declaration "${layer_declaration}namespace ncnn {\n${layer_declaration_class}\n{\n")
         set(layer_declaration "${layer_declaration}public:\n")
         set(layer_declaration "${layer_declaration}    virtual int create_pipeline(const Option& opt) {\n${create_pipeline_content}        return 0;\n    }\n")
@@ -118,15 +118,6 @@ macro(ncnn_add_layer class)
         source_group ("sources\\\\layers" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${name}.cpp")
     endif()
 
-    if(WITH_LAYER_${name}_${NCNN_TARGET_ARCH})
-        set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.h\"\n")
-        set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}")
-        set(create_pipeline_content "${create_pipeline_content}        { int ret = ${class}_${NCNN_TARGET_ARCH}::create_pipeline(opt); if (ret) return ret; }\n")
-        set(destroy_pipeline_content "        { int ret = ${class}_${NCNN_TARGET_ARCH}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}")
-
-        source_group ("sources\\\\layers\\\\${NCNN_TARGET_ARCH}" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.cpp")
-    endif()
-
     if(WITH_LAYER_${name}_vulkan)
         set(layer_declaration "${layer_declaration}#include \"layer/vulkan/${name}_vulkan.h\"\n")
         set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_vulkan")
@@ -143,6 +134,15 @@ macro(ncnn_add_layer class)
         source_group ("sources\\\\layers\\\\vulkan" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/vulkan/${name}_vulkan.cpp")
     endif()
 
+    if(WITH_LAYER_${name}_${NCNN_TARGET_ARCH})
+        set(layer_declaration "${layer_declaration}#include \"layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.h\"\n")
+        set(layer_declaration_class "${layer_declaration_class}, virtual public ${class}_${NCNN_TARGET_ARCH}")
+        set(create_pipeline_content "${create_pipeline_content}        { int ret = ${class}_${NCNN_TARGET_ARCH}::create_pipeline(opt); if (ret) return ret; }\n")
+        set(destroy_pipeline_content "        { int ret = ${class}_${NCNN_TARGET_ARCH}::destroy_pipeline(opt); if (ret) return ret; }\n${destroy_pipeline_content}")
+
+        source_group ("sources\\\\layers\\\\${NCNN_TARGET_ARCH}" FILES "${CMAKE_CURRENT_SOURCE_DIR}/layer/${NCNN_TARGET_ARCH}/${name}_${NCNN_TARGET_ARCH}.cpp")
+    endif()
+
     if(WITH_LAYER_${name})
         set(layer_declaration "${layer_declaration}namespace ncnn {\n${layer_declaration_class}\n{\n")
         set(layer_declaration "${layer_declaration}public:\n")

src/command.cpp

@@ -3175,7 +3175,7 @@ void VkTransfer::record_upload(const Mat& src, VkImageMat& dst, const Option& op
     //     NCNN_LOGE("record_upload image src = %d | %d %d %d @ %d", src.dims, src.w, src.h, src.c, src.elempack);
 
     // NOTE keep the hack here ?
-    if (src.elemsize == src.elempack * 4u)
+    if (src.elembits() == 32)
     {
         if (opt.use_fp16_storage || (opt.use_fp16_packed && src.elempack % 4 == 0))
         {

src/layer/arm/convolution_arm.h

@@ -50,6 +50,7 @@ protected:
 public:
     Layer* activation;
 
+    Mat weight_data_tm;
     Mat weight_3x3s2_data;
 
     Mat weight_sgemm_data;
@@ -59,22 +60,10 @@ public:
     // forwardDilation
     Layer* convolution_dilation1;
 
-    // pack4
-    Mat weight_data_packed;
-
     // fp16
-    Mat weight_data_fp16;
     Mat bias_data_fp16;
 
-#if NCNN_BF16
-    // bf16
-    Mat weight_data_bf16;
-#endif
-
 #if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
-
     //     Mat weight_3x3s2_data_int8;
     std::vector<Mat> weight_3x3_winograd23_data_int8;
 #endif

src/layer/arm/convolutiondepthwise_arm.cpp

@@ -111,7 +111,7 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
                 Mat weight_data_r2_packed;
                 convert_packing(weight_data_r2, weight_data_r2_packed, 8, opt);
 
-                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt);
             }
 
             if (elempack == 4)
@@ -120,12 +120,12 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
                 Mat weight_data_r2_packed;
                 convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt);
 
-                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt);
             }
 
             if (elempack == 1)
             {
-                ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data, weight_data_tm, opt);
             }
 
             ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);
@@ -141,15 +141,16 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
             if (elempack == 4)
             {
                 Mat weight_data_r2 = weight_data.reshape(maxk, group);
-                convert_packing(weight_data_r2, weight_data_pack4, 4, opt);
+                Mat weight_data_r2_packed;
+                convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt);
 
-                ncnn::cast_float32_to_bfloat16(weight_data_pack4, weight_data_pack4_bf16, opt);
+                ncnn::cast_float32_to_bfloat16(weight_data_r2_packed, weight_data_tm, opt);
             }
 #endif // __ARM_NEON
 
             if (elempack == 1)
             {
-                ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt);
+                ncnn::cast_float32_to_bfloat16(weight_data, weight_data_tm, opt);
             }
 
             return 0;
@@ -161,7 +162,7 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
         if (elempack == 4)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_pack4, 4, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 4, opt);
 
             return 0;
         }
@@ -171,18 +172,22 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
         }
@@ -191,6 +196,11 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
     // group convolution
     create_group_ops(opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -212,7 +222,7 @@ int ConvolutionDepthWise_arm::create_group_ops(const Option& opt)
 
     for (int g = 0; g < group; g++)
     {
-        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
+        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone();
         Mat bias_data_g;
         if (bias_term)
             bias_data_g = bias_data.range(num_output_g * g, num_output_g);
@@ -313,7 +323,7 @@ int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt)
 int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
 #if NCNN_INT8
-    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    if (opt.use_int8_inference && int8_scale_term)
     {
         return forward_int8_arm(bottom_blob, top_blob, opt);
     }
@@ -376,7 +386,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
+                convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -387,7 +397,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
+                convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -398,7 +408,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
+                convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -409,7 +419,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
+                convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -445,7 +455,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob.channel(g);
-                    const float* kptr = (const float*)weight_data_pack4 + maxk * g * 4;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -486,7 +496,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -497,7 +507,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -508,7 +518,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -519,7 +529,7 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -762,7 +772,7 @@ int ConvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blo
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -824,7 +834,7 @@ int ConvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blo
                 for (int g = 0; g < group; g++)
                 {
                     __fp16* outptr = top_blob.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -949,7 +959,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw3x3s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -958,7 +968,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
             }
             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw3x3s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -967,7 +977,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw5x5s1_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -976,7 +986,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw5x5s2_pack8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -1010,7 +1020,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 8;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1072,7 +1082,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1110,7 +1120,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw3x3s1_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -1119,7 +1129,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
             }
             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);
+                convdw3x3s2_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data_fp16, opt);
 
                 if (activation)
                 {
@@ -1153,7 +1163,7 @@ int ConvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_bl
                 for (int g = 0; g < group; g++)
                 {
                     __fp16* outptr = top_blob.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1288,7 +1298,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);
+                convdw3x3s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -1297,7 +1307,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             }
             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);
+                convdw3x3s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -1306,7 +1316,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);
+                convdw5x5s1_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -1315,7 +1325,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             }
             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_pack4_bf16, bias_data, opt);
+                convdw5x5s2_pack4_bf16s_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -1349,7 +1359,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
                 for (int g = 0; g < channels; g++)
                 {
                     unsigned short* outptr = top_blob.channel(g);
-                    const unsigned short* kptr = (const unsigned short*)weight_data_pack4_bf16 + maxk * g * 4;
+                    const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1390,7 +1400,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
         {
             //             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             //             {
-            //                 convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt);
+            //                 convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             //
             //                 if (activation)
             //                 {
@@ -1401,7 +1411,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             //             }
             //             else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             //             {
-            //                 convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt);
+            //                 convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             //
             //                 if (activation)
             //                 {
@@ -1412,7 +1422,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             //             }
             //             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             //             {
-            //                 convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt);
+            //                 convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             //
             //                 if (activation)
             //                 {
@@ -1423,7 +1433,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
             //             }
             //             else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             //             {
-            //                 convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_bf16, bias_data, opt);
+            //                 convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             //
             //                 if (activation)
             //                 {
@@ -1459,7 +1469,7 @@ int ConvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blo
                 for (int g = 0; g < group; g++)
                 {
                     unsigned short* outptr = top_blob.channel(g);
-                    const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g;
+                    const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1573,7 +1583,12 @@ int ConvolutionDepthWise_arm::create_pipeline_int8_arm(const Option& opt)
         if (elempack == 8)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_int8, 8, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 8, opt);
+        }
+
+        if (elempack == 1)
+        {
+            weight_data_tm = weight_data;
         }
 
         return 0;
@@ -1582,6 +1597,11 @@ int ConvolutionDepthWise_arm::create_pipeline_int8_arm(const Option& opt)
     // group convolution
     create_group_ops(opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -1674,7 +1694,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                 if (top_blob_int32.empty())
                     return -100;
 
-                convdw3x3s1_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_int8, opt);
+                convdw3x3s1_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt);
 
                 Mat scale_in_data(group);
                 for (int g = 0; g < group; g++)
@@ -1710,7 +1730,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                 if (top_blob_int32.empty())
                     return -100;
 
-                convdw3x3s2_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_int8, opt);
+                convdw3x3s2_pack8_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt);
 
                 Mat scale_in_data(group);
                 for (int g = 0; g < group; g++)
@@ -1767,7 +1787,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                 {
                     signed char* outptr_s8 = top_blob.channel(g);
                     float* outptr_f32 = top_blob.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data_int8 + maxk * g * 8;
+                    const signed char* kptr = (const signed char*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1862,7 +1882,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                         requantize_scales.push_back(scale_out);
                     }
 
-                    convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+                    convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt);
                 }
                 else
                 {
@@ -1871,8 +1891,8 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                     if (top_blob_int32.empty())
                         return -100;
 
-                    convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data, opt);
-                    //                 convdw3x3s1_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data, bias_data, dequantize_scales, opt);
+                    convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt);
+                    //                 convdw3x3s1_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, bias_data, dequantize_scales, opt);
 
                     Mat scale_data(group);
                     for (int g = 0; g < group; g++)
@@ -1914,7 +1934,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                         requantize_scales.push_back(scale_out);
                     }
 
-                    convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+                    convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt);
                 }
                 else
                 {
@@ -1923,8 +1943,8 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                     if (top_blob_int32.empty())
                         return -100;
 
-                    convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data, opt);
-                    //                 convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data, bias_data, dequantize_scales, opt);
+                    convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, opt);
+                    //                 convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob_int32, weight_data_tm, bias_data, dequantize_scales, opt);
 
                     Mat scale_data(group);
                     for (int g = 0; g < group; g++)
@@ -1975,7 +1995,7 @@ int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_
                 {
                     signed char* outptr_s8 = top_blob.channel(g);
                     float* outptr_f32 = top_blob.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data + maxk * g;
+                    const signed char* kptr = (const signed char*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)

src/layer/arm/convolutiondepthwise_arm.h

@@ -49,23 +49,10 @@ public:
     Layer* activation;
     std::vector<ncnn::Layer*> group_ops;
 
-    // packing
-    Mat weight_data_pack4;
+    Mat weight_data_tm;
 
     // fp16
-    Mat weight_data_fp16;
     Mat bias_data_fp16;
-
-#if NCNN_BF16
-    // bf16
-    Mat weight_data_bf16;
-    Mat weight_data_pack4_bf16;
-#endif
-
-#if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
-#endif
 };
 
 } // namespace ncnn

src/layer/arm/deconvolution_arm.cpp

@@ -139,7 +139,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-            weight_data_pack4.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16);
+            weight_data_tm.create(maxk, num_input / 4, num_output / 4, (size_t)4 * 16, 16);
 
             for (int q = 0; q + 3 < num_output; q += 4)
             {
@@ -148,7 +148,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
                 const Mat k2 = weight_data_r2.channel(q + 2);
                 const Mat k3 = weight_data_r2.channel(q + 3);
 
-                float* g00 = weight_data_pack4.channel(q / 4);
+                float* g00 = weight_data_tm.channel(q / 4);
 
                 for (int p = 0; p + 3 < num_input; p += 4)
                 {
@@ -209,7 +209,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-            weight_data_pack1to4.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4);
+            weight_data_tm.create(maxk, num_input, num_output / 4, (size_t)4 * 4, 4);
 
             for (int q = 0; q + 3 < num_output; q += 4)
             {
@@ -218,7 +218,7 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
                 const Mat k2 = weight_data_r2.channel(q + 2);
                 const Mat k3 = weight_data_r2.channel(q + 3);
 
-                float* g00 = weight_data_pack1to4.channel(q / 4);
+                float* g00 = weight_data_tm.channel(q / 4);
 
                 for (int p = 0; p < num_input; p++)
                 {
@@ -249,12 +249,12 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-            weight_data_pack4to1.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4);
+            weight_data_tm.create(maxk, num_input / 4, num_output, (size_t)4 * 4, 4);
 
             for (int q = 0; q < num_output; q++)
             {
                 const Mat k0 = weight_data_r2.channel(q);
-                float* g00 = weight_data_pack4to1.channel(q);
+                float* g00 = weight_data_tm.channel(q);
 
                 for (int p = 0; p + 3 < num_input; p += 4)
                 {
@@ -281,7 +281,31 @@ int Deconvolution_arm::create_pipeline(const Option& opt)
     // pack1
     if (elempack == 1 && out_elempack == 1)
     {
-        weight_data_pack1 = weight_data_transposed;
+        if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        {
+            weight_data_tm = weight_data;
+        }
+        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        {
+            weight_data_tm = weight_data;
+        }
+        else if (kernel_w == 4 && kernel_h == 4 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        {
+            weight_data_tm = weight_data;
+        }
+        else if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        {
+            weight_data_tm = weight_data;
+        }
+        else
+        {
+            weight_data_tm = weight_data_transposed;
+        }
+    }
+
+    if (opt.lightmode)
+    {
+        weight_data.release();
     }
 
     return 0;
@@ -318,9 +342,6 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
         return forward_bf16s(bottom_blob, top_blob, opt);
 #endif
 
-    // deconvolv with NxN kernel
-    // value = value + bias
-
     int w = bottom_blob.w;
     int h = bottom_blob.h;
     int channels = bottom_blob.c;
@@ -378,7 +399,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                         _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                     }
 
-                    const float* kptr = (const float*)weight_data_pack4 + maxk * channels * p * 16;
+                    const float* kptr = weight_data_tm.channel(p);
 
                     // channels
                     for (int q = 0; q < channels; q++)
@@ -462,7 +483,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                         _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                     }
 
-                    const float* kptr = (const float*)weight_data_pack1to4 + maxk * channels * p * 4;
+                    const float* kptr = weight_data_tm.channel(p);
 
                     // channels
                     for (int q = 0; q < channels; q++)
@@ -518,7 +539,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     {
         // num_output
         #pragma omp parallel for num_threads(opt.num_threads)
-        for (int p = 0; p < num_output / out_elempack; p++)
+        for (int p = 0; p < num_output; p++)
         {
             float* outptr = top_blob_bordered.channel(p);
 
@@ -533,7 +554,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                         sum = bias_data[p];
                     }
 
-                    const float* kptr = (const float*)weight_data_pack4to1 + maxk * channels * p * 4;
+                    const float* kptr = weight_data_tm.channel(p);
 
                     // channels
                     for (int q = 0; q < channels; q++)
@@ -597,7 +618,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     {
         if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
         {
-            deconv3x3s1_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt);
+            deconv3x3s1_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -606,7 +627,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
         }
         else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
         {
-            deconv3x3s2_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt);
+            deconv3x3s2_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -615,7 +636,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
         }
         else if (kernel_w == 4 && kernel_h == 4 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
         {
-            deconv4x4s1_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt);
+            deconv4x4s1_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -624,7 +645,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
         }
         else if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
         {
-            deconv4x4s2_neon(bottom_blob, top_blob_bordered, weight_data, bias_data, opt);
+            deconv4x4s2_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -650,7 +671,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                             sum = bias_data[p];
                         }
 
-                        const float* kptr = (const float*)weight_data_pack1 + maxk * channels * p;
+                        const float* kptr = (const float*)weight_data_tm + maxk * channels * p;
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -692,28 +713,7 @@ int Deconvolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                             kptr += maxk;
                         }
 
-                        if (activation_type == 1)
-                        {
-                            sum = std::max(sum, 0.f);
-                        }
-                        else if (activation_type == 2)
-                        {
-                            float slope = activation_params[0];
-                            sum = sum > 0.f ? sum : sum * slope;
-                        }
-                        else if (activation_type == 3)
-                        {
-                            float min = activation_params[0];
-                            float max = activation_params[1];
-                            if (sum < min)
-                                sum = min;
-                            if (sum > max)
-                                sum = max;
-                        }
-                        else if (activation_type == 4)
-                        {
-                            sum = static_cast<float>(1.f / (1.f + exp(-sum)));
-                        }
+                        sum = activation_ss(sum, activation_type, activation_params);
 
                         outptr[j] = sum;
                     }
@@ -768,11 +768,11 @@ int Deconvolution_arm::create_pipeline_fp16s(const Option& opt)
     {
         Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-        weight_data_fp16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);
+        weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            __fp16* g00 = weight_data_fp16.channel(q / out_elempack);
+            __fp16* g00 = weight_data_tm.channel(q / out_elempack);
 
             for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
             {
@@ -798,12 +798,17 @@ int Deconvolution_arm::create_pipeline_fp16s(const Option& opt)
     {
         if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
         {
-            ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt);
+            ncnn::cast_float32_to_float16(weight_data, weight_data_tm, opt);
         }
     }
 
     ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -863,7 +868,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -942,7 +947,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1015,7 +1020,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
                             sum = bias_data[p];
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1090,7 +1095,7 @@ int Deconvolution_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_blob, cons
                             sum = bias_data[p];
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1210,7 +1215,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1297,7 +1302,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1370,7 +1375,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1q_f16((const __fp16*)bias_data_fp16 + p * 8);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1449,7 +1454,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             sum = bias_data[p];
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1525,7 +1530,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1612,7 +1617,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1691,7 +1696,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             _sum = vld1_f16((const __fp16*)bias_data_fp16 + p * 4);
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1764,7 +1769,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             sum = bias_data[p];
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1823,7 +1828,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
     {
         if (kernel_w == 4 && kernel_h == 4 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
         {
-            deconv4x4s2_fp16sa_neon(bottom_blob, top_blob_bordered, weight_data_fp16, bias_data_fp16, opt);
+            deconv4x4s2_fp16sa_neon(bottom_blob, top_blob_bordered, weight_data_tm, bias_data_fp16, opt);
 
             if (activation)
             {
@@ -1849,7 +1854,7 @@ int Deconvolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, con
                             sum = bias_data[p];
                         }
 
-                        const __fp16* kptr = weight_data_fp16.channel(p);
+                        const __fp16* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1948,11 +1953,11 @@ int Deconvolution_arm::create_pipeline_bf16s(const Option& opt)
     {
         Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-        weight_data_bf16.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);
+        weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)2u * elempack * out_elempack, elempack * out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            unsigned short* g00 = weight_data_bf16.channel(q / out_elempack);
+            unsigned short* g00 = weight_data_tm.channel(q / out_elempack);
 
             for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
             {
@@ -1974,6 +1979,11 @@ int Deconvolution_arm::create_pipeline_bf16s(const Option& opt)
         }
     }
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -2040,7 +2050,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                         }
 
-                        const unsigned short* kptr = weight_data_bf16.channel(p);
+                        const unsigned short* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -2126,7 +2136,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
                             _sum = vld1q_f32(((const float*)bias_data) + p * 4);
                         }
 
-                        const unsigned short* kptr = weight_data_bf16.channel(p);
+                        const unsigned short* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -2199,7 +2209,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
                             sum = bias_data[p];
                         }
 
-                        const unsigned short* kptr = weight_data_bf16.channel(p);
+                        const unsigned short* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -2280,7 +2290,7 @@ int Deconvolution_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_blob, cons
                             sum = bias_data[p];
                         }
 
-                        const unsigned short* kptr = weight_data_bf16.channel(p);
+                        const unsigned short* kptr = weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)

src/layer/arm/deconvolution_arm.h

@@ -43,20 +43,10 @@ protected:
 public:
     Layer* activation;
 
-    // pack4
-    Mat weight_data_pack4;
-    Mat weight_data_pack1to4;
-    Mat weight_data_pack4to1;
-    Mat weight_data_pack1;
+    Mat weight_data_tm;
 
     // fp16
-    Mat weight_data_fp16;
     Mat bias_data_fp16;
-
-#if NCNN_BF16
-    // bf16
-    Mat weight_data_bf16;
-#endif
 };
 
 } // namespace ncnn

src/layer/arm/deconvolutiondepthwise_arm.cpp

@@ -86,7 +86,7 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
                 Mat weight_data_r2_packed;
                 convert_packing(weight_data_r2, weight_data_r2_packed, 8, opt);
 
-                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt);
             }
 
             if (elempack == 4)
@@ -95,12 +95,12 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
                 Mat weight_data_r2_packed;
                 convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt);
 
-                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data_r2_packed, weight_data_tm, opt);
             }
 
             if (elempack == 1)
             {
-                ncnn::cast_float32_to_float16(weight_data_transposed, weight_data_fp16, opt);
+                ncnn::cast_float32_to_float16(weight_data_transposed, weight_data_tm, opt);
             }
 
             ncnn::cast_float32_to_float16(bias_data, bias_data_fp16, opt);
@@ -116,15 +116,16 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
             if (elempack == 4)
             {
                 Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-                convert_packing(weight_data_r2, weight_data_pack4, 4, opt);
+                Mat weight_data_r2_packed;
+                convert_packing(weight_data_r2, weight_data_r2_packed, 4, opt);
 
-                ncnn::cast_float32_to_bfloat16(weight_data_pack4, weight_data_bf16, opt);
+                ncnn::cast_float32_to_bfloat16(weight_data_r2_packed, weight_data_tm, opt);
             }
 #endif // __ARM_NEON
 
             if (elempack == 1)
             {
-                ncnn::cast_float32_to_bfloat16(weight_data_transposed, weight_data_bf16, opt);
+                ncnn::cast_float32_to_bfloat16(weight_data_transposed, weight_data_tm, opt);
             }
 
             return 0;
@@ -136,14 +137,14 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
         if (elempack == 4)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_pack4, 4, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 4, opt);
         }
 #endif // __ARM_NEON
 
         // pack1
         if (elempack == 1)
         {
-            weight_data_pack1 = weight_data_transposed;
+            weight_data_tm = weight_data_transposed;
         }
     }
     else
@@ -161,7 +162,7 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
 
         for (int g = 0; g < group; g++)
         {
-            Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
+            Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone();
             Mat bias_data_g;
             if (bias_term)
                 bias_data_g = bias_data.range(num_output_g * g, num_output_g);
@@ -211,6 +212,11 @@ int DeconvolutionDepthWise_arm::create_pipeline(const Option& opt)
         }
     }
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -293,7 +299,7 @@ int DeconvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, c
             for (int g = 0; g < channels; g++)
             {
                 float* outptr = top_blob_bordered.channel(g);
-                const float* kptr = (const float*)weight_data_pack4 + maxk * g * 4;
+                const float* kptr = (const float*)weight_data_tm + maxk * g * 4;
                 const Mat m = bottom_blob.channel(g);
 
                 for (int i = 0; i < outh; i++)
@@ -356,7 +362,7 @@ int DeconvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, c
             for (int g = 0; g < channels; g++)
             {
                 float* outptr = top_blob_bordered.channel(g);
-                const float* kptr = (const float*)weight_data_pack1 + maxk * g;
+                const float* kptr = (const float*)weight_data_tm + maxk * g;
                 const Mat m = bottom_blob.channel(g);
 
                 for (int i = 0; i < outh; i++)
@@ -540,7 +546,7 @@ int DeconvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_b
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob_bordered.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -604,7 +610,7 @@ int DeconvolutionDepthWise_arm::forward_fp16s(const Mat& bottom_blob, Mat& top_b
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob_bordered.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -764,7 +770,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob_bordered.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 8;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -828,7 +834,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob_bordered.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g * 4;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -892,7 +898,7 @@ int DeconvolutionDepthWise_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_
                 for (int g = 0; g < channels; g++)
                 {
                     __fp16* outptr = top_blob_bordered.channel(g);
-                    const __fp16* kptr = (const __fp16*)weight_data_fp16 + maxk * g;
+                    const __fp16* kptr = (const __fp16*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1061,7 +1067,7 @@ int DeconvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_b
             for (int g = 0; g < channels; g++)
             {
                 unsigned short* outptr = top_blob_bordered.channel(g);
-                const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g * 4;
+                const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g * 4;
                 const Mat m = bottom_blob.channel(g);
 
                 for (int i = 0; i < outh; i++)
@@ -1124,7 +1130,7 @@ int DeconvolutionDepthWise_arm::forward_bf16s(const Mat& bottom_blob, Mat& top_b
             for (int g = 0; g < channels; g++)
             {
                 unsigned short* outptr = top_blob_bordered.channel(g);
-                const unsigned short* kptr = (const unsigned short*)weight_data_bf16 + maxk * g;
+                const unsigned short* kptr = (const unsigned short*)weight_data_tm + maxk * g;
                 const Mat m = bottom_blob.channel(g);
 
                 for (int i = 0; i < outh; i++)

src/layer/arm/deconvolutiondepthwise_arm.h

@@ -41,18 +41,10 @@ protected:
 public:
     std::vector<ncnn::Layer*> group_ops;
 
-    // packing
-    Mat weight_data_pack4;
-    Mat weight_data_pack1;
+    Mat weight_data_tm;
 
     // fp16
-    Mat weight_data_fp16;
     Mat bias_data_fp16;
-
-#if NCNN_BF16
-    // bf16
-    Mat weight_data_bf16;
-#endif
 };
 
 } // namespace ncnn

src/layer/arm/innerproduct_arm.h

@@ -48,21 +48,14 @@ protected:
 
 public:
     Layer* flatten;
-    Layer* activation;
+
+    Mat weight_data_tm;
 
     // fp16
-    Mat weight_data_fp16;
     Mat bias_data_fp16;
 
-#if NCNN_BF16
-    // bf16
-    Mat weight_data_bf16;
-#endif
-
 #if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
-    Mat scales_in;
+    Mat scale_in_data;
 #endif
 };
 

src/layer/vulkan/convolution_vulkan.h

@@ -36,6 +36,11 @@ public:
 public:
     ncnn::Layer* padding;
 
+    Mat weight_data_packed;
+    Mat weight_winograd23_data_packed;
+    Mat weight_winograd43_data_packed;
+    Mat bias_data_packed;
+
     VkMat weight_data_gpu;
     VkMat bias_data_gpu;
 
@@ -61,7 +66,8 @@ public:
     Pipeline* pipeline_convolution_3x3s1d1_winograd43_transform_output;
 
     // convolution as fc
-    ncnn::Layer* innerproduct;
+    ncnn::Layer* reshape_1x1xw;
+    ncnn::Layer* reshape_w;
 };
 
 } // namespace ncnn

src/layer/vulkan/convolutiondepthwise_vulkan.cpp

@@ -214,6 +214,14 @@ int ConvolutionDepthWise_vulkan::create_pipeline(const Option& _opt)
     // depth-wise
     if (channels == group && group == num_output)
     {
+        Mat weight_data_r2 = weight_data.reshape(maxk, group);
+        convert_packing(weight_data_r2, weight_data_packed, elempack, opt);
+
+        if (bias_term)
+        {
+            convert_packing(bias_data, bias_data_packed, out_elempack, opt);
+        }
+
         specializations[11 + 0].i = shape_bordered_packed.dims;
         specializations[11 + 1].i = shape_bordered_packed.w;
         specializations[11 + 2].i = shape_bordered_packed.h;
@@ -260,6 +268,51 @@ int ConvolutionDepthWise_vulkan::create_pipeline(const Option& _opt)
         return 0;
     }
 
+    // src = kw-kh-inch-outch
+    // dst = pa-pb-kw-kh-inch/pa-outch/pb
+    {
+        Mat weight_data_r2_groups = weight_data.reshape(maxk, channels_g, num_output_g * group);
+
+        weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g);
+
+        for (int g = 0; g < group; g++)
+        {
+            const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g);
+
+            Mat weight_data_packed = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g);
+
+            for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g)
+            {
+                float* g00 = weight_data_packed.channel(q / out_elempack_g);
+
+                for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g)
+                {
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        for (int i = 0; i < out_elempack_g; i++)
+                        {
+                            const Mat k0 = weight_data_r2.channel(q + i);
+
+                            for (int j = 0; j < elempack_g; j++)
+                            {
+                                const float* k00 = k0.row(p + j);
+
+                                g00[0] = k00[k];
+
+                                g00++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    if (bias_term)
+    {
+        convert_packing(bias_data, bias_data_packed, out_elempack_g, opt);
+    }
+
     specializations[11 + 0].i = shape_bordered_g_packed.dims;
     specializations[11 + 1].i = shape_bordered_g_packed.w;
     specializations[11 + 2].i = shape_bordered_g_packed.h;
@@ -412,16 +465,9 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt
     const int maxk = kernel_w * kernel_h;
     int channels = (weight_data_size / group) / maxk / (num_output / group) * group;
 
-    int elempack = opt.use_shader_pack8 && channels % 8 == 0 ? 8 : channels % 4 == 0 ? 4 : 1;
-    int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
-
     // depth-wise
     if (channels == group && group == num_output)
     {
-        Mat weight_data_packed;
-        Mat weight_data_r2 = weight_data.reshape(maxk, group);
-        convert_packing(weight_data_r2, weight_data_packed, elempack, opt);
-
         if (support_image_storage && opt.use_image_storage)
         {
             cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt);
@@ -431,11 +477,10 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt
             cmd.record_upload(weight_data_packed, weight_data_gpu, opt);
         }
 
+        weight_data_packed.release();
+
         if (bias_term)
         {
-            Mat bias_data_packed;
-            convert_packing(bias_data, bias_data_packed, out_elempack, opt);
-
             if (support_image_storage && opt.use_image_storage)
             {
                 cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
@@ -444,59 +489,13 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt
             {
                 cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
             }
+
+            bias_data_packed.release();
         }
 
         return 0;
     }
 
-    // group convolution
-    const int channels_g = channels / group;
-    const int num_output_g = num_output / group;
-
-    int elempack_g = opt.use_shader_pack8 && channels_g % 8 == 0 ? 8 : channels_g % 4 == 0 ? 4 : 1;
-    int out_elempack_g = opt.use_shader_pack8 && num_output_g % 8 == 0 ? 8 : num_output_g % 4 == 0 ? 4 : 1;
-
-    // src = kw-kh-inch-outch
-    // dst = pa-pb-kw-kh-inch/pa-outch/pb
-    Mat weight_data_packed_groups;
-    {
-        Mat weight_data_r2_groups = weight_data.reshape(maxk, channels_g, num_output_g * group);
-
-        weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g);
-
-        for (int g = 0; g < group; g++)
-        {
-            const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g);
-
-            Mat weight_data_packed = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g);
-
-            for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g)
-            {
-                float* g00 = weight_data_packed.channel(q / out_elempack_g);
-
-                for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g)
-                {
-                    for (int k = 0; k < maxk; k++)
-                    {
-                        for (int i = 0; i < out_elempack_g; i++)
-                        {
-                            const Mat k0 = weight_data_r2.channel(q + i);
-
-                            for (int j = 0; j < elempack_g; j++)
-                            {
-                                const float* k00 = k0.row(p + j);
-
-                                g00[0] = k00[k];
-
-                                g00++;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
     if (support_image_storage && opt.use_image_storage)
     {
         cmd.record_upload(weight_data_packed_groups, weight_data_gpu_image, opt);
@@ -506,11 +505,10 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt
         cmd.record_upload(weight_data_packed_groups, weight_data_gpu, opt);
     }
 
+    weight_data_packed_groups.release();
+
     if (bias_term)
     {
-        Mat bias_data_packed;
-        convert_packing(bias_data, bias_data_packed, out_elempack_g, opt);
-
         if (support_image_storage && opt.use_image_storage)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
@@ -519,6 +517,8 @@ int ConvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& opt
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
         }
+
+        bias_data_packed.release();
     }
 
     return 0;

src/layer/vulkan/convolutiondepthwise_vulkan.h

@@ -34,6 +34,10 @@ public:
     virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const;
 
 public:
+    Mat weight_data_packed;
+    Mat weight_data_packed_groups;
+    Mat bias_data_packed;
+
     VkMat weight_data_gpu;
     VkMat bias_data_gpu;
 

src/layer/vulkan/deconvolution_vulkan.cpp

@@ -94,8 +94,8 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt)
     }
 
     // check weight shape
-    Mat weight_data_packed(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
-    if (!vkdev->shape_support_image_storage(weight_data_packed))
+    Mat weight_data_packed_shape(maxk, num_input / elempack, num_output / out_elempack, (void*)0, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
+    if (!vkdev->shape_support_image_storage(weight_data_packed_shape))
     {
         support_image_storage = false;
         opt.use_image_storage = false;
@@ -139,10 +139,76 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt)
         output_crop->create_pipeline(opt);
     }
 
+    if (bias_term)
+    {
+        convert_packing(bias_data, bias_data_packed, out_elempack, opt);
+    }
+
     if (opt.use_sgemm_convolution)
     {
         bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0;
 
+        // src = kw-kh-inch-outch
+        // dst = pa-pb-inch/pa-kw-kh-outch/pb (sgemm)
+        if (use_cooperative_matrix)
+        {
+            // dst = 8a-8b-inch/8a-maxk-outch/8b
+            // dst = 16a-16b-inch/16a-maxk-outch/16b
+            Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+            weight_data_packed.create(num_input / 8, maxk * num_output / 8, (size_t)4 * 8 * 8, 8 * 8);
+
+            for (int q = 0; q + 7 < num_output; q += 8)
+            {
+                for (int k = 0; k < maxk; k++)
+                {
+                    float* g00 = weight_data_packed.row(q / 8 * maxk + k);
+
+                    for (int p = 0; p + 7 < num_input; p += 8)
+                    {
+                        for (int i = 0; i < 8; i++)
+                        {
+                            for (int j = 0; j < 8; j++)
+                            {
+                                const float* k00 = weight_data_r2.channel(q + j).row(p + i);
+                                g00[0] = k00[k];
+                                g00++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        else
+        {
+            Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
+
+            weight_data_packed.create(num_input / elempack, maxk * num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
+
+            for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
+            {
+                for (int k = 0; k < maxk; k++)
+                {
+                    float* g00 = weight_data_packed.row(q / out_elempack * maxk + k);
+
+                    for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
+                    {
+                        for (int i = 0; i < out_elempack; i++)
+                        {
+                            const Mat k0 = weight_data_r2.channel(q + i);
+
+                            for (int j = 0; j < elempack; j++)
+                            {
+                                const float* k00 = k0.row(p + j);
+                                g00[0] = k00[k];
+                                g00++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         Mat out_shape_col;
         if (shape.dims != 0 && out_shape.dims != 0)
         {
@@ -249,6 +315,54 @@ int Deconvolution_vulkan::create_pipeline(const Option& _opt)
         return 0;
     }
 
+    Mat weight_data_transposed(weight_data.w);
+    {
+        float* pt = weight_data_transposed;
+        const float* p = weight_data;
+
+        for (int i = 0; i < num_input * num_output; i++)
+        {
+            for (int k = 0; k < maxk; k++)
+            {
+                pt[maxk - 1 - k] = p[k];
+            }
+
+            p += maxk;
+            pt += maxk;
+        }
+    }
+
+    // src = kw-kh-inch-outch
+    // dst = pa-pb-kw-kh-inch/pa-outch/pb
+    {
+        Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
+
+        weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
+
+        for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
+        {
+            float* g00 = weight_data_packed.channel(q / out_elempack);
+
+            for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
+            {
+                for (int k = 0; k < maxk; k++)
+                {
+                    for (int i = 0; i < out_elempack; i++)
+                    {
+                        const Mat k0 = weight_data_r2.channel(q + i);
+
+                        for (int j = 0; j < elempack; j++)
+                        {
+                            const float* k00 = k0.row(p + j);
+                            g00[0] = k00[k];
+                            g00++;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     std::vector<vk_specialization_type> specializations(10 + 10);
     specializations[0].i = kernel_w;
     specializations[1].i = kernel_h;
@@ -337,136 +451,6 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
         output_crop->upload_model(cmd, opt);
     }
 
-    const int maxk = kernel_w * kernel_h;
-    int num_input = weight_data_size / maxk / num_output;
-
-    int elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
-    int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
-
-    Mat weight_data_transposed(weight_data.w);
-    if (opt.use_sgemm_convolution)
-    {
-        weight_data_transposed = weight_data;
-    }
-    else
-    {
-        float* pt = weight_data_transposed;
-        const float* p = weight_data;
-
-        for (int i = 0; i < num_input * num_output; i++)
-        {
-            for (int k = 0; k < maxk; k++)
-            {
-                pt[maxk - 1 - k] = p[k];
-            }
-
-            p += maxk;
-            pt += maxk;
-        }
-    }
-
-    // src = kw-kh-inch-outch
-    // dst = pa-pb-kw-kh-inch/pa-outch/pb
-    // dst = pa-pb-inch/pa-kw-kh-outch/pb (sgemm)
-    Mat weight_data_packed;
-    if (opt.use_sgemm_convolution)
-    {
-        bool use_cooperative_matrix = vkdev->info.support_cooperative_matrix_16_8_8() && opt.use_cooperative_matrix && !opt.use_image_storage && !opt.use_shader_pack8 && opt.use_fp16_storage && num_input % 8 == 0 && num_output % 8 == 0;
-        if (use_cooperative_matrix)
-        {
-            // dst = 8a-8b-inch/8a-maxk-outch/8b
-            // dst = 16a-16b-inch/16a-maxk-outch/16b
-            Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
-
-            weight_data_packed.create(num_input / 8, maxk * num_output / 8, (size_t)4 * 8 * 8, 8 * 8);
-
-            for (int q = 0; q + 7 < num_output; q += 8)
-            {
-                for (int k = 0; k < maxk; k++)
-                {
-                    float* g00 = weight_data_packed.row(q / 8 * maxk + k);
-
-                    for (int p = 0; p + 7 < num_input; p += 8)
-                    {
-                        for (int i = 0; i < 8; i++)
-                        {
-                            for (int j = 0; j < 8; j++)
-                            {
-                                const float* k00 = weight_data_r2.channel(q + j).row(p + i);
-
-                                g00[0] = k00[k];
-
-                                g00++;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        else
-        {
-            Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
-
-            weight_data_packed.create(num_input / elempack, maxk * num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
-
-            for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
-            {
-                for (int k = 0; k < maxk; k++)
-                {
-                    float* g00 = weight_data_packed.row(q / out_elempack * maxk + k);
-
-                    for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
-                    {
-                        for (int i = 0; i < out_elempack; i++)
-                        {
-                            const Mat k0 = weight_data_r2.channel(q + i);
-
-                            for (int j = 0; j < elempack; j++)
-                            {
-                                const float* k00 = k0.row(p + j);
-
-                                g00[0] = k00[k];
-
-                                g00++;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    else
-    {
-        Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
-
-        weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4 * elempack * out_elempack, elempack * out_elempack);
-
-        for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
-        {
-            float* g00 = weight_data_packed.channel(q / out_elempack);
-
-            for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
-            {
-                for (int k = 0; k < maxk; k++)
-                {
-                    for (int i = 0; i < out_elempack; i++)
-                    {
-                        const Mat k0 = weight_data_r2.channel(q + i);
-
-                        for (int j = 0; j < elempack; j++)
-                        {
-                            const float* k00 = k0.row(p + j);
-
-                            g00[0] = k00[k];
-
-                            g00++;
-                        }
-                    }
-                }
-            }
-        }
-    }
-
     if (support_image_storage && opt.use_image_storage)
     {
         cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt);
@@ -476,11 +460,10 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
         cmd.record_upload(weight_data_packed, weight_data_gpu, opt);
     }
 
+    weight_data_packed.release();
+
     if (bias_term)
     {
-        Mat bias_data_packed;
-        convert_packing(bias_data, bias_data_packed, out_elempack, opt);
-
         if (support_image_storage && opt.use_image_storage)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
@@ -489,6 +472,8 @@ int Deconvolution_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
         }
+
+        bias_data_packed.release();
     }
 
     return 0;

src/layer/vulkan/deconvolution_vulkan.h

@@ -34,6 +34,9 @@ public:
     virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const;
 
 public:
+    Mat weight_data_packed;
+    Mat bias_data_packed;
+
     VkMat weight_data_gpu;
     VkMat bias_data_gpu;
 

src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp

@@ -198,6 +198,23 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt)
         output_crop->create_pipeline(opt);
     }
 
+    Mat weight_data_transposed(weight_data.w);
+    {
+        float* pt = weight_data_transposed;
+        const float* p = weight_data;
+
+        for (int i = 0; i < (channels / group) * (num_output / group) * group; i++)
+        {
+            for (int k = 0; k < maxk; k++)
+            {
+                pt[maxk - 1 - k] = p[k];
+            }
+
+            p += maxk;
+            pt += maxk;
+        }
+    }
+
     std::vector<vk_specialization_type> specializations(11 + 10);
     specializations[0].i = kernel_w;
     specializations[1].i = kernel_h;
@@ -214,6 +231,14 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt)
     // depth-wise
     if (channels == group && group == num_output)
     {
+        Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
+        convert_packing(weight_data_r2, weight_data_packed, elempack, opt);
+
+        if (bias_term)
+        {
+            convert_packing(bias_data, bias_data_packed, out_elempack, opt);
+        }
+
         specializations[11 + 0].i = shape_packed.dims;
         specializations[11 + 1].i = shape_packed.w;
         specializations[11 + 2].i = shape_packed.h;
@@ -260,6 +285,51 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& _opt)
         return 0;
     }
 
+    // src = kw-kh-inch-outch
+    // dst = pa-pb-kw-kh-inch/pa-outch/pb
+    {
+        Mat weight_data_r2_groups = weight_data_transposed.reshape(maxk, channels_g, num_output_g * group);
+
+        weight_data_packed.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g);
+
+        for (int g = 0; g < group; g++)
+        {
+            const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g);
+
+            Mat weight_data_pack4 = weight_data_packed.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g);
+
+            for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g)
+            {
+                float* g00 = weight_data_pack4.channel(q / out_elempack_g);
+
+                for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g)
+                {
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        for (int i = 0; i < out_elempack_g; i++)
+                        {
+                            const Mat k0 = weight_data_r2.channel(q + i);
+
+                            for (int j = 0; j < elempack_g; j++)
+                            {
+                                const float* k00 = k0.row(p + j);
+
+                                g00[0] = k00[k];
+
+                                g00++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    if (bias_term)
+    {
+        convert_packing(bias_data, bias_data_packed, out_elempack_g, opt);
+    }
+
     specializations[11 + 0].i = shape_g_packed.dims;
     specializations[11 + 1].i = shape_g_packed.w;
     specializations[11 + 2].i = shape_g_packed.h;
@@ -421,125 +491,19 @@ int DeconvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& o
         output_crop->upload_model(cmd, opt);
     }
 
-    const int maxk = kernel_w * kernel_h;
-    int channels = (weight_data_size / group) / maxk / (num_output / group) * group;
-
-    int elempack = opt.use_shader_pack8 && channels % 8 == 0 ? 8 : channels % 4 == 0 ? 4 : 1;
-    int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
-
-    Mat weight_data_transposed(weight_data.w);
-    {
-        float* pt = weight_data_transposed;
-        const float* p = weight_data;
-
-        for (int i = 0; i < (channels / group) * (num_output / group) * group; i++)
-        {
-            for (int k = 0; k < maxk; k++)
-            {
-                pt[maxk - 1 - k] = p[k];
-            }
-
-            p += maxk;
-            pt += maxk;
-        }
-    }
-
-    // depth-wise
-    if (channels == group && group == num_output)
-    {
-        Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-        Mat weight_data_r2_packed;
-        convert_packing(weight_data_r2, weight_data_r2_packed, elempack, opt);
-
-        if (support_image_storage && opt.use_image_storage)
-        {
-            cmd.record_upload(weight_data_r2_packed, weight_data_gpu_image, opt);
-        }
-        else
-        {
-            cmd.record_upload(weight_data_r2_packed, weight_data_gpu, opt);
-        }
-
-        if (bias_term)
-        {
-            Mat bias_data_packed;
-            convert_packing(bias_data, bias_data_packed, out_elempack, opt);
-
-            if (support_image_storage && opt.use_image_storage)
-            {
-                cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
-            }
-            else
-            {
-                cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
-            }
-        }
-
-        return 0;
-    }
-
-    // group deconvolution
-    const int channels_g = channels / group;
-    const int num_output_g = num_output / group;
-
-    int elempack_g = opt.use_shader_pack8 && channels_g % 8 == 0 ? 8 : channels_g % 4 == 0 ? 4 : 1;
-    int out_elempack_g = opt.use_shader_pack8 && num_output_g % 8 == 0 ? 8 : num_output_g % 4 == 0 ? 4 : 1;
-
-    // src = kw-kh-inch-outch
-    // dst = pa-pb-kw-kh-inch/pa-outch/pb
-    Mat weight_data_packed_groups;
-    {
-        Mat weight_data_r2_groups = weight_data_transposed.reshape(maxk, channels_g, num_output_g * group);
-
-        weight_data_packed_groups.create(maxk, channels_g / elempack_g, num_output_g / out_elempack_g * group, (size_t)4 * elempack_g * out_elempack_g, elempack_g * out_elempack_g);
-
-        for (int g = 0; g < group; g++)
-        {
-            const Mat weight_data_r2 = weight_data_r2_groups.channel_range(num_output_g * g, num_output_g);
-
-            Mat weight_data_pack4 = weight_data_packed_groups.channel_range(num_output_g / out_elempack_g * g, num_output_g / out_elempack_g);
-
-            for (int q = 0; q + (out_elempack_g - 1) < num_output_g; q += out_elempack_g)
-            {
-                float* g00 = weight_data_pack4.channel(q / out_elempack_g);
-
-                for (int p = 0; p + (elempack_g - 1) < channels_g; p += elempack_g)
-                {
-                    for (int k = 0; k < maxk; k++)
-                    {
-                        for (int i = 0; i < out_elempack_g; i++)
-                        {
-                            const Mat k0 = weight_data_r2.channel(q + i);
-
-                            for (int j = 0; j < elempack_g; j++)
-                            {
-                                const float* k00 = k0.row(p + j);
-
-                                g00[0] = k00[k];
-
-                                g00++;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
     if (support_image_storage && opt.use_image_storage)
     {
-        cmd.record_upload(weight_data_packed_groups, weight_data_gpu_image, opt);
+        cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt);
     }
     else
     {
-        cmd.record_upload(weight_data_packed_groups, weight_data_gpu, opt);
+        cmd.record_upload(weight_data_packed, weight_data_gpu, opt);
     }
 
+    weight_data_packed.release();
+
     if (bias_term)
     {
-        Mat bias_data_packed;
-        convert_packing(bias_data, bias_data_packed, out_elempack_g, opt);
-
         if (support_image_storage && opt.use_image_storage)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
@@ -548,6 +512,8 @@ int DeconvolutionDepthWise_vulkan::upload_model(VkTransfer& cmd, const Option& o
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
         }
+
+        bias_data_packed.release();
     }
 
     return 0;

src/layer/vulkan/deconvolutiondepthwise_vulkan.h

@@ -34,6 +34,9 @@ public:
     virtual int forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, VkCompute& cmd, const Option& opt) const;
 
 public:
+    Mat weight_data_packed;
+    Mat bias_data_packed;
+
     VkMat weight_data_gpu;
     VkMat bias_data_gpu;
 

src/layer/vulkan/innerproduct_vulkan.cpp

@@ -45,6 +45,40 @@ int InnerProduct_vulkan::create_pipeline(const Option& _opt)
     int in_elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
     int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
 
+    // src = inch-outch
+    // dst = pa-pb-inch/pa-outch/pb
+    {
+        Mat weight_data_r2 = weight_data.reshape(num_input, num_output);
+
+        weight_data_packed.create(num_input / in_elempack, num_output / out_elempack, (size_t)4 * in_elempack * out_elempack, in_elempack * out_elempack);
+
+        for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
+        {
+            float* g00 = weight_data_packed.row(q / out_elempack);
+
+            for (int p = 0; p + (in_elempack - 1) < num_input; p += in_elempack)
+            {
+                for (int i = 0; i < out_elempack; i++)
+                {
+                    const float* k0 = weight_data_r2.row(q + i);
+                    k0 += p;
+
+                    for (int j = 0; j < in_elempack; j++)
+                    {
+                        g00[0] = k0[j];
+
+                        g00++;
+                    }
+                }
+            }
+        }
+    }
+
+    if (bias_term)
+    {
+        convert_packing(bias_data, bias_data_packed, out_elempack, opt);
+    }
+
     if (shape.dims == 2 && shape.w == num_input && shape.h > 1)
     {
         // gemm
@@ -358,41 +392,6 @@ int InnerProduct_vulkan::destroy_pipeline(const Option& opt)
 
 int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
 {
-    const int num_input = weight_data_size / num_output;
-
-    int in_elempack = opt.use_shader_pack8 && num_input % 8 == 0 ? 8 : num_input % 4 == 0 ? 4 : 1;
-    int out_elempack = opt.use_shader_pack8 && num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
-
-    // src = inch-outch
-    // dst = pa-pb-inch/pa-outch/pb
-    Mat weight_data_packed;
-    {
-        Mat weight_data_r2 = weight_data.reshape(num_input, num_output);
-
-        weight_data_packed.create(num_input / in_elempack, num_output / out_elempack, (size_t)4 * in_elempack * out_elempack, in_elempack * out_elempack);
-
-        for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
-        {
-            float* g00 = weight_data_packed.row(q / out_elempack);
-
-            for (int p = 0; p + (in_elempack - 1) < num_input; p += in_elempack)
-            {
-                for (int i = 0; i < out_elempack; i++)
-                {
-                    const float* k0 = weight_data_r2.row(q + i);
-                    k0 += p;
-
-                    for (int j = 0; j < in_elempack; j++)
-                    {
-                        g00[0] = k0[j];
-
-                        g00++;
-                    }
-                }
-            }
-        }
-    }
-
     if (support_image_storage && opt.use_image_storage)
     {
         cmd.record_upload(weight_data_packed, weight_data_gpu_image, opt);
@@ -402,11 +401,10 @@ int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
         cmd.record_upload(weight_data_packed, weight_data_gpu, opt);
     }
 
+    weight_data_packed.release();
+
     if (bias_term)
     {
-        Mat bias_data_packed;
-        convert_packing(bias_data, bias_data_packed, out_elempack, opt);
-
         if (support_image_storage && opt.use_image_storage)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu_image, opt);
@@ -415,6 +413,8 @@ int InnerProduct_vulkan::upload_model(VkTransfer& cmd, const Option& opt)
         {
             cmd.record_upload(bias_data_packed, bias_data_gpu, opt);
         }
+
+        bias_data_packed.release();
     }
 
     return 0;

src/layer/vulkan/innerproduct_vulkan.h

@@ -36,6 +36,9 @@ public:
 public:
     ncnn::Layer* flatten;
 
+    Mat weight_data_packed;
+    Mat bias_data_packed;
+
     VkMat weight_data_gpu;
     VkMat bias_data_gpu;
 

src/layer/x86/convolution_1x1.h

@@ -205,3 +205,54 @@ static void conv1x1s2_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& _ker
         }
     }
 }
+
+static void conv1x1s1_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    const int size = w * h;
+
+    Mat bottom_im2col = bottom_blob;
+    bottom_im2col.w = size;
+    bottom_im2col.h = 1;
+
+    im2col_sgemm_sse(bottom_im2col, top_blob, kernel, _bias, opt);
+}
+
+static void conv1x1s2_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+    int elempack = bottom_blob.elempack;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+
+    const int tailstep = w - 2 * outw + w;
+
+    Mat bottom_blob_shrinked;
+    bottom_blob_shrinked.create(outw, outh, channels, elemsize, elempack, opt.workspace_allocator);
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = 0; p < channels; p++)
+    {
+        const float* r0 = bottom_blob.channel(p);
+        float* outptr = bottom_blob_shrinked.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                outptr[0] = r0[0];
+
+                r0 += 2;
+                outptr += 1;
+            }
+
+            r0 += tailstep;
+        }
+    }
+
+    conv1x1s1_sgemm_sse(bottom_blob_shrinked, top_blob, kernel, _bias, opt);
+}

src/layer/x86/convolution_x86.cpp

@@ -140,7 +140,7 @@ Convolution_x86::Convolution_x86()
     convolution_dilation1 = 0;
 }
 
-static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat& weight_data_packed, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack)
+static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack)
 {
     const int maxk = kernel_w * kernel_h;
 
@@ -149,11 +149,11 @@ static void convolution_transform_kernel_packed_sse(const Mat& weight_data, Mat&
     {
         Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
 
-        weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);
+        weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            float* g00 = weight_data_packed.channel(q / out_elempack);
+            float* g00 = weight_data_tm.channel(q / out_elempack);
 
             for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
             {
@@ -232,6 +232,11 @@ int Convolution_x86::create_pipeline(const Option& opt)
 
         convolution_dilation1->create_pipeline(opt);
 
+        if (opt.lightmode)
+        {
+            weight_data.release();
+        }
+
         return 0;
     }
 
@@ -278,7 +283,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -298,7 +303,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -318,7 +323,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -338,7 +343,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -358,7 +363,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -378,7 +383,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -400,7 +405,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
         else if (opt.use_sgemm_convolution)
@@ -409,7 +414,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -435,12 +440,12 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
         else if (kernel_w == 2 && kernel_h == 2 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
         else if (opt.use_sgemm_convolution)
         {
@@ -448,7 +453,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -469,7 +474,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -486,11 +491,11 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
         else if (opt.use_sgemm_convolution)
         {
@@ -498,7 +503,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -519,7 +524,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -542,7 +547,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
         else if (opt.use_sgemm_convolution)
@@ -551,7 +556,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 #endif // __AVX__
@@ -584,7 +589,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
     }
@@ -602,11 +607,11 @@ int Convolution_x86::create_pipeline(const Option& opt)
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
         else
         {
@@ -620,7 +625,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
     }
@@ -652,7 +657,7 @@ int Convolution_x86::create_pipeline(const Option& opt)
             }
             else
             {
-                convolution_transform_kernel_packed_sse(weight_data, weight_data_packed, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+                convolution_transform_kernel_packed_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
             }
         }
     }
@@ -661,7 +666,15 @@ int Convolution_x86::create_pipeline(const Option& opt)
     // pack1
     if (elempack == 1 && out_elempack == 1)
     {
-        if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        {
+            convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
+        }
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
+        }
+        else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             if (num_input >= 16 && num_output >= 16)
             {
@@ -676,6 +689,15 @@ int Convolution_x86::create_pipeline(const Option& opt)
         {
             convolution_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
         }
+        else
+        {
+            weight_data_tm = weight_data;
+        }
+    }
+
+    if (opt.lightmode)
+    {
+        weight_data.release();
     }
 
     return 0;
@@ -702,19 +724,51 @@ int Convolution_x86::destroy_pipeline(const Option& opt)
 
 int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
-    // convolv with NxN kernel
-    // value = value + bias
-
 #if NCNN_INT8
-    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    if (opt.use_int8_inference && int8_scale_term)
     {
         return forward_int8_x86(bottom_blob, top_blob, opt);
     }
 #endif
 
-    if (bottom_blob.dims != 3)
+    // flattened blob, implement as InnerProduct
+    if (bottom_blob.dims == 1 && kernel_w == 1 && kernel_h == 1)
     {
-        return Convolution::forward(bottom_blob, top_blob, opt);
+        Mat bottom_blob_3d;
+        if (bottom_blob.elemsize % 16 == 0)
+        {
+            bottom_blob_3d = bottom_blob;
+            bottom_blob_3d.dims = 3;
+            bottom_blob_3d.w = 1;
+            bottom_blob_3d.h = 1;
+            bottom_blob_3d.c = bottom_blob.w;
+            bottom_blob_3d.cstep = 1;
+        }
+        else
+        {
+            bottom_blob_3d = bottom_blob.reshape(1, 1, bottom_blob.w, opt.workspace_allocator);
+        }
+
+        Mat top_blob_3d;
+        int ret = forward(bottom_blob_3d, top_blob_3d, opt);
+        if (ret != 0)
+            return ret;
+
+        if (top_blob_3d.elemsize % 16 == 0)
+        {
+            top_blob = top_blob_3d;
+            top_blob.dims = 1;
+            top_blob.w = top_blob_3d.c;
+            top_blob.h = 1;
+            top_blob.c = 1;
+            bottom_blob_3d.cstep = top_blob_3d.c;
+        }
+        else
+        {
+            top_blob = top_blob_3d.reshape(top_blob_3d.c, opt.blob_allocator);
+        }
+
+        return 0;
     }
 
     int w = bottom_blob.w;
@@ -816,7 +870,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -851,7 +905,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack8to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack8to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -886,7 +940,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack16to8_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack16to8_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -921,7 +975,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack4to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack4to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -956,7 +1010,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack16to4_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack16to4_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -991,7 +1045,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack1to16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack1to16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1024,7 +1078,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                conv3x3s1_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                conv3x3s1_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             }
 
             if (activation)
@@ -1043,7 +1097,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack16to1_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1085,7 +1139,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                conv3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                conv3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             }
 
             if (activation)
@@ -1095,7 +1149,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else if (kernel_w == 2 && kernel_h == 2 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            conv2x2s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            conv2x2s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1113,7 +1167,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1139,7 +1193,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            conv3x3s1_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            conv3x3s1_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1148,7 +1202,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            conv3x3s2_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            conv3x3s2_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1166,7 +1220,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack1to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1201,7 +1255,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack4to8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack4to8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1234,7 +1288,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                conv3x3s1_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                conv3x3s1_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
             }
 
             if (activation)
@@ -1253,7 +1307,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack8to1_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
@@ -1288,7 +1342,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else
         {
-            convolution_pack8to4_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            convolution_pack8to4_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 #endif // __AVX__
@@ -1347,7 +1401,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                convolution_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+                convolution_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
             }
         }
     }
@@ -1374,7 +1428,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            conv3x3s1_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            conv3x3s1_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1383,7 +1437,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            conv3x3s2_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            conv3x3s2_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1407,7 +1461,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                convolution_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+                convolution_pack1to4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
             }
         }
     }
@@ -1437,7 +1491,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             // TODO more proper condition
             conv3x3s1_winograd63_pack4to1_sse(bottom_blob_bordered, top_blob, weight_winograd63_data, bias_data, opt);
 
-            // conv3x3s1_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+            // conv3x3s1_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
             if (activation)
             {
@@ -1461,7 +1515,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
             }
             else
             {
-                convolution_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+                convolution_pack4to1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
             }
         }
     }
@@ -1469,7 +1523,25 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 
     if (elempack == 1 && out_elempack == 1)
     {
-        if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        {
+            conv1x1s1_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            conv1x1s2_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+        }
+        else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             if (num_input >= 16 && num_output >= 16)
             {
@@ -1533,7 +1605,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                             sum = bias_data[p];
                         }
 
-                        const float* kptr = (const float*)weight_data + maxk * channels * p;
+                        const float* kptr = (const float*)weight_data_tm + maxk * channels * p;
 
                         // channels
                         for (int q = 0; q < channels; q++)
@@ -1640,7 +1712,7 @@ int Convolution_x86::forward(const std::vector<Mat>& bottom_blobs, std::vector<M
 }
 
 #if NCNN_INT8
-static void convolution_transform_kernel_packed_int8_sse(const Mat& weight_data, Mat& weight_data_int8, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack)
+static void convolution_transform_kernel_packed_int8_sse(const Mat& weight_data, Mat& weight_data_tm, int num_input, int num_output, int kernel_w, int kernel_h, int elempack, int out_elempack)
 {
     const int maxk = kernel_w * kernel_h;
 
@@ -1649,11 +1721,11 @@ static void convolution_transform_kernel_packed_int8_sse(const Mat& weight_data,
     {
         Mat weight_data_r2 = weight_data.reshape(maxk, num_input, num_output);
 
-        weight_data_int8.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)elempack * out_elempack, elempack * out_elempack);
+        weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)elempack * out_elempack, elempack * out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            signed char* g00 = weight_data_int8.channel(q / out_elempack);
+            signed char* g00 = weight_data_tm.channel(q / out_elempack);
 
             for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
             {
@@ -1712,7 +1784,7 @@ int Convolution_x86::create_pipeline_int8_x86(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_int8, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -1744,7 +1816,7 @@ int Convolution_x86::create_pipeline_int8_x86(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_int8, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 
@@ -1768,19 +1840,13 @@ int Convolution_x86::create_pipeline_int8_x86(const Option& opt)
         }
         else
         {
-            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_int8, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
+            convolution_transform_kernel_packed_int8_sse(weight_data, weight_data_tm, num_input, num_output, kernel_w, kernel_h, elempack, out_elempack);
         }
     }
 #endif // __SSE2__
 
     if (elempack == 1 && out_elempack == 1)
     {
-        if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16)
-        {
-            conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_winograd23_data, num_input, num_output, opt);
-            // conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_winograd43_data, num_input, num_output, opt);
-        }
-
         if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
@@ -1789,12 +1855,37 @@ int Convolution_x86::create_pipeline_int8_x86(const Option& opt)
         {
             convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
         }
+        else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16)
+        {
+            conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_winograd23_data, num_input, num_output, opt);
+            // conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_winograd43_data, num_input, num_output, opt);
+        }
         else if (opt.use_sgemm_convolution)
         {
             convolution_im2col_sgemm_transform_kernel_int8_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_w, kernel_h);
         }
+        else
+        {
+            weight_data_tm = weight_data;
+        }
+    }
 
-        return 0;
+    scale_in_data.create(num_output);
+    for (int p = 0; p < num_output; p++)
+    {
+        // requantize and relu
+        float scale_in;
+        if (weight_data_int8_scales[p] == 0)
+            scale_in = 0;
+        else
+            scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
+
+        scale_in_data[p] = scale_in;
+    }
+
+    if (opt.lightmode)
+    {
+        weight_data.release();
     }
 
     return 0;
@@ -1885,34 +1976,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con
         }
         else
         {
-            convolution_pack8to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
-        }
-
-        Mat scale_in_data(num_output);
-        for (int p = 0; p < num_output; p++)
-        {
-            // requantize and relu
-            float scale_in;
-            if (weight_data_int8_scales[p] == 0)
-                scale_in = 0;
-            else
-                scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
-
-            scale_in_data[p] = scale_in;
-        }
-
-        if (use_int8_requantize)
-        {
-            requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);
-        }
-        else
-        {
-            dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt);
-
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
-            }
+            convolution_pack8to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
         }
     }
 
@@ -1944,34 +2008,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con
         }
         else
         {
-            convolution_pack1to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
-        }
-
-        Mat scale_in_data(num_output);
-        for (int p = 0; p < num_output; p++)
-        {
-            // requantize and relu
-            float scale_in;
-            if (weight_data_int8_scales[p] == 0)
-                scale_in = 0;
-            else
-                scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
-
-            scale_in_data[p] = scale_in;
-        }
-
-        if (use_int8_requantize)
-        {
-            requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);
-        }
-        else
-        {
-            dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt);
-
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
-            }
+            convolution_pack1to4_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
         }
     }
 
@@ -1995,34 +2032,7 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con
         }
         else
         {
-            convolution_pack8to1_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
-        }
-
-        Mat scale_in_data(num_output);
-        for (int p = 0; p < num_output; p++)
-        {
-            // requantize and relu
-            float scale_in;
-            if (weight_data_int8_scales[p] == 0)
-                scale_in = 0;
-            else
-                scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
-
-            scale_in_data[p] = scale_in;
-        }
-
-        if (use_int8_requantize)
-        {
-            requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);
-        }
-        else
-        {
-            dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt);
-
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
-            }
+            convolution_pack8to1_int8_sse(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
         }
     }
 #endif // __SSE2__
@@ -2048,35 +2058,21 @@ int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, con
         }
         else
         {
-            //         convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_int8, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
-            convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
+            convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
         }
+    }
 
-        Mat scale_in_data(num_output);
-        for (int p = 0; p < num_output; p++)
-        {
-            // requantize and relu
-            float scale_in;
-            if (weight_data_int8_scales[p] == 0)
-                scale_in = 0;
-            else
-                scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
-
-            scale_in_data[p] = scale_in;
-        }
+    if (use_int8_requantize)
+    {
+        requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);
+    }
+    else
+    {
+        dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt);
 
-        if (use_int8_requantize)
+        if (activation)
         {
-            requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);
-        }
-        else
-        {
-            dequantize_from_int32(top_blob_int32, top_blob, scale_in_data, bias_data, opt);
-
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
-            }
+            activation->forward_inplace(top_blob, opt);
         }
     }
 

src/layer/x86/convolution_x86.h

@@ -41,6 +41,7 @@ protected:
 public:
     Layer* activation;
 
+    Mat weight_data_tm;
     Mat weight_sgemm_data;
     Mat weight_winograd23_data;
     Mat weight_winograd43_data;
@@ -49,12 +50,8 @@ public:
     // forwardDilation
     Layer* convolution_dilation1;
 
-    // pack4/8
-    Mat weight_data_packed;
-
 #if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
+    Mat scale_in_data;
 #endif
 };
 

src/layer/x86/convolutiondepthwise_x86.cpp

@@ -95,7 +95,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 16)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 16, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 16, opt);
 
             return 0;
         }
@@ -105,7 +105,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 8)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 8, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 8, opt);
 
             return 0;
         }
@@ -115,7 +115,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 4)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 4, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 4, opt);
 
             return 0;
         }
@@ -126,10 +126,12 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
             // depth-wise specific
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
+                weight_data_tm = weight_data;
                 return 0;
             }
         }
@@ -138,6 +140,11 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
     // group convolution
     create_group_ops(opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -159,7 +166,7 @@ int ConvolutionDepthWise_x86::create_group_ops(const Option& opt)
 
     for (int g = 0; g < group; g++)
     {
-        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
+        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone();
         Mat bias_data_g;
         if (bias_term)
             bias_data_g = bias_data.range(num_output_g * g, num_output_g);
@@ -260,7 +267,7 @@ int ConvolutionDepthWise_x86::destroy_pipeline(const Option& opt)
 int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
 #if NCNN_INT8
-    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    if (opt.use_int8_inference && int8_scale_term)
     {
         return forward_int8_x86(bottom_blob, top_blob, opt);
     }
@@ -314,7 +321,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -325,7 +332,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -336,7 +343,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s1_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -347,7 +354,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s2_pack16_avx512(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -383,7 +390,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 16;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 16;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -425,7 +432,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -436,7 +443,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -447,7 +454,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -458,7 +465,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -494,7 +501,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 8;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -538,7 +545,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -549,7 +556,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw3x3s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -560,7 +567,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw5x5s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s1_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -571,7 +578,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw5x5s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt);
+                convdw5x5s2_pack4_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -606,7 +613,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 4;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -647,7 +654,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
-                convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -658,7 +665,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             }
             if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+                convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, opt);
 
                 if (activation)
                 {
@@ -831,7 +838,12 @@ int ConvolutionDepthWise_x86::create_pipeline_int8_x86(const Option& opt)
         if (elempack == 8)
         {
             Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_int8, 8, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 8, opt);
+        }
+
+        if (elempack == 1)
+        {
+            weight_data_tm = weight_data;
         }
 
         return 0;
@@ -840,6 +852,11 @@ int ConvolutionDepthWise_x86::create_pipeline_int8_x86(const Option& opt)
     // group convolution
     create_group_ops(opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -938,7 +955,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                 {
                     signed char* outptr_s8 = top_blob.channel(g);
                     float* outptr_f32 = top_blob.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data_int8 + maxk * g * 8;
+                    const signed char* kptr = (const signed char*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -1045,7 +1062,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                         requantize_scales.push_back(scale_out);
                     }
 
-                    convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+                    convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt);
                 }
                 else
                 {
@@ -1057,7 +1074,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                         dequantize_scales.push_back(top_rescale);
                     }
 
-                    convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
+                    convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, dequantize_scales, opt);
                 }
 
                 if (activation)
@@ -1084,7 +1101,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                         requantize_scales.push_back(scale_out);
                     }
 
-                    convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+                    convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, requantize_scales, opt);
                 }
                 else
                 {
@@ -1096,7 +1113,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                         dequantize_scales.push_back(top_rescale);
                     }
 
-                    convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
+                    convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data_tm, bias_data, dequantize_scales, opt);
                 }
 
                 if (activation)
@@ -1132,7 +1149,7 @@ int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_
                 {
                     signed char* outptr_s8 = top_blob.channel(g);
                     float* outptr_f32 = top_blob.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data + maxk * g;
+                    const signed char* kptr = (const signed char*)weight_data_tm + maxk * g;
                     const Mat m = bottom_blob_bordered.channel(g);
 
                     for (int i = 0; i < outh; i++)

src/layer/x86/convolutiondepthwise_x86.h

@@ -42,13 +42,7 @@ public:
     Layer* activation;
     std::vector<ncnn::Layer*> group_ops;
 
-    // packing
-    Mat weight_data_packed;
-
-#if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
-#endif
+    Mat weight_data_tm;
 };
 
 } // namespace ncnn

src/layer/x86/deconvolution_x86.cpp

@@ -104,11 +104,11 @@ int Deconvolution_x86::create_pipeline(const Option& opt)
     {
         Mat weight_data_r2 = weight_data_transposed.reshape(maxk, num_input, num_output);
 
-        weight_data_packed.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);
+        weight_data_tm.create(maxk, num_input / elempack, num_output / out_elempack, (size_t)4u * elempack * out_elempack, elempack * out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            float* g00 = weight_data_packed.channel(q / out_elempack);
+            float* g00 = weight_data_tm.channel(q / out_elempack);
 
             for (int p = 0; p + (elempack - 1) < num_input; p += elempack)
             {
@@ -130,6 +130,11 @@ int Deconvolution_x86::create_pipeline(const Option& opt)
         }
     }
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -192,49 +197,49 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     if (elempack == 16 && out_elempack == 16)
     {
         {
-            deconvolution_pack16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 8 && out_elempack == 16)
     {
         {
-            deconvolution_pack8to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack8to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 16 && out_elempack == 8)
     {
         {
-            deconvolution_pack16to8_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack16to8_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 4 && out_elempack == 16)
     {
         {
-            deconvolution_pack4to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack4to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 16 && out_elempack == 4)
     {
         {
-            deconvolution_pack16to4_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack16to4_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 1 && out_elempack == 16)
     {
         {
-            deconvolution_pack1to16_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack1to16_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 16 && out_elempack == 1)
     {
         {
-            deconvolution_pack16to1_avx512(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack16to1_avx512(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 #endif // __AVX512F__
@@ -242,35 +247,35 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     if (elempack == 8 && out_elempack == 8)
     {
         {
-            deconvolution_pack8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 4 && out_elempack == 8)
     {
         {
-            deconvolution_pack4to8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack4to8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 8 && out_elempack == 4)
     {
         {
-            deconvolution_pack8to4_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack8to4_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 1 && out_elempack == 8)
     {
         {
-            deconvolution_pack1to8_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack1to8_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 8 && out_elempack == 1)
     {
         {
-            deconvolution_pack8to1_avx(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack8to1_avx(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 #endif // __AVX__
@@ -278,21 +283,21 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     if (elempack == 4 && out_elempack == 4)
     {
         {
-            deconvolution_pack4_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack4_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 1 && out_elempack == 4)
     {
         {
-            deconvolution_pack1to4_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack1to4_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 
     if (elempack == 4 && out_elempack == 1)
     {
         {
-            deconvolution_pack4to1_sse(bottom_blob, top_blob_bordered, weight_data_packed, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
+            deconvolution_pack4to1_sse(bottom_blob, top_blob_bordered, weight_data_tm, bias_data, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, activation_type, activation_params, opt);
         }
     }
 #endif // __SSE2__
@@ -317,7 +322,7 @@ int Deconvolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
                             sum = bias_data[p];
                         }
 
-                        const float* kptr = (const float*)weight_data_packed.channel(p);
+                        const float* kptr = (const float*)weight_data_tm.channel(p);
 
                         // channels
                         for (int q = 0; q < channels; q++)

src/layer/x86/deconvolution_x86.h

@@ -30,8 +30,7 @@ public:
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
-    // packn
-    Mat weight_data_packed;
+    Mat weight_data_tm;
 };
 
 } // namespace ncnn

src/layer/x86/deconvolutiondepthwise_x86.cpp

@@ -81,7 +81,7 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 16)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 16, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 16, opt);
         }
 #endif // __AVX512F__
 
@@ -89,7 +89,7 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 8)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 8, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 8, opt);
         }
 #endif // __AVX__
 
@@ -97,13 +97,13 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt)
         if (elempack == 4)
         {
             Mat weight_data_r2 = weight_data_transposed.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_packed, 4, opt);
+            convert_packing(weight_data_r2, weight_data_tm, 4, opt);
         }
 #endif // __SSE2__
 
         if (elempack == 1)
         {
-            weight_data_packed = weight_data_transposed;
+            weight_data_tm = weight_data_transposed;
         }
 
         return 0;
@@ -112,6 +112,11 @@ int DeconvolutionDepthWise_x86::create_pipeline(const Option& opt)
     // group convolution
     create_group_ops(opt);
 
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -133,7 +138,7 @@ int DeconvolutionDepthWise_x86::create_group_ops(const Option& opt)
 
     for (int g = 0; g < group; g++)
     {
-        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
+        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g).clone();
         Mat bias_data_g;
         if (bias_term)
             bias_data_g = bias_data.range(num_output_g * g, num_output_g);
@@ -256,7 +261,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob_bordered.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 16;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 16;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -318,7 +323,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob_bordered.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 8;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 8;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -380,7 +385,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c
                 for (int g = 0; g < channels; g++)
                 {
                     float* outptr = top_blob_bordered.channel(g);
-                    const float* kptr = (const float*)weight_data_packed + maxk * g * 4;
+                    const float* kptr = (const float*)weight_data_tm + maxk * g * 4;
                     const Mat m = bottom_blob.channel(g);
 
                     for (int i = 0; i < outh; i++)
@@ -441,7 +446,7 @@ int DeconvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, c
             for (int g = 0; g < channels; g++)
             {
                 float* outptr = top_blob_bordered.channel(g);
-                const float* kptr = (const float*)weight_data_packed + maxk * g;
+                const float* kptr = (const float*)weight_data_tm + maxk * g;
                 const Mat m = bottom_blob.channel(g);
 
                 for (int i = 0; i < outh; i++)

src/layer/x86/deconvolutiondepthwise_x86.h

@@ -35,8 +35,7 @@ protected:
 public:
     std::vector<ncnn::Layer*> group_ops;
 
-    // packing
-    Mat weight_data_packed;
+    Mat weight_data_tm;
 };
 
 } // namespace ncnn

src/layer/x86/innerproduct_x86.cpp

@@ -35,7 +35,6 @@ InnerProduct_x86::InnerProduct_x86()
 #endif // __SSE2__
 
     flatten = 0;
-    activation = 0;
 }
 
 int InnerProduct_x86::create_pipeline(const Option& opt)
@@ -82,11 +81,11 @@ int InnerProduct_x86::create_pipeline(const Option& opt)
         {
             Mat weight_data_r2 = weight_data.reshape(num_input, num_output);
 
-            weight_data_packed.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack);
+            weight_data_tm.create(num_input, num_output / out_elempack, (size_t)4u * out_elempack, out_elempack);
 
             for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
             {
-                float* g0 = weight_data_packed.row(q / out_elempack);
+                float* g0 = weight_data_tm.row(q / out_elempack);
 
                 for (int p = 0; p < num_input; p++)
                 {
@@ -98,6 +97,15 @@ int InnerProduct_x86::create_pipeline(const Option& opt)
             }
         }
     }
+    else
+    {
+        weight_data_tm = weight_data;
+    }
+
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
 
     return 0;
 }
@@ -111,20 +119,13 @@ int InnerProduct_x86::destroy_pipeline(const Option& opt)
         flatten = 0;
     }
 
-    if (activation)
-    {
-        activation->destroy_pipeline(opt);
-        delete activation;
-        activation = 0;
-    }
-
     return 0;
 }
 
 int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
 #if NCNN_INT8
-    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    if (opt.use_int8_inference && int8_scale_term)
     {
         return forward_int8_x86(bottom_blob, top_blob, opt);
     }
@@ -169,7 +170,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 16;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m512 _sum0 = _mm512_set1_ps(0.f);
@@ -276,7 +277,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 16;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m512 _sum = _mm512_set1_ps(0.f);
@@ -310,7 +311,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 16;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m128 _sum0 = _mm_set1_ps(0.f);
@@ -418,7 +419,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 16;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m256 _sum0 = _mm256_set1_ps(0.f);
@@ -526,7 +527,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output; p++)
                 {
-                    const float* kptr = (const float*)weight_data + num_input * p;
+                    const float* kptr = (const float*)weight_data_tm + num_input * p;
                     const float* m = bottom_blob.row(j);
 
                     __m512 _sum0 = _mm512_set1_ps(0.f);
@@ -560,7 +561,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 4;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m512 _sum0 = _mm512_set1_ps(0.f);
@@ -608,7 +609,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 8;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m512 _sum0 = _mm512_set1_ps(0.f);
@@ -678,7 +679,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 8;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m256 _sum0 = _mm256_set1_ps(0.f);
@@ -753,7 +754,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 8;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m256 _sum = _mm256_set1_ps(0.f);
@@ -837,7 +838,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 8;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m128 _sum0 = _mm_set1_ps(0.f);
@@ -905,7 +906,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output; p++)
                 {
-                    const float* kptr = (const float*)weight_data + num_input * p;
+                    const float* kptr = (const float*)weight_data_tm + num_input * p;
                     const float* m = bottom_blob.row(j);
 
                     __m256 _sum0 = _mm256_set1_ps(0.f);
@@ -982,7 +983,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 4;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m256 _sum0 = _mm256_set1_ps(0.f);
@@ -1059,7 +1060,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 4;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m128 _sum0 = _mm_set1_ps(0.f);
@@ -1133,7 +1134,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output / num_output_elempack; p++)
                 {
-                    const float* kptr = (const float*)weight_data_packed + num_input * p * 4;
+                    const float* kptr = weight_data_tm.row(p);
                     const float* m = bottom_blob.row(j);
 
                     __m128 _sum = _mm_set1_ps(0.f);
@@ -1219,7 +1220,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output; p++)
                 {
-                    const float* kptr = (const float*)weight_data + num_input * p;
+                    const float* kptr = (const float*)weight_data_tm + num_input * p;
                     const float* m = bottom_blob.row(j);
 
                     __m128 _sum0 = _mm_set1_ps(0.f);
@@ -1297,7 +1298,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 
                 for (int p = 0; p < num_output; p++)
                 {
-                    const float* kptr = (const float*)weight_data + num_input * p;
+                    const float* kptr = (const float*)weight_data_tm + num_input * p;
                     const float* m = bottom_blob.row(j);
 
                     float sum = 0.f;
@@ -1409,7 +1410,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
                 _sum0 = _mm512_loadu_ps((const float*)bias_data + p * 16);
             }
 
-            const float* kptr = weight_data_packed.row(p);
+            const float* kptr = weight_data_tm.row(p);
 
             const float* sptr = bottom_blob_flattened;
 
@@ -1511,7 +1512,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
                 _sum0 = _mm256_loadu_ps((const float*)bias_data + p * 8);
             }
 
-            const float* kptr = weight_data_packed.row(p);
+            const float* kptr = weight_data_tm.row(p);
 
             const float* sptr = bottom_blob_flattened;
 
@@ -1613,7 +1614,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
                 _sum0 = _mm_loadu_ps((const float*)bias_data + p * 4);
             }
 
-            const float* kptr = weight_data_packed.row(p);
+            const float* kptr = weight_data_tm.row(p);
 
             const float* sptr = bottom_blob_flattened;
 
@@ -1725,14 +1726,14 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
                 sums[7] = bias_data[p + 7];
             }
 
-            const float* w0 = (const float*)weight_data + num_input * p;
-            const float* w1 = (const float*)weight_data + num_input * (p + 1);
-            const float* w2 = (const float*)weight_data + num_input * (p + 2);
-            const float* w3 = (const float*)weight_data + num_input * (p + 3);
-            const float* w4 = (const float*)weight_data + num_input * (p + 4);
-            const float* w5 = (const float*)weight_data + num_input * (p + 5);
-            const float* w6 = (const float*)weight_data + num_input * (p + 6);
-            const float* w7 = (const float*)weight_data + num_input * (p + 7);
+            const float* w0 = (const float*)weight_data_tm + num_input * p;
+            const float* w1 = (const float*)weight_data_tm + num_input * (p + 1);
+            const float* w2 = (const float*)weight_data_tm + num_input * (p + 2);
+            const float* w3 = (const float*)weight_data_tm + num_input * (p + 3);
+            const float* w4 = (const float*)weight_data_tm + num_input * (p + 4);
+            const float* w5 = (const float*)weight_data_tm + num_input * (p + 5);
+            const float* w6 = (const float*)weight_data_tm + num_input * (p + 6);
+            const float* w7 = (const float*)weight_data_tm + num_input * (p + 7);
 
             const float* m = bottom_blob_flattened;
 
@@ -1829,10 +1830,10 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
                 sums[3] = bias_data[p + 3];
             }
 
-            const float* w0 = (const float*)weight_data + num_input * p;
-            const float* w1 = (const float*)weight_data + num_input * (p + 1);
-            const float* w2 = (const float*)weight_data + num_input * (p + 2);
-            const float* w3 = (const float*)weight_data + num_input * (p + 3);
+            const float* w0 = (const float*)weight_data_tm + num_input * p;
+            const float* w1 = (const float*)weight_data_tm + num_input * (p + 1);
+            const float* w2 = (const float*)weight_data_tm + num_input * (p + 2);
+            const float* w3 = (const float*)weight_data_tm + num_input * (p + 3);
 
             const float* m = bottom_blob_flattened;
 
@@ -1919,7 +1920,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
         int remain_num_output_start = 0;
 #endif // __SSE2__
 
-// num_output
         #pragma omp parallel for num_threads(opt.num_threads)
         for (int p = remain_num_output_start; p < num_output; p++)
         {
@@ -1928,7 +1928,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
             if (bias_term)
                 sum = bias_data[p];
 
-            const float* w = (const float*)weight_data + num_input * p;
+            const float* w = (const float*)weight_data_tm + num_input * p;
 
             const float* m = bottom_blob_flattened;
 
@@ -1986,8 +1986,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
 #if NCNN_INT8
 int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt)
 {
-    activation = create_activation_layer(activation_type, activation_params, opt);
-
     const int num_input = weight_data_size / num_output;
 
     int out_elempack = 1;
@@ -2003,11 +2001,11 @@ int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt)
     {
         Mat weight_data_r2 = weight_data.reshape(num_input, num_output);
 
-        weight_data_int8.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack);
+        weight_data_tm.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack);
 
         for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
         {
-            signed char* g0 = weight_data_int8.row<signed char>(q / out_elempack);
+            signed char* g0 = weight_data_tm.row<signed char>(q / out_elempack);
 
             for (int p = 0; p < num_input; p++)
             {
@@ -2019,6 +2017,24 @@ int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt)
         }
     }
 
+    scale_in_data.create(num_output);
+    for (int p = 0; p < num_output; p++)
+    {
+        // dequantize
+        float scale_in;
+        if (weight_data_int8_scales[p] == 0)
+            scale_in = 0;
+        else
+            scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
+
+        scale_in_data[p] = scale_in;
+    }
+
+    if (opt.lightmode)
+    {
+        weight_data.release();
+    }
+
     return 0;
 }
 
@@ -2026,17 +2042,6 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co
 {
     const int num_input = weight_data_size / num_output;
 
-    if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)
-    {
-        // gemm
-        Mat bottom_blob_unpacked;
-        Option opt_unpack = opt;
-        opt_unpack.blob_allocator = opt.workspace_allocator;
-        convert_packing(bottom_blob, bottom_blob_unpacked, 1, opt_unpack);
-
-        return forward_int8(bottom_blob_unpacked, top_blob, opt);
-    }
-
     int elembits = bottom_blob.elembits();
 
     Mat bottom_blob_int8 = bottom_blob;
@@ -2047,6 +2052,327 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co
         quantize_to_int8(bottom_blob, bottom_blob_int8, bottom_blob_int8_scales, opt_q);
     }
 
+    if (bottom_blob_int8.dims == 2 && bottom_blob_int8.w == num_input && bottom_blob_int8.h * bottom_blob_int8.elempack > 1)
+    {
+        // gemm
+        Mat bottom_blob_int8_unpacked;
+        Option opt_unpack = opt;
+        opt_unpack.blob_allocator = opt.workspace_allocator;
+        convert_packing(bottom_blob_int8, bottom_blob_int8_unpacked, 1, opt_unpack);
+
+        int h = bottom_blob_int8_unpacked.h;
+
+        int out_elempack = 1;
+#if __SSE2__
+        if (opt.use_packing_layout)
+        {
+            out_elempack = h % 4 == 0 ? 4 : 1;
+        }
+#endif
+
+        int outh = h / out_elempack;
+
+        top_blob.create(num_output, outh, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator);
+        if (top_blob.empty())
+            return -100;
+
+        int num_output_elempack = 1;
+#if __SSE2__
+        if (opt.use_packing_layout)
+        {
+            num_output_elempack = num_output % 8 == 0 ? 8 : 1;
+        }
+#endif
+
+#if __SSE2__
+        if (num_output_elempack == 8 && out_elempack == 4)
+        {
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int j = 0; j < outh; j++)
+            {
+                float* outptr = top_blob.row(j);
+
+                for (int p = 0; p < num_output / num_output_elempack; p++)
+                {
+                    const signed char* kptr = weight_data_tm.row<const signed char>(p);
+                    const signed char* m0 = bottom_blob_int8_unpacked.row<const signed char>(j * 4);
+                    const signed char* m1 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 1);
+                    const signed char* m2 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 2);
+                    const signed char* m3 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 3);
+
+                    __m128i _sum00 = _mm_setzero_si128();
+                    __m128i _sum01 = _mm_setzero_si128();
+                    __m128i _sum10 = _mm_setzero_si128();
+                    __m128i _sum11 = _mm_setzero_si128();
+                    __m128i _sum20 = _mm_setzero_si128();
+                    __m128i _sum21 = _mm_setzero_si128();
+                    __m128i _sum30 = _mm_setzero_si128();
+                    __m128i _sum31 = _mm_setzero_si128();
+
+                    int i = 0;
+                    for (; i < num_input; i++)
+                    {
+                        // TODO use _mm_cvtepi8_epi16 on sse4.1
+                        __m128i _w = _mm_loadl_epi64((const __m128i*)kptr);
+                        _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w));
+
+                        __m128i _val0 = _mm_set1_epi16((short)m0[0]);
+                        __m128i _val1 = _mm_set1_epi16((short)m1[0]);
+                        __m128i _val2 = _mm_set1_epi16((short)m2[0]);
+                        __m128i _val3 = _mm_set1_epi16((short)m3[0]);
+
+                        __m128i _s0l = _mm_mullo_epi16(_val0, _w);
+                        __m128i _s0h = _mm_mulhi_epi16(_val0, _w);
+                        __m128i _s1l = _mm_mullo_epi16(_val1, _w);
+                        __m128i _s1h = _mm_mulhi_epi16(_val1, _w);
+                        __m128i _s2l = _mm_mullo_epi16(_val2, _w);
+                        __m128i _s2h = _mm_mulhi_epi16(_val2, _w);
+                        __m128i _s3l = _mm_mullo_epi16(_val3, _w);
+                        __m128i _s3h = _mm_mulhi_epi16(_val3, _w);
+                        __m128i _s00 = _mm_unpacklo_epi16(_s0l, _s0h);
+                        __m128i _s01 = _mm_unpackhi_epi16(_s0l, _s0h);
+                        __m128i _s10 = _mm_unpacklo_epi16(_s1l, _s1h);
+                        __m128i _s11 = _mm_unpackhi_epi16(_s1l, _s1h);
+                        __m128i _s20 = _mm_unpacklo_epi16(_s2l, _s2h);
+                        __m128i _s21 = _mm_unpackhi_epi16(_s2l, _s2h);
+                        __m128i _s30 = _mm_unpacklo_epi16(_s3l, _s3h);
+                        __m128i _s31 = _mm_unpackhi_epi16(_s3l, _s3h);
+
+                        _sum00 = _mm_add_epi32(_sum00, _s00);
+                        _sum01 = _mm_add_epi32(_sum01, _s01);
+                        _sum10 = _mm_add_epi32(_sum10, _s10);
+                        _sum11 = _mm_add_epi32(_sum11, _s11);
+                        _sum20 = _mm_add_epi32(_sum20, _s20);
+                        _sum21 = _mm_add_epi32(_sum21, _s21);
+                        _sum30 = _mm_add_epi32(_sum30, _s30);
+                        _sum31 = _mm_add_epi32(_sum31, _s31);
+
+                        m0++;
+                        m1++;
+                        m2++;
+                        m3++;
+                        kptr += 8;
+                    }
+
+                    // dequantize and relu
+                    __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);
+                    __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);
+
+                    __m128 _sumfp32_00 = _mm_cvtepi32_ps(_sum00);
+                    __m128 _sumfp32_01 = _mm_cvtepi32_ps(_sum01);
+                    __m128 _sumfp32_10 = _mm_cvtepi32_ps(_sum10);
+                    __m128 _sumfp32_11 = _mm_cvtepi32_ps(_sum11);
+                    __m128 _sumfp32_20 = _mm_cvtepi32_ps(_sum20);
+                    __m128 _sumfp32_21 = _mm_cvtepi32_ps(_sum21);
+                    __m128 _sumfp32_30 = _mm_cvtepi32_ps(_sum30);
+                    __m128 _sumfp32_31 = _mm_cvtepi32_ps(_sum31);
+                    if (bias_term)
+                    {
+                        __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);
+                        __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);
+                        _sumfp32_00 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_00, _scale_in0));
+                        _sumfp32_01 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_01, _scale_in1));
+                        _sumfp32_10 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_10, _scale_in0));
+                        _sumfp32_11 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_11, _scale_in1));
+                        _sumfp32_20 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_20, _scale_in0));
+                        _sumfp32_21 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_21, _scale_in1));
+                        _sumfp32_30 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_30, _scale_in0));
+                        _sumfp32_31 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_31, _scale_in1));
+                    }
+                    else
+                    {
+                        _sumfp32_00 = _mm_mul_ps(_sumfp32_00, _scale_in0);
+                        _sumfp32_01 = _mm_mul_ps(_sumfp32_01, _scale_in1);
+                        _sumfp32_10 = _mm_mul_ps(_sumfp32_10, _scale_in0);
+                        _sumfp32_11 = _mm_mul_ps(_sumfp32_11, _scale_in1);
+                        _sumfp32_20 = _mm_mul_ps(_sumfp32_20, _scale_in0);
+                        _sumfp32_21 = _mm_mul_ps(_sumfp32_21, _scale_in1);
+                        _sumfp32_30 = _mm_mul_ps(_sumfp32_30, _scale_in0);
+                        _sumfp32_31 = _mm_mul_ps(_sumfp32_31, _scale_in1);
+                    }
+
+                    _sumfp32_00 = activation_sse(_sumfp32_00, activation_type, activation_params);
+                    _sumfp32_01 = activation_sse(_sumfp32_01, activation_type, activation_params);
+                    _sumfp32_10 = activation_sse(_sumfp32_10, activation_type, activation_params);
+                    _sumfp32_11 = activation_sse(_sumfp32_11, activation_type, activation_params);
+                    _sumfp32_20 = activation_sse(_sumfp32_20, activation_type, activation_params);
+                    _sumfp32_21 = activation_sse(_sumfp32_21, activation_type, activation_params);
+                    _sumfp32_30 = activation_sse(_sumfp32_30, activation_type, activation_params);
+                    _sumfp32_31 = activation_sse(_sumfp32_31, activation_type, activation_params);
+
+                    // transpose 4x8
+                    _MM_TRANSPOSE4_PS(_sumfp32_00, _sumfp32_10, _sumfp32_20, _sumfp32_30);
+                    _MM_TRANSPOSE4_PS(_sumfp32_01, _sumfp32_11, _sumfp32_21, _sumfp32_31);
+
+                    _mm_storeu_ps(outptr, _sumfp32_00);
+                    _mm_storeu_ps(outptr + 4, _sumfp32_10);
+                    _mm_storeu_ps(outptr + 8, _sumfp32_20);
+                    _mm_storeu_ps(outptr + 12, _sumfp32_30);
+                    _mm_storeu_ps(outptr + 16, _sumfp32_01);
+                    _mm_storeu_ps(outptr + 20, _sumfp32_11);
+                    _mm_storeu_ps(outptr + 24, _sumfp32_21);
+                    _mm_storeu_ps(outptr + 28, _sumfp32_31);
+
+                    outptr += 32;
+                }
+            }
+        }
+
+        if (num_output_elempack == 1 && out_elempack == 4)
+        {
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int j = 0; j < outh; j++)
+            {
+                float* outptr = top_blob.row(j);
+
+                for (int p = 0; p < num_output; p++)
+                {
+                    const signed char* kptr = weight_data_tm.row<const signed char>(p);
+                    const signed char* m0 = bottom_blob_int8_unpacked.row<const signed char>(j * 4);
+                    const signed char* m1 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 1);
+                    const signed char* m2 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 2);
+                    const signed char* m3 = bottom_blob_int8_unpacked.row<const signed char>(j * 4 + 3);
+
+                    int sum0 = 0;
+                    int sum1 = 0;
+                    int sum2 = 0;
+                    int sum3 = 0;
+
+                    int i = 0;
+                    for (; i < num_input; i++)
+                    {
+                        sum0 += *m0++ * kptr[0];
+                        sum1 += *m1++ * kptr[0];
+                        sum2 += *m2++ * kptr[0];
+                        sum3 += *m3++ * kptr[0];
+                        kptr += 1;
+                    }
+
+                    // dequantize and relu
+                    float sumfp32_0 = sum0 * scale_in_data[p];
+                    float sumfp32_1 = sum1 * scale_in_data[p];
+                    float sumfp32_2 = sum2 * scale_in_data[p];
+                    float sumfp32_3 = sum3 * scale_in_data[p];
+
+                    if (bias_term)
+                    {
+                        sumfp32_0 += bias_data[p];
+                        sumfp32_1 += bias_data[p];
+                        sumfp32_2 += bias_data[p];
+                        sumfp32_3 += bias_data[p];
+                    }
+
+                    outptr[0] = activation_ss(sumfp32_0, activation_type, activation_params);
+                    outptr[1] = activation_ss(sumfp32_1, activation_type, activation_params);
+                    outptr[2] = activation_ss(sumfp32_2, activation_type, activation_params);
+                    outptr[3] = activation_ss(sumfp32_3, activation_type, activation_params);
+                    outptr += 4;
+                }
+            }
+        }
+
+        if (num_output_elempack == 8 && out_elempack == 1)
+        {
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int j = 0; j < outh; j++)
+            {
+                float* outptr = top_blob.row(j);
+
+                for (int p = 0; p < num_output / num_output_elempack; p++)
+                {
+                    const signed char* kptr = weight_data_tm.row<const signed char>(p);
+                    const signed char* m = bottom_blob_int8_unpacked.row<const signed char>(j);
+
+                    __m128i _sum0 = _mm_setzero_si128();
+                    __m128i _sum1 = _mm_setzero_si128();
+
+                    int i = 0;
+                    for (; i < num_input; i++)
+                    {
+                        __m128i _val = _mm_set1_epi16((short)m[0]);
+
+                        // TODO use _mm_cvtepi8_epi16 on sse4.1
+                        __m128i _w = _mm_loadl_epi64((const __m128i*)kptr);
+                        _w = _mm_unpacklo_epi8(_w, _mm_cmpgt_epi8(_mm_setzero_si128(), _w));
+
+                        __m128i _sl = _mm_mullo_epi16(_val, _w);
+                        __m128i _sh = _mm_mulhi_epi16(_val, _w);
+                        __m128i _s0 = _mm_unpacklo_epi16(_sl, _sh);
+                        __m128i _s1 = _mm_unpackhi_epi16(_sl, _sh);
+
+                        _sum0 = _mm_add_epi32(_sum0, _s0);
+                        _sum1 = _mm_add_epi32(_sum1, _s1);
+
+                        m++;
+                        kptr += 8;
+                    }
+
+                    // dequantize and relu
+                    __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);
+                    __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);
+
+                    __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0);
+                    __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1);
+
+                    if (bias_term)
+                    {
+                        __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);
+                        __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);
+                        _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0));
+                        _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1));
+                    }
+                    else
+                    {
+                        _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0);
+                        _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1);
+                    }
+
+                    _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params);
+                    _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params);
+
+                    _mm_storeu_ps(outptr, _sumfp32_0);
+                    _mm_storeu_ps(outptr + 4, _sumfp32_1);
+                    outptr += 8;
+                }
+            }
+        }
+#endif // __SSE2__
+
+        if (num_output_elempack == 1 && out_elempack == 1)
+        {
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int j = 0; j < outh; j++)
+            {
+                float* outptr = top_blob.row(j);
+
+                for (int p = 0; p < num_output; p++)
+                {
+                    const signed char* kptr = weight_data_tm.row<const signed char>(p);
+                    const signed char* m = bottom_blob_int8_unpacked.row<const signed char>(j);
+
+                    int sum = 0;
+
+                    int i = 0;
+                    for (; i < num_input; i++)
+                    {
+                        sum += *m++ * *kptr++;
+                    }
+
+                    // dequantize and relu
+                    float sumfp32 = sum * scale_in_data[p];
+
+                    if (bias_term)
+                        sumfp32 += bias_data[p];
+
+                    outptr[0] = activation_ss(sumfp32, activation_type, activation_params);
+                    outptr += 1;
+                }
+            }
+        }
+
+        return 0;
+    }
+
     Mat bottom_blob_int8_flattened = bottom_blob_int8;
     if (bottom_blob_int8.dims != 1)
     {
@@ -2070,22 +2396,16 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co
     if (top_blob.empty())
         return -100;
 
-    Mat top_blob_int32;
-    top_blob_int32.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.workspace_allocator);
-    if (top_blob_int32.empty())
-        return -100;
-
 #if __SSE2__
     if (out_elempack == 8)
     {
-// num_output
         #pragma omp parallel for num_threads(opt.num_threads)
         for (int p = 0; p < num_output / out_elempack; p++)
         {
             __m128i _sum0 = _mm_setzero_si128();
             __m128i _sum1 = _mm_setzero_si128();
 
-            const signed char* kptr = weight_data_int8.row<const signed char>(p);
+            const signed char* kptr = weight_data_tm.row<const signed char>(p);
             const signed char* sptr = bottom_blob_int8_flattened;
 
             int i = 0;
@@ -2109,22 +2429,44 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co
                 kptr += 8;
             }
 
-            int* outptr = (int*)top_blob_int32;
-            _mm_storeu_si128((__m128i*)(outptr + p * 8), _sum0);
-            _mm_storeu_si128((__m128i*)(outptr + p * 8 + 4), _sum1);
+            // dequantize and relu
+            __m128 _scale_in0 = _mm_loadu_ps((const float*)scale_in_data + p * 8);
+            __m128 _scale_in1 = _mm_loadu_ps((const float*)scale_in_data + p * 8 + 4);
+
+            __m128 _sumfp32_0 = _mm_cvtepi32_ps(_sum0);
+            __m128 _sumfp32_1 = _mm_cvtepi32_ps(_sum1);
+
+            if (bias_term)
+            {
+                __m128 _bias0 = _mm_loadu_ps((const float*)bias_data + p * 8);
+                __m128 _bias1 = _mm_loadu_ps((const float*)bias_data + p * 8 + 4);
+                _sumfp32_0 = _mm_add_ps(_bias0, _mm_mul_ps(_sumfp32_0, _scale_in0));
+                _sumfp32_1 = _mm_add_ps(_bias1, _mm_mul_ps(_sumfp32_1, _scale_in1));
+            }
+            else
+            {
+                _sumfp32_0 = _mm_mul_ps(_sumfp32_0, _scale_in0);
+                _sumfp32_1 = _mm_mul_ps(_sumfp32_1, _scale_in1);
+            }
+
+            _sumfp32_0 = activation_sse(_sumfp32_0, activation_type, activation_params);
+            _sumfp32_1 = activation_sse(_sumfp32_1, activation_type, activation_params);
+
+            float* outptr = (float*)top_blob + p * 8;
+            _mm_storeu_ps(outptr, _sumfp32_0);
+            _mm_storeu_ps(outptr + 4, _sumfp32_1);
         }
     }
 #endif // __SSE2__
 
     if (out_elempack == 1)
     {
-// num_output
         #pragma omp parallel for num_threads(opt.num_threads)
         for (int p = 0; p < num_output / out_elempack; p++)
         {
             int sum = 0;
 
-            const signed char* kptr = weight_data_int8.row<const signed char>(p);
+            const signed char* kptr = weight_data_tm.row<const signed char>(p);
             const signed char* sptr = bottom_blob_int8_flattened;
 
             int i = 0;
@@ -2140,29 +2482,16 @@ int InnerProduct_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, co
                 kptr += 1;
             }
 
-            int* outptr = (int*)top_blob_int32;
-            outptr[p] = sum;
-        }
-    }
+            // dequantize and relu
+            float sumfp32 = sum * scale_in_data[p];
 
-    Mat scale_data(num_output);
-    for (int p = 0; p < num_output; p++)
-    {
-        // dequantize
-        float scale_in;
-        if (weight_data_int8_scales[p] == 0)
-            scale_in = 0;
-        else
-            scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
-
-        scale_data[p] = scale_in;
-    }
+            if (bias_term)
+                sumfp32 += bias_data[p];
 
-    dequantize_from_int32(top_blob_int32, top_blob, scale_data, bias_data, opt);
+            sumfp32 = activation_ss(sumfp32, activation_type, activation_params);
 
-    if (activation)
-    {
-        activation->forward_inplace(top_blob, opt);
+            top_blob[p] = sumfp32;
+        }
     }
 
     return 0;

src/layer/x86/innerproduct_x86.h

@@ -1,19 +1,16 @@
-// Tencent is pleased to support the open source community by making ncnn
-// available.
+// Tencent is pleased to support the open source community by making ncnn available.
 //
 // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 //
-// Licensed under the BSD 3-Clause License (the "License"); you may not use this
-// file except in compliance with the License. You may obtain a copy of the
-// License at
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
 //
 // https://opensource.org/licenses/BSD-3-Clause
 //
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
 
 #ifndef LAYER_INNERPRODUCT_X86_H
 #define LAYER_INNERPRODUCT_X86_H
@@ -40,14 +37,11 @@ protected:
 
 public:
     Layer* flatten;
-    Layer* activation;
 
-    Mat weight_data_packed;
+    Mat weight_data_tm;
 
 #if NCNN_INT8
-    // int8
-    Mat weight_data_int8;
-    Mat scales_in;
+    Mat scale_in_data;
 #endif
 };
 

tests/testutil.h

@@ -363,6 +363,7 @@ int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.lightmode = false;
     opt.use_packing_layout = false;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
@@ -808,6 +809,7 @@ int test_layer_naive(int typeindex, const ncnn::ParamDict& pd, const std::vector
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.lightmode = false;
     opt.use_packing_layout = false;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;