int8 code refactoring wip, add int8 test

benchmark/benchncnn.cpp

@@ -221,9 +221,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("squeezenet_int8", ncnn::Mat(227, 227, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("mobilenet", ncnn::Mat(224, 224, 3), opt);
@@ -232,9 +230,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("mobilenet_int8", ncnn::Mat(224, 224, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("mobilenet_v2", ncnn::Mat(224, 224, 3), opt);
@@ -260,9 +256,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("googlenet_int8", ncnn::Mat(224, 224, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("resnet18", ncnn::Mat(224, 224, 3), opt);
@@ -271,9 +265,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("resnet18_int8", ncnn::Mat(224, 224, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("alexnet", ncnn::Mat(227, 227, 3), opt);
@@ -284,9 +276,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("vgg16_int8", ncnn::Mat(224, 224, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("resnet50", ncnn::Mat(224, 224, 3), opt);
@@ -295,9 +285,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("resnet50_int8", ncnn::Mat(224, 224, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("squeezenet_ssd", ncnn::Mat(300, 300, 3), opt);
@@ -306,9 +294,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("squeezenet_ssd_int8", ncnn::Mat(300, 300, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("mobilenet_ssd", ncnn::Mat(300, 300, 3), opt);
@@ -317,9 +303,7 @@ int main(int argc, char** argv)
     if (!use_vulkan_compute)
 #endif // NCNN_VULKAN
     {
-    opt.use_packing_layout = false;
     benchmark("mobilenet_ssd_int8", ncnn::Mat(300, 300, 3), opt);
-    opt.use_packing_layout = true;
     }
 
     benchmark("mobilenet_yolo", ncnn::Mat(416, 416, 3), opt);

src/layer/arm/convolution_3x3_int8.h

@@ -4460,25 +4460,3 @@ static void conv3x3s2_packed_int8_neon(const Mat& bottom_blob, Mat& top_blob, co
         }
     }
 }
-
-static void conv3x3s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv3x3s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}

src/layer/arm/convolution_5x5_int8.h

@@ -1,35 +0,0 @@
-// BUG1989 is pleased to support the open source community by supporting ncnn available.
-//
-// Copyright (C) 2019 BUG1989. All rights reserved.
-//
-// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// https://opensource.org/licenses/BSD-3-Clause
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-static void conv5x5s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv5x5s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}

src/layer/arm/convolution_7x7_int8.h

@@ -1,35 +0,0 @@
-// BUG1989 is pleased to support the open source community by supporting ncnn available.
-//
-// Copyright (C) 2019 BUG1989. All rights reserved.
-//
-// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// https://opensource.org/licenses/BSD-3-Clause
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-static void conv7x7s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv7x7s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}

src/layer/arm/convolution_arm.cpp

@@ -35,8 +35,6 @@ namespace ncnn {
 #include "convolution_sgemm_int8.h"
 #include "convolution_1x1_int8.h"
 #include "convolution_3x3_int8.h"
-#include "convolution_5x5_int8.h"
-#include "convolution_7x7_int8.h"
 
 #if __ARM_NEON
 #include "convolution_1x1_pack4.h"
@@ -100,38 +98,14 @@ int Convolution_arm::create_pipeline(const Option& opt)
         activation->create_pipeline(opt);
     }
 
-    const int maxk = kernel_w * kernel_h;
-    const int num_input = weight_data_size / maxk / num_output;
-
-    if (use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
         support_packing = false;
 
-        if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
-        {
-            use_winograd3x3 = true;
-            // conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_int8_data, num_input, num_output);
-            conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_int8_data, num_input, num_output);
-        }
-
-        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
-        {
-            conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_int8_data, num_input, num_output);
-        }
-        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
-        {
-            use_sgemm1x1 = true;
-            conv1x1s1_sgemm_transform_kernel_int8_neon(weight_data, weight_1x1s1_sgemm_int8_data, num_input, num_output);
-        }
-        else
-        {
-            conv_im2col_sgemm_transform_kernel_int8_neon(weight_data, weight_sgemm_int8_data, num_input, num_output, maxk);
-        }
-
-        return 0;
+        return create_pipeline_int8_arm(opt);
     }
 
-    if (opt.use_packing_layout == false && kernel_w == kernel_h && stride_w == 1 && stride_h == 1 && dilation_w != 1 && dilation_h == dilation_w)
+    if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
         convolution_dilation1 = ncnn::create_layer(ncnn::LayerType::Convolution);
 
@@ -173,6 +147,9 @@ int Convolution_arm::create_pipeline(const Option& opt)
         return 0;
     }
 
+    const int maxk = kernel_w * kernel_h;
+    const int num_input = weight_data_size / maxk / num_output;
+
     int elempack = (opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1;
     int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1;
 
@@ -180,15 +157,15 @@ int Convolution_arm::create_pipeline(const Option& opt)
     // pack4
     if (elempack == 4 && out_elempack == 4)
     {
-        if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output);
         }
@@ -307,15 +284,15 @@ int Convolution_arm::create_pipeline(const Option& opt)
     // pack4to1
     if (elempack == 4 && out_elempack == 1)
     {
-        if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output);
         }
@@ -418,12 +395,12 @@ int Convolution_arm::create_pipeline(const Option& opt)
             conv3x3s2_transform_kernel_neon(weight_data, weight_3x3s2_data, num_input, num_output);
         }
 
-        if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk);
         }
 
-        if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk);
         }
@@ -451,90 +428,6 @@ int Convolution_arm::destroy_pipeline(const Option& opt)
     return 0;
 }
 
-int Convolution_arm::forwardDilation(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
-{
-    int w = bottom_blob.w;
-    int h = bottom_blob.h;
-    size_t elemsize = bottom_blob.elemsize;
-
-    const int kernel_size = kernel_w;
-    const int stride = stride_w;
-    const int dilation = dilation_w;
-    const int kernel_extent = dilation * (kernel_size - 1) + 1;
-
-    int outw = (w - kernel_extent) / stride + 1;
-    int outh = (h - kernel_extent) / stride + 1;
-
-    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
-    if (top_blob.empty())
-        return -100;
-
-    // Make (dilation * dilation) batches
-    Mat inner_bottom_blob;
-    Mat inner_top_blob;
-    for (int x = 0; x < dilation; x ++)
-    {
-        for (int y = 0; y < dilation; y ++)
-        {
-            int inner_w = (w - y + dilation - 1) / dilation;
-            int inner_h = (h - x + dilation - 1) / dilation;
-
-            int inner_outw = (inner_w - kernel_size) / stride + 1;
-            int inner_outh = (inner_h - kernel_size) / stride + 1;
-
-            inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator);
-            if (inner_bottom_blob.empty())
-                return -100;
-
-            inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator);
-            if (inner_top_blob.empty())
-                return -100;
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int c = 0; c < bottom_blob.c; c ++)
-            {
-                float *outptr = inner_bottom_blob.channel(c);
-
-                for (int i = 0; i < inner_h; i ++)
-                {
-                    const float *ptr = (const float *) bottom_blob.channel(c) + dilation * i * w + x * w + y;
-                    for (int j = 0; j < inner_w; j ++)
-                    {
-                        outptr[j] = ptr[j*dilation];
-                    }
-                    outptr += inner_w;
-                }
-            }
-
-            Option opt_g = opt;
-            opt_g.blob_allocator = inner_top_blob.allocator;
-            convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g);
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int c = 0; c < num_output; c ++)
-            {
-                float *outptr = (float *) top_blob.channel(c) + x * outw + y;
-                for (int i = 0; i < inner_outh; i ++)
-                {
-                    const float *ptr = (const float *) inner_top_blob.channel(c) + i * inner_outw;
-                    for (int j = 0; j < inner_outw; j ++)
-                    {
-                        outptr[j*dilation] = ptr[j];
-                    }
-                    outptr += dilation * outw;
-                }
-            }
-        }
-    }
-
-    if (activation)
-    {
-        activation->forward_inplace(top_blob, opt);
-    }
-
-    return 0;
-}
-
 int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
     if (bottom_blob.dims != 3)
@@ -542,9 +435,9 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         return Convolution::forward(bottom_blob, top_blob, opt);
     }
 
-    if (use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
-        return forward_int8(bottom_blob, top_blob, opt);
+        return forward_int8_arm(bottom_blob, top_blob, opt);
     }
 
     int w = bottom_blob.w;
@@ -602,9 +495,9 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
     if (top_blob.empty())
         return -100;
 
-    if (opt.use_packing_layout == false && kernel_w == kernel_h && stride_w == 1 && stride_h == 1 && dilation_w != 1 && dilation_h == dilation_w)
+    if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
-        return forwardDilation(bottom_blob_bordered, top_blob, opt);
+        return forwardDilation_arm(bottom_blob_bordered, top_blob, opt);
     }
 
     const int maxk = kernel_w * kernel_h;
@@ -631,7 +524,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 #if __ARM_NEON
     if (elempack == 4 && out_elempack == 4)
     {
-        if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv1x1s1_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -640,7 +533,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv1x1s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -649,7 +542,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv3x3s1_winograd64_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -658,7 +551,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -667,7 +560,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -676,7 +569,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -750,7 +643,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 
     if (elempack == 1 && out_elempack == 4)
     {
-        if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv3x3s1_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);
 
@@ -759,7 +652,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);
 
@@ -768,7 +661,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 7 && kernel_h == 7 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv7x7s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt);
 
@@ -827,7 +720,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 
     if (elempack == 4 && out_elempack == 1)
     {
-        if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv1x1s1_sgemm_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);
 
@@ -836,7 +729,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv1x1s2_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);
 
@@ -845,7 +738,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             // TODO more proper condition
             conv3x3s1_winograd64_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt);
@@ -942,7 +835,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             if (use_sgemm1x1)
             {
@@ -958,7 +851,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
 //             conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
             conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);
@@ -968,7 +861,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             if (use_winograd3x3 && w <= 120 && h <= 120)
             {
@@ -985,7 +878,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
 //             conv3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
             if (outw >=8 && outh >=8)
@@ -998,7 +891,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 4 && kernel_h == 4 && stride_w == 4 && stride_h == 4 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 4 && kernel_h == 4 && dilation_w == 1 && dilation_h == 1 && stride_w == 4 && stride_h == 4)
         {
             conv4x4s4_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -1007,7 +900,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -1016,7 +909,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -1025,7 +918,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 7 && kernel_h == 7 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
             conv7x7s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -1034,7 +927,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
                 activation->forward_inplace(top_blob, opt);
             }
         }
-        else if (kernel_w == 7 && kernel_h == 7 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+        else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
             conv7x7s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -1094,61 +987,41 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
     return 0;
 }
 
-int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+int Convolution_arm::create_pipeline_int8_arm(const Option& opt)
 {
-    typedef void (*conv_int8_func)(const Mat&, Mat&, const Mat&, const Option&);
-    const int kernel_size = kernel_w;
-    const int stride = stride_w;
+    const int maxk = kernel_w * kernel_h;
+    const int num_input = weight_data_size / maxk / num_output;
+
+    use_winograd3x3_int8 = false;
+    use_sgemm1x1_int8 = false;
 
-    // kernel_size x stride
-    conv_int8_func conv_int8_func_table[7][4] =
+    if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
     {
-        {
-            conv1x1s1_int8_neon,
-            conv1x1s2_int8_neon,
-            0,
-            0
-        }, // kernel_size = 1
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 2
-        {
-            conv3x3s1_int8_neon,
-            conv3x3s2_int8_neon,
-            0,
-            0
-        }, // kernel_size = 3
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 4
-        {
-            conv5x5s1_int8_neon,
-            conv5x5s2_int8_neon,
-            0,
-            0
-        }, // kernel_size = 5
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 6
-        {            
-            conv7x7s1_int8_neon,           
-            conv7x7s2_int8_neon,
-            0,
-            0
-        }  // kernel_size = 7                
-    };
-
-    conv_int8_func conv_int8 = conv_int8_func_table[kernel_size-1][stride-1];
-    if (!conv_int8)
+        use_winograd3x3_int8 = true;
+//         conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);
+        conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);
+    }
+
+    if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+    {
+        conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_data_int8, num_input, num_output);
+    }
+    else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+    {
+        use_sgemm1x1_int8 = true;
+        conv1x1s1_sgemm_transform_kernel_int8_neon(weight_data, weight_1x1s1_sgemm_data_int8, num_input, num_output);
+    }
+    else
+    {
+        conv_im2col_sgemm_transform_kernel_int8_neon(weight_data, weight_sgemm_data_int8, num_input, num_output, maxk);
+    }
+
+    return 0;
+}
+
+int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    if (dilation_w > 1 || dilation_h > 1)
     {
         return Convolution::forward(bottom_blob, top_blob, opt);
     }
@@ -1158,23 +1031,18 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
     int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
 
+//     fprintf(stderr, "Convolution_arm input %d x %d  ksize=%d %d  stride=%d %d\n", w, h, kernel_w, kernel_h, stride_w, stride_h);
+
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
     Mat bottom_blob_unbordered = bottom_blob;
     if (elemsize != 1)
     {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
-            return -100;
+        Option opt_g = opt;
+        opt_g.blob_allocator = opt.workspace_allocator;
 
-        // quantize, scale and round to nearest
-        {
-            Option opt_g = opt;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
-
-            quantize->forward(bottom_blob, bottom_blob_int8, opt_g);
-        }
-
-        bottom_blob_unbordered = bottom_blob_int8;             
+        quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g);
     }
 
     Mat bottom_blob_bordered = bottom_blob_unbordered;
@@ -1186,8 +1054,9 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
     }
     else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
-        int wpad = kernel_size + (w - 1) / stride * stride - w;
-        int hpad = kernel_size + (h - 1) / stride * stride - h;
+        // tensorflow padding=SAME or onnx padding=SAME_UPPER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
@@ -1197,8 +1066,9 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
     }
     else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
     {
-        int wpad = kernel_size + (w - 1) / stride * stride - w;
-        int hpad = kernel_size + (h - 1) / stride * stride - h;
+        // onnx padding=SAME_LOWER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
@@ -1212,8 +1082,15 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
     w = bottom_blob_bordered.w;
     h = bottom_blob_bordered.h;
 
-    int outw = (w - kernel_size) / stride + 1;
-    int outh = (h - kernel_size) / stride + 1;
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
+
+    // int8
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
+
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
 
     // int8
     if (use_int8_requantize == true)
@@ -1223,90 +1100,107 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
         if (top_blob_tm.empty())
             return -100;
         
-        top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-        if (top_blob.empty())
-            return -100; 
-
-        if (use_sgemm1x1)
-        {              
-            conv1x1s1_sgemm_int8_requant_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_int8_data, bias_data, requantize_scales, opt);
-            
+        if (use_sgemm1x1_int8)
+        {
+            std::vector<float> requantize_scales;
+            for (int p=0; p<num_output; p++)
+            {
+                float scale_in;
+                if (weight_data_int8_scales[p] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                float scale_out = top_blob_int8_scale;
+
+                requantize_scales.push_back(scale_in);
+                requantize_scales.push_back(scale_out);
+            }
+
+            conv1x1s1_sgemm_int8_requant_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_data_int8, bias_data, requantize_scales, opt);
+
             if (activation)
             {
                 activation->forward_inplace(top_blob, opt);
-            }  
+            }
 
             return 0;
         }
-        else if (use_winograd3x3)
+        else if (use_winograd3x3_int8)
         {
-            // conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_int8_data, opt);
-            conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_int8_data, opt);
+//             conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);
+            conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3s2_int8_data, opt);
+            conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3s2_data_int8, opt);
         }
         else
         {
-            conv_int8(bottom_blob_bordered, top_blob_tm, weight_sgemm_int8_data, opt);     
+            conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob_tm, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt);
         }
 
         // requantize, reverse scale inplace
         #pragma omp parallel for num_threads(opt.num_threads)
         for (int p=0; p<num_output; p++)
         {
-            ncnn::Option opt_g = opt;
+            Option opt_g = opt;
             opt_g.num_threads = 1;
             opt_g.blob_allocator = top_blob.allocator;
 
             Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1);
             Mat top_blob_g = top_blob.channel_range(p, 1);
-            requantize_ops[p]->forward(top_blob_tm_g, top_blob_g, opt_g);
-        }                     
+
+            // requantize and relu
+            float scale_in;
+            if (weight_data_int8_scales[p] == 0)
+                scale_in = 0;
+            else
+                scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+            float scale_out = top_blob_int8_scale;//FIXME load param
+
+            requantize_int8_to_int8(top_blob_tm, top_blob, scale_in, scale_out, &bias_data[p], bias_term ? 1 : 0, 0, opt_g);
+        }
     }
     else
     {
-        top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-        if (top_blob.empty())
-            return -100; 
-
-        if (use_sgemm1x1)
+        if (use_sgemm1x1_int8)
         {
-            conv1x1s1_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_int8_data, opt);
+            conv1x1s1_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_data_int8, opt);
         }
-        else if (use_winograd3x3)
+        else if (use_winograd3x3_int8)
         {
-            // conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, opt);
-            // conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, opt);
-            conv3x3s1_winograd43_dequant_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, bias_data, dequantize_scales, opt);
-
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
-            }  
-
-            return 0;
+//             conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);
+            conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);
         }
         else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
         {
-            conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob, weight_3x3s2_int8_data, opt);
+            conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data_int8, opt);
         }
         else
         {
-            conv_int8(bottom_blob_bordered, top_blob, weight_sgemm_int8_data, opt);
+            conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt);
         }        
 
         // dequantize, reverse scale inplace
         #pragma omp parallel for num_threads(opt.num_threads)
         for (int p=0; p<num_output; p++)
         {
-            ncnn::Option opt_g = opt;
+            Option opt_g = opt;
             opt_g.num_threads = 1;
             opt_g.blob_allocator = top_blob.allocator;
 
             Mat top_blob_g = top_blob.channel_range(p, 1);
-            dequantize_ops[p]->forward_inplace(top_blob_g, opt_g);
+
+            // dequantize
+            float scale_in;
+            if (weight_data_int8_scales[p] == 0)
+                scale_in = 0;
+            else
+                scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+            dequantize_int32_to_float32(top_blob_g, scale_in, &bias_data[p], bias_term ? 1 : 0, opt_g);
         }
     }
 
@@ -1318,5 +1212,88 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O
     return 0;
 }
 
+int Convolution_arm::forwardDilation_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    size_t elemsize = bottom_blob.elemsize;
+
+    const int kernel_size = kernel_w;
+    const int stride = stride_w;
+    const int dilation = dilation_w;
+    const int kernel_extent = dilation * (kernel_size - 1) + 1;
+
+    int outw = (w - kernel_extent) / stride + 1;
+    int outh = (h - kernel_extent) / stride + 1;
+
+    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // Make (dilation * dilation) batches
+    Mat inner_bottom_blob;
+    Mat inner_top_blob;
+    for (int x = 0; x < dilation; x ++)
+    {
+        for (int y = 0; y < dilation; y ++)
+        {
+            int inner_w = (w - y + dilation - 1) / dilation;
+            int inner_h = (h - x + dilation - 1) / dilation;
+
+            int inner_outw = (inner_w - kernel_size) / stride + 1;
+            int inner_outh = (inner_h - kernel_size) / stride + 1;
+
+            inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator);
+            if (inner_bottom_blob.empty())
+                return -100;
+
+            inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator);
+            if (inner_top_blob.empty())
+                return -100;
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int c = 0; c < bottom_blob.c; c ++)
+            {
+                float *outptr = inner_bottom_blob.channel(c);
+
+                for (int i = 0; i < inner_h; i ++)
+                {
+                    const float *ptr = (const float *) bottom_blob.channel(c) + dilation * i * w + x * w + y;
+                    for (int j = 0; j < inner_w; j ++)
+                    {
+                        outptr[j] = ptr[j*dilation];
+                    }
+                    outptr += inner_w;
+                }
+            }
+
+            Option opt_g = opt;
+            opt_g.blob_allocator = inner_top_blob.allocator;
+            convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g);
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int c = 0; c < num_output; c ++)
+            {
+                float *outptr = (float *) top_blob.channel(c) + x * outw + y;
+                for (int i = 0; i < inner_outh; i ++)
+                {
+                    const float *ptr = (const float *) inner_top_blob.channel(c) + i * inner_outw;
+                    for (int j = 0; j < inner_outw; j ++)
+                    {
+                        outptr[j*dilation] = ptr[j];
+                    }
+                    outptr += dilation * outw;
+                }
+            }
+        }
+    }
+
+    if (activation)
+    {
+        activation->forward_inplace(top_blob, opt);
+    }
+
+    return 0;
+}
 
 } // namespace ncnn

src/layer/arm/convolution_arm.h

@@ -27,9 +27,12 @@ public:
     virtual int create_pipeline(const Option& opt);
     virtual int destroy_pipeline(const Option& opt);
 
-    virtual int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
-    virtual int forwardDilation(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
+protected:
+    int create_pipeline_int8_arm(const Option& opt);
+    int forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+    int forwardDilation_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
     Layer* activation;
@@ -38,12 +41,7 @@ public:
     Mat weight_3x3_winograd64_data;
     Mat weight_1x1_sgemm_data;
     Mat weight_3x3s2_data;
-    Mat weight_3x3s2_int8_data;
-    Mat weight_1x1s1_sgemm_int8_data;
-    Mat weight_3x3_winograd23_data;
-    Mat weight_sgemm_int8_data;
     Mat weight_sgemm_data;
-    std::vector<Mat> weight_3x3_winograd23_int8_data;
 
     // forwardDilation
     Layer* convolution_dilation1;
@@ -55,6 +53,14 @@ public:
 
     Mat weight_3x3_winograd64_data_pack4;
     Mat weight_1x1_sgemm_data_pack4;
+
+    // int8
+    bool use_winograd3x3_int8;
+    bool use_sgemm1x1_int8;
+    Mat weight_3x3s2_data_int8;
+    Mat weight_1x1s1_sgemm_data_int8;
+    Mat weight_sgemm_data_int8;
+    std::vector<Mat> weight_3x3_winograd23_data_int8;
 };
 
 } // namespace ncnn

src/layer/arm/convolution_sgemm_int8.h

@@ -13,6 +13,8 @@
 // specific language governing permissions and limitations under the License.
 
 #if __aarch64__
+
+#if 0// FIXME chgemm produce wrong result
 #include "gemm_symm_int8.h"
 
 static void conv_im2col_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size)
@@ -86,6 +88,691 @@ static void conv_im2col_sgemm_int8_neon(const Mat &bottom_blob, Mat &top_blob, c
 }
 #else
 static void conv_im2col_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size)
+{
+    const signed char* kernel = _kernel;
+
+    // kernel memory packed 4 x 4
+    kernel_tm.create(4*kernel_size, inch, outch/4 + outch%4, (size_t)1u);
+
+    int nn_outch = 0;
+    int remain_outch_start = 0;
+
+    nn_outch = outch >> 2;
+    remain_outch_start = nn_outch << 2;
+
+    for (int pp=0; pp<nn_outch; pp++)
+    {
+        int p = pp * 4;
+
+        const signed char* k0 = kernel + (p+0)*inch*kernel_size;
+        const signed char* k1 = kernel + (p+1)*inch*kernel_size;
+        const signed char* k2 = kernel + (p+2)*inch*kernel_size;
+        const signed char* k3 = kernel + (p+3)*inch*kernel_size;
+
+        signed char* ktmp = kernel_tm.channel(p/4);
+
+        int q=0;
+        for (; q+1<inch*kernel_size; q+=2)
+        {
+            ktmp[0] = k0[0];
+            ktmp[1] = k0[1];
+            ktmp[2] = k1[0];
+            ktmp[3] = k1[1];
+            ktmp[4] = k2[0];
+            ktmp[5] = k2[1];
+            ktmp[6] = k3[0];
+            ktmp[7] = k3[1];
+
+            ktmp += 8;
+
+            k0 += 2;
+            k1 += 2;
+            k2 += 2;
+            k3 += 2;
+        }
+
+        for (; q<inch*kernel_size; q++)
+        {
+            ktmp[0] = k0[0];
+            ktmp[1] = k1[0];
+            ktmp[2] = k2[0];
+            ktmp[3] = k3[0];
+            ktmp += 4;
+
+            k0 += 1;
+            k1 += 1;
+            k2 += 1;
+            k3 += 1;
+        }
+    }
+
+    for (int p=remain_outch_start; p<outch; p++)
+    {
+        const signed char* k0 = kernel + (p+0)*inch*kernel_size;
+
+        signed char* ktmp = kernel_tm.channel(p/4 + p%4);
+
+        int q=0;
+        for (; q+1<inch*kernel_size; q=q+2)
+        {
+            ktmp[0] = k0[0];
+            ktmp[1] = k0[1];
+            ktmp += 2;
+            k0 += 2;
+        }
+
+        for (; q<inch*kernel_size; q++)
+        {
+            ktmp[0] = k0[0];
+            ktmp++;
+            k0++;
+        }
+    }
+}
+
+static void conv_im2col_sgemm_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &kernel_tm, \
+            const int kernel_w, const int kernel_h, const int stride_w, const int stride_h, const Option& opt)
+{
+    int w = bottom_blob.w;
+    int inch = bottom_blob.c;
+
+    int outw = top_blob.w;
+    int outh = top_blob.h;
+    int outch = top_blob.c;
+
+    // im2row
+    Mat bottom_im2row(kernel_h*kernel_w*inch, outw*outh, 1UL, opt.workspace_allocator);
+    {
+        int out_stride = kernel_h*kernel_w*inch*outw;
+        signed char* ret = (signed char*)bottom_im2row;
+
+        // #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i=0; i<outh; i++)
+        {
+            int retID = out_stride * i;
+            for (int j=0; j<outw; j++)
+            {
+                for (int p=0; p<inch; p++)
+                {
+                    const signed char* input = bottom_blob.channel(p);
+
+                    for (int u=0; u<kernel_h; u++)
+                    {
+                        for (int v=0; v<kernel_w; v++)
+                        {
+                            int row = u + i * stride_h;
+                            int col = v + j * stride_w;
+                            int index = row * w + col;
+                            ret[retID] = input[index];
+                            retID++;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    int kernel_size = kernel_w * kernel_h;
+    int out_size = outw * outh;
+
+    // int M = outch;  // outch
+    int N = outw * outh; // outsize or out stride
+    int K = kernel_w * kernel_h * inch; // ksize * inch
+
+    // bottom_im2row memory packed 4 x 4
+    Mat bottom_tm(4*kernel_size, inch, out_size/4 + out_size%4, (size_t)1u, opt.workspace_allocator);
+    {
+        int nn_size = out_size >> 2;
+        int remain_size_start = nn_size << 2;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int ii=0; ii<nn_size; ii++)
+        {
+            int i = ii * 4;
+
+            const signed char* img0 = bottom_im2row.row<signed char>(i);
+            const signed char* img1 = bottom_im2row.row<signed char>(i+1);
+            const signed char* img2 = bottom_im2row.row<signed char>(i+2);
+            const signed char* img3 = bottom_im2row.row<signed char>(i+3);
+
+            signed char* tmpptr = bottom_tm.channel(i/4);
+
+            int q = 0;
+            for (; q+1<inch*kernel_size; q=q+2)
+            {
+                tmpptr[0] = img0[0];
+                tmpptr[1] = img0[1];
+                tmpptr[2] = img1[0];
+                tmpptr[3] = img1[1];
+                tmpptr[4] = img2[0];
+                tmpptr[5] = img2[1];
+                tmpptr[6] = img3[0];
+                tmpptr[7] = img3[1];
+
+                tmpptr += 8;
+                img0 += 2;
+                img1 += 2;
+                img2 += 2;
+                img3 += 2;
+            }
+
+            for (; q<inch*kernel_size; q++)
+            {
+                tmpptr[0] = img0[0];
+                tmpptr[1] = img1[0];
+                tmpptr[2] = img2[0];
+                tmpptr[3] = img3[0];
+
+                tmpptr += 4;
+                img0 += 1;
+                img1 += 1;
+                img2 += 1;
+                img3 += 1;
+            }
+        }
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i=remain_size_start; i<out_size; i++)
+        {
+            const signed char* img0 = bottom_im2row.row<signed char>(i);
+
+            signed char* tmpptr = bottom_tm.channel(i/4 + i%4);
+
+            int q=0;
+            for (; q+1<inch*kernel_size; q=q+2)
+            {
+                tmpptr[0] = img0[0];
+                tmpptr[1] = img0[1];
+
+                tmpptr += 2;
+                img0 += 2;
+            }
+
+            for (; q<inch*kernel_size; q++)
+            {
+                tmpptr[0] = img0[0];
+
+                tmpptr += 1;
+                img0 += 1;
+            }
+        }
+    }
+
+    // 4x4
+    // sgemm(int M, int N, int K, float* A, float* B, float* C)
+    {
+        // int M = outch;  // outch
+        // int N = outw * outh; // outsize or out stride
+        // int L = kernel_w * kernel_h * inch; // ksize * inch
+
+        int nn_outch = 0;
+        int remain_outch_start = 0;
+
+        nn_outch = outch >> 2;
+        remain_outch_start = nn_outch << 2;
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int pp=0; pp<nn_outch; pp++)
+        {
+            int i = pp * 4;
+
+            int* output0 = top_blob.channel(i);
+            int* output1 = top_blob.channel(i+1);
+            int* output2 = top_blob.channel(i+2);
+            int* output3 = top_blob.channel(i+3);
+
+            int j=0;
+            for (; j+3<N; j=j+4)
+            {
+                const signed char* vb = bottom_tm.channel(j/4);
+                const signed char* va = kernel_tm.channel(i/4);
+
+#if __ARM_NEON
+                asm volatile(
+                    "prfm   pldl1keep, [%4, #128]        \n"
+                    "prfm   pldl1keep, [%5, #128]        \n"
+                    "eor    v16.16b, v16.16b, v16.16b    \n" // sum0
+                    "eor    v17.16b, v17.16b, v17.16b    \n" // sum1
+                    "eor    v18.16b, v18.16b, v18.16b    \n" // sum2
+                    "eor    v19.16b, v19.16b, v19.16b    \n" // sum3
+
+                    "lsr    w4, %w12, #2                 \n"// r4 = nn = L >> 2
+                    "cmp    w4, #0                       \n"
+                    "beq    1f                           \n"
+
+                    "0:                                  \n"// for (; k+3<L; k=k+4)
+                    "ld1    {v0.16b}, [%4]               \n"// i0, i1, i2, i3
+                    "ld1    {v4.16b}, [%5]               \n"// k0, k1, k2, k3
+                    "add    %4, %4, #16                  \n"
+                    "add    %5, %5, #16                  \n"
+
+                    "rev32    v1.8h, v0.8h               \n"// i1, i0, i3, i2
+                    "rev64    v2.4s, v0.4s               \n"// i2, i3, i0, i1
+                    "rev64    v3.8h, v0.8h               \n"// i3, i2, i1, i0
+
+                    "smull	  v8.8h, v4.8b, v0.8b        \n"
+                    "smull	  v9.8h, v4.8b, v1.8b        \n"
+                    "smull	  v10.8h, v4.8b, v2.8b       \n"
+                    "smull	  v11.8h, v4.8b, v3.8b       \n"
+
+                    "prfm     pldl1keep, [%4, #128]      \n"
+                    "prfm     pldl1keep, [%5, #128]      \n"
+
+                    "smlal2	  v8.8h, v4.16b, v0.16b      \n"
+                    "smlal2	  v9.8h, v4.16b, v1.16b      \n"
+                    "smlal2	  v10.8h, v4.16b, v2.16b     \n"
+                    "smlal2	  v11.8h, v4.16b, v3.16b     \n"
+
+                    "sadalp	  v16.4s, v8.8h              \n"// i0k0, i1k1, i2k2, i3k3
+                    "sadalp	  v17.4s, v9.8h              \n"// i1k0, i0k1, i3k2, i2k3
+                    "sadalp	  v18.4s, v10.8h             \n"// i2k0, i3k1, i0k2, i1k3
+                    "sadalp	  v19.4s, v11.8h             \n"// i3k0, i2k1, i1k2, i0k3
+
+                    "subs     w4, w4, #1                 \n"
+                    "bne      0b                         \n"
+
+                    "1:                                  \n"// for (; k+1<L; k=k+2)
+
+                    // remain loop
+                    "and      w4, %w12, #3               \n"// w4 = remain = K & 3;
+                    "cmp      w4, #0                     \n"
+                    "beq      3f                         \n"
+
+                    "lsr      w4, w4, #1                 \n"// r4 = nn = L >> 1
+                    "cmp      w4, #0                     \n"
+                    "beq      3f                         \n"
+
+                    "2:                                  \n"// for (; k+1<L; k=k+2)
+
+                    "ld1      {v0.8b}, [%4]              \n"// i0, i1, i2, i3
+                    "ld1      {v4.8b}, [%5]              \n"// k0, k1, k2, k3
+                    "add      %4, %4, #8                 \n"
+                    "add      %5, %5, #8                 \n"
+
+                    "rev32	  v1.4h, v0.4h               \n"// i2, i3, i0, i1
+                    "rev64    v2.2s, v0.2s               \n"// i1, i0, i3, i2
+                    "rev64    v3.4h, v0.4h               \n"// i0, i1, i2, i3
+
+                    "smull	  v8.8h, v4.8b, v0.8b        \n"
+                    "smull	  v9.8h, v4.8b, v1.8b        \n"
+                    "smull    v10.8h, v4.8b, v2.8b       \n"
+                    "smull	  v11.8h, v4.8b, v3.8b       \n"
+                    "sadalp	  v16.4s, v8.8h              \n"
+                    "sadalp	  v17.4s, v9.8h              \n"
+                    "sadalp	  v18.4s,v10.8h              \n"
+                    "sadalp	  v19.4s,v11.8h              \n"
+
+                    "subs     w4, w4, #1                 \n"
+                    "bne      2b                         \n"
+
+                    "3:                                  \n"// realloc
+
+                    "mov      v20.s[0], v16.s[0]         \n"
+                    "mov      v20.s[1], v17.s[0]         \n"
+                    "mov      v20.s[2], v18.s[0]         \n"
+                    "mov      v20.s[3], v19.s[0]         \n"
+
+                    "mov      v21.s[0], v17.s[1]         \n"
+                    "mov      v21.s[1], v16.s[1]         \n"
+                    "mov      v21.s[2], v19.s[1]         \n"
+                    "mov      v21.s[3], v18.s[1]         \n"
+
+                    "mov      v22.s[0], v18.s[2]         \n"
+                    "mov      v22.s[1], v19.s[2]         \n"
+                    "mov      v22.s[2], v16.s[2]         \n"
+                    "mov      v22.s[3], v17.s[2]         \n"
+
+                    "mov      v23.s[0], v19.s[3]         \n"
+                    "mov      v23.s[1], v18.s[3]         \n"
+                    "mov      v23.s[2], v17.s[3]         \n"
+                    "mov      v23.s[3], v16.s[3]         \n"
+
+                    "and      w4, %w12, #1               \n"// w4 = remain = K & 1;
+                    "cmp      w4, #0                     \n"
+                    "beq      5f                         \n"
+
+                    "4:                                  \n"
+                    "ld1      {v0.8b}, [%4]              \n"
+                    "ld1      {v1.8b}, [%5]              \n"
+                    "add      %4, %4, #4                 \n"
+                    "add      %5, %5, #4                 \n"
+
+                    "sshll    v0.8h, v0.8b, #0           \n"// i0[0], i1[0], i2[0], i3[0]
+                    "sshll    v1.8h, v1.8b, #0           \n"// k0[0], k1[0], k2[0], k3[0]
+
+                    "smlal    v20.4s, v0.4h, v1.h[0]     \n"// i0k0, i1k0, i2k0, i3k0
+                    "smlal    v21.4s, v0.4h, v1.h[1]     \n"// i0k1, i1k1, i2k1, i3k1
+                    "smlal    v22.4s, v0.4h, v1.h[2]     \n"// i0k2, i1k2, i2k2, i3k2
+                    "smlal    v23.4s, v0.4h, v1.h[3]     \n"// i0k3, i1k3, i2k3, i3k3
+
+                    "subs     w4, w4, #1                 \n"
+
+                    "bne      2b                         \n"
+
+                    "5:                                  \n"
+
+                    "st1      {v20.4s}, [%0]             \n"
+                    "st1      {v21.4s}, [%1]             \n"
+                    "st1      {v22.4s}, [%2]             \n"
+                    "st1      {v23.4s}, [%3]             \n"
+
+                    : "=r"(output0), // %0
+                      "=r"(output1), // %1
+                      "=r"(output2), // %2
+                      "=r"(output3), // %3
+                      "=r"(vb),      // %4
+                      "=r"(va)       // %5
+                    : "0"(output0),
+                      "1"(output1),
+                      "2"(output2),
+                      "3"(output3),
+                      "4"(vb),
+                      "5"(va),
+                      "r"(K)         // %12
+                    : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
+                );
+#else
+                int sum0[4] = {0};
+                int sum1[4] = {0};
+                int sum2[4] = {0};
+                int sum3[4] = {0};
+
+                int k=0;
+
+                for (; k+1<K; k=k+2)
+                {
+                    for (int n=0; n<4; n++)
+                    {
+                        sum0[n] += (int)va[0] * vb[2*n];   // k0
+                        sum0[n] += (int)va[1] * vb[2*n+1];
+
+                        sum1[n] += (int)va[2] * vb[2*n];   // k1
+                        sum1[n] += (int)va[3] * vb[2*n+1];
+
+                        sum2[n] += (int)va[4] * vb[2*n];   // k2
+                        sum2[n] += (int)va[5] * vb[2*n+1];
+
+                        sum3[n] += (int)va[6] * vb[2*n];   // k3
+                        sum3[n] += (int)va[7] * vb[2*n+1];
+                    }
+
+                    va += 8;
+                    vb += 8;
+                }
+
+                for (; k<K; k++)
+                {
+                    for (int n=0; n<4; n++)
+                    {
+                        sum0[n] += (int)va[0] * vb[n];
+                        sum1[n] += (int)va[1] * vb[n];
+                        sum2[n] += (int)va[2] * vb[n];
+                        sum3[n] += (int)va[3] * vb[n];
+                    }
+
+                    va += 4;
+                    vb += 4;
+                }
+
+                for (int n=0; n<4; n++)
+                {
+                    output0[n] = sum0[n];
+                    output1[n] = sum1[n];
+                    output2[n] = sum2[n];
+                    output3[n] = sum3[n];
+                }
+#endif
+                output0 += 4;
+                output1 += 4;
+                output2 += 4;
+                output3 += 4;
+            }
+
+            for (; j<N; j++)
+            {
+                const signed char* vb = bottom_tm.channel(j/4 + j%4);
+                const signed char* va = kernel_tm.channel(i/4);
+
+#if 0//__ARM_NEON
+                int32x4_t _sum = vdupq_n_s32(0);
+
+                int k=0;
+
+                for (; k+3<K; k=k+4)
+                {
+                    int8x8_t _r0 = vld1_s8(vb);     // i0[0-3]
+                    int8x8x2_t _k = vld2_s8(va);    // k0[0-1], k1[0-1], k2[0-1], k3[0-1];k0[2-3], k1[2-3], k2[2-3], k3[2-3]
+
+                    int16x8_t _r0_s16 = vmovl_s8(_r0);          // i0[0],i0[1],i0[2],i0[3]
+                    int16x8_t _k02_s16 = vmovl_s8(_k.val[0]);   // k0[0],k1[0],k2[0],k3[0],k0[2],k1[2],k2[2],k3[2]
+                    int16x8_t _k13_s16 = vmovl_s8(_k.val[1]);   // k0[1],k1[1],k2[1],k3[1],k0[3],k1[3],k2[3],k3[3]
+
+                    _sum = vmlal_lane_s16(_sum, vget_low_s16(_k02_s16), vget_low_s16(_r0_s16), 0);    // i0[0]*k[0-3][0]
+                    _sum = vmlal_lane_s16(_sum, vget_low_s16(_k13_s16), vget_low_s16(_r0_s16), 1);    // i0[1]*k[0-3][1]
+                    _sum = vmlal_lane_s16(_sum, vget_high_s16(_k02_s16), vget_low_s16(_r0_s16), 2);   // i0[2]*k[0-3][2]
+                    _sum = vmlal_lane_s16(_sum, vget_high_s16(_k13_s16), vget_low_s16(_r0_s16), 3);   // i0[3]*k[0-3][3]
+
+                    va += 16;
+                    vb += 4;
+                }
+
+                for (; k+1<K; k=k+2)
+                {
+                    int8x8_t _r0 = vld1_s8(vb);     // i0[0-3]
+                    int8x8_t _k = vld1_s8(va);      // k0[0-1], k1[0-1], k2[0-1], k3[0-1]
+
+                    _r0[2] = _r0[0];
+                    _r0[3] = _r0[1];
+                    _r0[4] = _r0[0];
+                    _r0[5] = _r0[1];
+                    _r0[6] = _r0[0];
+                    _r0[7] = _r0[1];
+
+                    int16x8_t _tp0 = vmull_s8(_k, _r0);
+                    _sum = vpadalq_s16(_sum, _tp0);
+
+                    va += 8;
+                    vb += 2;
+                }
+
+                for (; k<K; k++)
+                {
+                    int8x8_t _r0 = vld1_s8(vb);     // i0[0-3]
+                    int8x8_t _k = vld1_s8(va);      // k[0-3][0]
+
+                    int16x8_t _tp0 = vmull_s8(_k, _r0);
+
+                    _sum = vaddw_s16(_sum, vget_low_s16(_tp0));
+
+                    va += 4;
+                    vb += 1;
+                }
+
+                vst1q_lane_s32(output0, _sum, 0);
+                vst1q_lane_s32(output1, _sum, 1);
+                vst1q_lane_s32(output2, _sum, 2);
+                vst1q_lane_s32(output3, _sum, 3);
+#else
+                int sum0 = 0;
+                int sum1 = 0;
+                int sum2 = 0;
+                int sum3 = 0;
+
+                int k=0;
+
+                for (; k+1<K; k=k+2)
+                {
+                    sum0 += (int)va[0] * vb[0];
+                    sum0 += (int)va[1] * vb[1];
+
+                    sum1 += (int)va[2] * vb[0];
+                    sum1 += (int)va[3] * vb[1];
+
+                    sum2 += (int)va[4] * vb[0];
+                    sum2 += (int)va[5] * vb[1];
+
+                    sum3 += (int)va[6] * vb[0];
+                    sum3 += (int)va[7] * vb[1];
+
+                    va += 8;
+                    vb += 2;
+                }
+
+                for (; k<K; k++)
+                {
+                    sum0 += (int)va[0] * vb[0];
+                    sum1 += (int)va[1] * vb[0];
+                    sum2 += (int)va[2] * vb[0];
+                    sum3 += (int)va[3] * vb[0];
+
+                    va += 4;
+                    vb += 1;
+                }
+
+                output0[0] = sum0;
+                output1[0] = sum1;
+                output2[0] = sum2;
+                output3[0] = sum3;
+#endif
+                output0++;
+                output1++;
+                output2++;
+                output3++;
+            }
+        }
+
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int i=remain_outch_start; i<outch; i++)
+        {
+            int* output = top_blob.channel(i);
+
+            int j=0;
+            for (; j+3<N; j=j+4)
+            {
+                const signed char* vb = bottom_tm.channel(j/4);
+                const signed char* va = kernel_tm.channel(i/4 + i%4);
+
+#if __ARM_NEON
+                int32x4_t _sum = vdupq_n_s32(0);
+
+                int k=0;
+                for (; k+1<K; k=k+2)
+                {
+                    int8x8_t _r0 = vld1_s8(vb);     // i0[0-1], i1[0-1], i2[0-1], i3[0-1]
+                    int8x8_t _k = vld1_s8(va);      // k0[0-1]
+
+                    _k[2] = _k[0];
+                    _k[3] = _k[1];
+                    _k[4] = _k[0];
+                    _k[5] = _k[1];
+                    _k[6] = _k[0];
+                    _k[7] = _k[1];
+
+                    int16x8_t _tp0 = vmull_s8(_k, _r0);
+                    _sum = vpadalq_s16(_sum, _tp0);
+
+                    va += 2;
+                    vb += 8;
+                }
+
+                for (; k<K; k++)
+                {
+                    int8x8_t _r0 = vld1_s8(vb);     // i0[0], i1[0], i2[0], i3[0]
+                    int8x8_t _k = vld1_s8(va);      // k[0][0]
+
+                    int16x8_t _r0_s16 = vmovl_s8(_r0);
+                    int16x8_t _k_s16 = vmovl_s8(_k);
+
+                    _sum = vmlal_lane_s16(_sum, vget_low_s16(_r0_s16), vget_low_s16(_k_s16), 0); // i0k0, i1k0, i2k0, i3k0
+
+                    va += 1;
+                    vb += 4;
+                }
+
+                vst1q_s32(output, _sum);
+#else
+                int sum[4] = {0};
+                int k=0;
+                for (; k+1<K; k=k+2)
+                {
+                    for (int n=0; n<4; n++)
+                    {
+                        sum[n] += (int)va[0] * vb[2*n];
+                        sum[n] += (int)va[1] * vb[2*n+1];
+                    }
+                    va += 2;
+                    vb += 8;
+                }
+
+                for (; k<K; k++)
+                {
+                    for (int n=0; n<4; n++)
+                    {
+                        sum[n] += (int)va[0] * vb[n];
+                    }
+                    va += 1;
+                    vb += 4;
+                }
+
+                for (int n=0; n<4; n++)
+                {
+                    output[n] = sum[n];
+                }
+#endif
+                output += 4;
+            }
+
+            for (; j<N; j++)
+            {
+                int sum = 0;
+
+                const signed char* vb = bottom_tm.channel(j/4 + j%4);
+                const signed char* va = kernel_tm.channel(i/4 + i%4);
+
+                for (int k=0; k<K; k++)
+                {
+                    sum += (int)va[0] * vb[0];
+
+                    va += 1;
+                    vb += 1;
+                }
+                output[0] = sum;
+
+                output++;
+            }
+        }
+    }
+
+    // // sgemm(int M, int N, int K, float* A, float* B, float* C)
+    // {
+    //     for (int i=0; i<M; i++)
+    //     {
+    //         int* output = top_blob.channel(i);
+
+    //         for (int j=0; j<N; j++)
+    //         {
+    //             int sum = 0;
+
+    //             signed char* vb = (signed char*)bottom_im2row + K * j;
+    //             const signed char* va = kernel + K * i;
+
+    //             for (int k=0; k<K; k++)
+    //             {
+    //                 sum += (int)va[0] * vb[0];
+
+    //                 va += 1;
+    //                 vb += 1;
+    //             }
+    //             output[0] = sum;
+
+    //             output++;
+    //         }
+    //     }
+    // }
+}
+#endif
+#else
+static void conv_im2col_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size)
 {
 
     const signed char* kernel = _kernel;

src/layer/arm/convolutiondepthwise_arm.cpp

@@ -84,11 +84,9 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
         activation->create_pipeline(opt);
     }
 
-    if (use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
         support_packing = false;
-
-        return 0;
     }
 
     // create Convolution op for each group
@@ -98,361 +96,147 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt)
     // depth-wise
     if (channels == group && group == num_output)
     {
-        int elempack = (opt.use_packing_layout && channels % 4 == 0) ? 4 : 1;
-
-#if __ARM_NEON
-        // pack4
-        if (elempack == 4)
+        if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
         {
-            Mat weight_data_r2 = weight_data.reshape(maxk, group);
-            convert_packing(weight_data_r2, weight_data_pack4, 4);
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+            {
+                return 0;
+            }
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+            {
+                return 0;
+            }
         }
-#endif // __ARM_NEON
-    }
-    else
-    {
-        // group convolution
-        for (int i=0; i<(int)group_ops.size(); i++)
-            delete group_ops[i];
-
-        group_ops.clear();
+        else
+        {
+            int elempack = (opt.use_packing_layout && channels % 4 == 0) ? 4 : 1;
 
-        const int channels_g = channels / group;
-        const int num_output_g = num_output / group;
+#if __ARM_NEON
+            // pack4
+            if (elempack == 4)
+            {
+                Mat weight_data_r2 = weight_data.reshape(maxk, group);
+                convert_packing(weight_data_r2, weight_data_pack4, 4);
 
-        group_ops.resize(group);
+                return 0;
+            }
+#endif // __ARM_NEON
 
-        for (int g=0; g<group; g++)
-        {
-            Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
-            Mat bias_data_g;
-            if (bias_term)
-                bias_data_g = bias_data.range(num_output_g * g, num_output_g);
-
-            ncnn::Layer* op = ncnn::create_layer(ncnn::LayerType::Convolution);
-
-            // set param
-            ncnn::ParamDict pd;
-            pd.set(0, num_output_g);// num_output
-            pd.set(1, kernel_w);
-            pd.set(11, kernel_h);
-            pd.set(2, dilation_w);
-            pd.set(12, dilation_h);
-            pd.set(3, stride_w);
-            pd.set(13, stride_h);
-            pd.set(4, 0);// pad_w
-            pd.set(14, 0);// pad_h
-            pd.set(5, bias_term);
-            pd.set(6, maxk * channels_g * num_output_g);// weight_data_size
-            pd.set(8, int8_scale_term);
-            pd.set(9, activation_type);
-            pd.set(10, activation_params);
-
-            op->load_param(pd);
-
-            // set weights
-            if (bias_term)
+            if (elempack == 1)
             {
-                ncnn::Mat weights[4];
-                weights[0] = weight_data_g;
-                weights[1] = bias_data_g;
-
-                if (int8_scale_term)
+                if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
                 {
-                    weights[2] = weight_data_int8_scales.range(g, 1);
-                    weights[3] = bottom_blob_int8_scales.range(g, 1);
+                    return 0;
                 }
-
-                op->load_model(ModelBinFromMatArray(weights));
-            }
-            else
-            {
-                ncnn::Mat weights[3];
-                weights[0] = weight_data_g;
-
-                if (int8_scale_term)
+                if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
                 {
-                    weights[1] = weight_data_int8_scales.range(g, 1);
-                    weights[2] = bottom_blob_int8_scales.range(g, 1);
+                    return 0;
+                }
+                if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+                {
+                    return 0;
+                }
+                if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+                {
+                    return 0;
                 }
-
-                op->load_model(ModelBinFromMatArray(weights));
             }
-
-            op->create_pipeline(opt);
-
-            group_ops[g] = op;
         }
     }
 
-    return 0;
-}
-
-int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt)
-{
-    if (activation)
-    {
-        activation->destroy_pipeline(opt);
-        delete activation;
-        activation = 0;
-    }
-
+    // group convolution
     for (int i=0; i<(int)group_ops.size(); i++)
-    {
-        group_ops[i]->destroy_pipeline(opt);
         delete group_ops[i];
-    }
-    group_ops.clear();
-
-    return 0;
-}
-
-int ConvolutionDepthWise_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
-{
-    int w = bottom_blob.w;
-    int h = bottom_blob.h;
-    int channels = bottom_blob.c;
-    size_t elemsize = bottom_blob.elemsize;
-//     int elempack = bottom_blob.elempack;
-
-    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
-    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
-
-    Mat bottom_blob_unbordered = bottom_blob;
-    if (elemsize != 1)
-    {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
-            return -100;
-
-        const int channels_g = channels / group;
 
-        // quantize, scale and round to nearest
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int g=0; g<group; g++)
-        {
-            Option opt_g = opt;
-            opt_g.num_threads = 1;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
-
-            const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g);
-            Mat bottom_blob_int8_g = bottom_blob_int8.channel_range(channels_g * g, channels_g);
-            quantize_ops[g]->forward(bottom_blob_g, bottom_blob_int8_g, opt_g);
-        }
-
-        bottom_blob_unbordered = bottom_blob_int8;
-    }
-
-    Mat bottom_blob_bordered = bottom_blob_unbordered;
-    if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
-    {
-        Option opt_b = opt;
-        opt_b.blob_allocator = opt.workspace_allocator;
-        copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
-    }
-    else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
-    {
-        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
-        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
-        if (wpad > 0 || hpad > 0)
-        {
-            Option opt_b = opt;
-            opt_b.blob_allocator = opt.workspace_allocator;
-            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-        }
-    }
-    else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
-    {
-        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
-        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
-        if (wpad > 0 || hpad > 0)
-        {
-            Option opt_b = opt;
-            opt_b.blob_allocator = opt.workspace_allocator;
-            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-        }
-    }
-    if (bottom_blob_bordered.empty())
-        return -100;
+    group_ops.clear();
 
-    w = bottom_blob_bordered.w;
-    h = bottom_blob_bordered.h;
+    const int channels_g = channels / group;
+    const int num_output_g = num_output / group;
 
-    int outw = (w - kernel_extent_w) / stride_w + 1;
-    int outh = (h - kernel_extent_h) / stride_h + 1;
-//     int out_elempack = num_output % 4 == 0 ? 4 : 1;
-//     size_t out_elemsize = elemsize / elempack * out_elempack;
+    group_ops.resize(group);
 
-    // int8
-    if (use_int8_requantize)
+    for (int g=0; g<group; g++)
     {
-        Mat top_blob_tm;
-        top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
-        if (top_blob_tm.empty())
-            return -100;
+        Mat weight_data_g = weight_data.range(maxk * channels_g * num_output_g * g, maxk * channels_g * num_output_g);
+        Mat bias_data_g;
+        if (bias_term)
+            bias_data_g = bias_data.range(num_output_g * g, num_output_g);
 
-        top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-        if (top_blob.empty())
-            return -100;
+        ncnn::Layer* op = ncnn::create_layer(ncnn::LayerType::Convolution);
 
-        // depth-wise
-        if (channels == group && group == num_output)
+        // set param
+        ncnn::ParamDict pd;
+        pd.set(0, num_output_g);// num_output
+        pd.set(1, kernel_w);
+        pd.set(11, kernel_h);
+        pd.set(2, dilation_w);
+        pd.set(12, dilation_h);
+        pd.set(3, stride_w);
+        pd.set(13, stride_h);
+        pd.set(4, 0);// pad_w
+        pd.set(14, 0);// pad_h
+        pd.set(5, bias_term);
+        pd.set(6, maxk * channels_g * num_output_g);// weight_data_size
+        pd.set(8, int8_scale_term);
+        pd.set(9, activation_type);
+        pd.set(10, activation_params);
+
+        op->load_param(pd);
+
+        // set weights
+        if (bias_term)
         {
-            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
-            {
-                if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2))
-                {
-                    if (stride_w == 1 && stride_h == 1)
-                    {
-                        convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
-                    }
-                    else if (stride_w == 2 && stride_h == 2)
-                    {
-                        convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
-                    }
-
-                    if (activation)
-                    {
-                        activation->forward_inplace(top_blob, opt);
-                    }
+            ncnn::Mat weights[4];
+            weights[0] = weight_data_g;
+            weights[1] = bias_data_g;
 
-                    return 0;
-                }
-            }
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int g=0; g<group; g++)
-            {
-                const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(g, 1);
-                Mat top_blob_tm_g = top_blob_tm.channel_range(g, 1);
-
-                const ncnn::Layer* op = group_ops[g];
-
-                Option opt_g = opt;
-                opt_g.num_threads = 1;
-                opt_g.blob_allocator = top_blob.allocator;
-
-                // forward
-                op->forward(bottom_blob_bordered_g, top_blob_tm_g, opt_g);
-            }
-
-            if (activation)
+            if (int8_scale_term)
             {
-                activation->forward_inplace(top_blob, opt);
+                weights[2] = weight_data_int8_scales.range(g, 1);
+                weights[3] = bottom_blob_int8_scales.range(g, 1);
             }
 
-            return 0;
-        }
-
-        const int channels_g = channels / group;
-        const int num_output_g = num_output / group;
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int g=0; g<group; g++)
-        {
-            const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
-            Mat top_blob_tm_g = top_blob_tm.channel_range(num_output_g * g, num_output_g);
-
-            const ncnn::Layer* op = group_ops[g];
-
-            Option opt_g = opt;
-            opt_g.blob_allocator = top_blob.allocator;
-
-            // forward
-            op->forward(bottom_blob_bordered_g, top_blob_tm_g, opt_g);
+            op->load_model(ModelBinFromMatArray(weights));
         }
-    }
-    else
-    {
-        top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-        if (top_blob.empty())
-            return -100;
-
-        // depth-wise
-        if (channels == group && group == num_output)
+        else
         {
-            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
-            {
-                if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2))
-                {
-                    if (stride_w == 1 && stride_h == 1)
-                    {
-                        convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt);
-                    }
-                    else if (stride_w == 2 && stride_h == 2)
-                    {
-                        convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt);
-                    }
+            ncnn::Mat weights[3];
+            weights[0] = weight_data_g;
 
-                    // dequantize, reverse scale inplace
-                    #pragma omp parallel for num_threads(opt.num_threads)
-                    for (int g=0; g<group; g++)
-                    {
-                        Option opt_g = opt;
-                        opt_g.num_threads = 1;
-                        opt_g.blob_allocator = top_blob.allocator;
-
-                        Mat top_blob_g = top_blob.channel(g);
-                        dequantize_ops[g]->forward_inplace(top_blob_g, opt_g);
-                    }
-
-                    if (activation)
-                    {
-                        activation->forward_inplace(top_blob, opt);
-                    }
-
-                    return 0;
-                }
-            }
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int g=0; g<group; g++)
-            {
-                const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(g, 1);
-                Mat top_blob_g = top_blob.channel_range(g, 1);
-
-                const ncnn::Layer* op = group_ops[g];
-
-                Option opt_g = opt;
-                opt_g.num_threads = 1;
-                opt_g.blob_allocator = top_blob.allocator;
-
-                // forward
-                op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-            }
-
-            if (activation)
+            if (int8_scale_term)
             {
-                activation->forward_inplace(top_blob, opt);
+                weights[1] = weight_data_int8_scales.range(g, 1);
+                weights[2] = bottom_blob_int8_scales.range(g, 1);
             }
 
-            return 0;
+            op->load_model(ModelBinFromMatArray(weights));
         }
 
-        const int channels_g = channels / group;
-        const int num_output_g = num_output / group;
+        op->create_pipeline(opt);
 
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int g=0; g<group; g++)
-        {
-            const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
-            Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
+//         op->use_int8_requantize = use_int8_requantize; FIXME
 
-            const ncnn::Layer* op = group_ops[g];
+        group_ops[g] = op;
+    }
 
-            Option opt_g = opt;
-            opt_g.blob_allocator = top_blob.allocator;
+    return 0;
+}
 
-            // forward
-            op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-        }
+int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt)
+{
+    if (activation)
+    {
+        activation->destroy_pipeline(opt);
+        delete activation;
+        activation = 0;
     }
 
-    if (activation)
+    for (int i=0; i<(int)group_ops.size(); i++)
     {
-        activation->forward_inplace(top_blob, opt);
+        group_ops[i]->destroy_pipeline(opt);
+        delete group_ops[i];
     }
+    group_ops.clear();
 
     return 0;
 }
@@ -462,9 +246,9 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
     // convolv with NxN kernel
     // value = value + bias
 
-    if (use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
-        return forward_int8(bottom_blob, top_blob, opt);
+        return forward_int8_arm(bottom_blob, top_blob, opt);
     }
 
     int w = bottom_blob.w;
@@ -524,31 +308,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
     // depth-wise
     if (channels * elempack == group && group == num_output)
     {
-        const int maxk = kernel_w * kernel_h;
-
-        // kernel offsets
-        std::vector<int> _space_ofs(maxk);
-        int* space_ofs = &_space_ofs[0];
-        {
-            int p1 = 0;
-            int p2 = 0;
-            int gap = w * dilation_h - kernel_w * dilation_w;
-            for (int i = 0; i < kernel_h; i++)
-            {
-                for (int j = 0; j < kernel_w; j++)
-                {
-                    space_ofs[p1] = p2;
-                    p1++;
-                    p2 += dilation_w;
-                }
-                p2 += gap;
-            }
-        }
-
 #if __ARM_NEON
         if (elempack == 4)
         {
-            if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
                 convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -556,8 +319,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
                 convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -565,8 +330,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
                 convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -574,8 +341,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
                 convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt);
 
@@ -583,9 +352,32 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
             else
             {
+                const int maxk = kernel_w * kernel_h;
+
+                // kernel offsets
+                std::vector<int> _space_ofs(maxk);
+                int* space_ofs = &_space_ofs[0];
+                {
+                    int p1 = 0;
+                    int p2 = 0;
+                    int gap = w * dilation_h - kernel_w * dilation_w;
+                    for (int i = 0; i < kernel_h; i++)
+                    {
+                        for (int j = 0; j < kernel_w; j++)
+                        {
+                            space_ofs[p1] = p2;
+                            p1++;
+                            p2 += dilation_w;
+                        }
+                        p2 += gap;
+                    }
+                }
+
                 #pragma omp parallel for num_threads(opt.num_threads)
                 for (int g=0; g<channels; g++)
                 {
@@ -621,13 +413,15 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                         outptr += outw * 4;
                     }
                 }
+
+                return 0;
             }
         }
 #endif // __ARM_NEON
 
         if (elempack == 1)
         {
-            if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
                 convdw3x3s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -635,8 +429,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
                 convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -644,8 +440,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
             {
                 convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -653,8 +451,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
+            else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
                 convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
@@ -662,98 +462,278 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con
                 {
                     activation->forward_inplace(top_blob, opt);
                 }
+
+                return 0;
             }
-            else
-            {
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<channels; g++)
-                {
-                    float* outptr = top_blob.channel(g);
-                    const float* kptr = (const float*)weight_data + maxk * g;
-                    const Mat m = bottom_blob_bordered.channel(g);
+        }
+    }
 
-                    for (int i = 0; i < outh; i++)
-                    {
-                        for (int j = 0; j < outw; j++)
-                        {
-                            float sum = 0.f;
+    // group convolution
+    const int channels_g = channels * elempack / group;
+    const int num_output_g = num_output / group;
 
-                            if (bias_term)
-                            {
-                                sum = bias_data[g];
-                            }
+    int g_elempack = (opt.use_packing_layout && channels_g % 4 == 0) ? 4 : 1;
+    int out_g_elempack = (opt.use_packing_layout && num_output_g % 4 == 0) ? 4 : 1;
 
-                            const float* sptr = m.row(i*stride_h) + j*stride_w;
+    // unpacking
+    Mat bottom_blob_bordered_unpacked = bottom_blob_bordered;
+    if (elempack == 4 && g_elempack == 1)
+    {
+        Option opt_p = opt;
+        opt_p.blob_allocator = opt.workspace_allocator;
+        convert_packing(bottom_blob_bordered, bottom_blob_bordered_unpacked, 1, opt_p);
+    }
 
-                            for (int k = 0; k < maxk; k++)
-                            {
-                                float val = sptr[ space_ofs[k] ];
-                                float w = kptr[ k ];
-                                sum += val * w;
-                            }
+    Mat top_blob_unpacked = top_blob;
+    if (out_g_elempack == 1 && out_elempack == 4)
+    {
+        top_blob_unpacked.create(outw, outh, num_output, out_elemsize / out_elempack, 1, opt.workspace_allocator);
+        if (top_blob_unpacked.empty())
+            return -100;
+    }
 
-                            sum = activation_ss(sum, activation_type, activation_params);
+    for (int g=0; g<group; g++)
+    {
+        const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack);
+        Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack);
 
-                            outptr[j] = sum;
-                        }
+        const ncnn::Layer* op = group_ops[g];
 
-                        outptr += outw;
-                    }
-                }
-            }
-        }
+        Option opt_g = opt;
+        opt_g.blob_allocator = top_blob_unpacked.allocator;
+
+        // forward
+        op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
+    }
+
+    // packing
+    if (out_g_elempack == 1 && out_elempack == 4)
+    {
+        convert_packing(top_blob_unpacked, top_blob, 4, opt);
     }
     else
     {
-        // group convolution
-        const int channels_g = channels * elempack / group;
-        const int num_output_g = num_output / group;
+        top_blob = top_blob_unpacked;
+    }
 
-        int g_elempack = (opt.use_packing_layout && channels_g % 4 == 0) ? 4 : 1;
-        int out_g_elempack = (opt.use_packing_layout && num_output_g % 4 == 0) ? 4 : 1;
+    return 0;
+}
 
-        // unpacking
-        Mat bottom_blob_bordered_unpacked = bottom_blob_bordered;
-        if (elempack == 4 && g_elempack == 1)
+int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
+    Mat bottom_blob_unbordered = bottom_blob;
+    if (elemsize != 1)
+    {
+        bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
+        if (bottom_blob_unbordered.empty())
+            return -100;
+
+        const int channels_g = channels / group;
+
+        // quantize, scale and round to nearest
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int g=0; g<group; g++)
         {
-            Option opt_p = opt;
-            opt_p.blob_allocator = opt.workspace_allocator;
-            convert_packing(bottom_blob_bordered, bottom_blob_bordered_unpacked, 1, opt_p);
+            Option opt_g = opt;
+            opt_g.num_threads = 1;
+            opt_g.blob_allocator = bottom_blob_unbordered.allocator;
+
+            const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g);
+            Mat bottom_blob_int8_g = bottom_blob_unbordered.channel_range(channels_g * g, channels_g);
+            quantize_float32_to_int8(bottom_blob_g, bottom_blob_int8_g, bottom_blob_int8_scales[g], opt_g);
         }
+    }
 
-        Mat top_blob_unpacked = top_blob;
-        if (out_g_elempack == 1 && out_elempack == 4)
+    Mat bottom_blob_bordered = bottom_blob_unbordered;
+    if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
+    {
+        Option opt_b = opt;
+        opt_b.blob_allocator = opt.workspace_allocator;
+        copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
+    }
+    else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
+    {
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
         {
-            top_blob_unpacked.create(outw, outh, num_output, out_elemsize / out_elempack, 1, opt.workspace_allocator);
-            if (top_blob_unpacked.empty())
-                return -100;
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
         }
-
-        for (int g=0; g<group; g++)
+    }
+    else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
+    {
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
         {
-            const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack);
-            Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack);
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    if (bottom_blob_bordered.empty())
+        return -100;
 
-            const ncnn::Layer* op = group_ops[g];
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
 
-            Option opt_g = opt;
-            opt_g.blob_allocator = top_blob_unpacked.allocator;
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
 
-            // forward
-            op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-        }
+    // int8
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
 
-        // packing
-        if (out_g_elempack == 1 && out_elempack == 4)
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // depth-wise
+    if (channels == group && group == num_output)
+    {
+        if (use_int8_requantize)
         {
-            convert_packing(top_blob_unpacked, top_blob, 4, opt);
+            std::vector<float> requantize_scales;
+            for (int g=0; g<group; g++)
+            {
+                float scale_in;
+                if (weight_data_int8_scales[g] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+
+                float scale_out = top_blob_int8_scale;
+
+                requantize_scales.push_back(scale_in);
+                requantize_scales.push_back(scale_out);
+            }
+
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+            {
+                convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
+
+                return 0;
+            }
+            else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+            {
+                convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
+
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
+
+                return 0;
+            }
         }
         else
         {
-            top_blob = top_blob_unpacked;
+//             std::vector<float> dequantize_scales;
+//             for (int g=0; g<group; g++)
+//             {
+//                 float top_rescale = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+//
+//                 dequantize_scales.push_back(top_rescale);
+//             }
+
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+            {
+                convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt);
+//                 convdw3x3s1_int8_dequant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
+
+                // dequantize, reverse scale inplace
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int g=0; g<group; g++)
+                {
+                    // dequantize and relu
+                    float scale_in;
+                    if (weight_data_int8_scales[g] == 0)
+                        scale_in = 0;
+                    else
+                        scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+
+                    Option opt_g = opt;
+                    opt_g.num_threads = 1;
+                    opt_g.blob_allocator = top_blob.allocator;
+
+                    Mat top_blob_g = top_blob.channel(g);
+                    dequantize_int32_to_float32(top_blob_g, scale_in, &bias_data[g], bias_term ? 0 : 1, opt_g);
+                }
+
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
+
+                return 0;
+            }
+            else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+            {
+                convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt);
+//                 convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
+
+                // dequantize, reverse scale inplace
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int g=0; g<group; g++)
+                {
+                    // dequantize and relu
+                    float scale_in;
+                    if (weight_data_int8_scales[g] == 0)
+                        scale_in = 0;
+                    else
+                        scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+
+                    Option opt_g = opt;
+                    opt_g.num_threads = 1;
+                    opt_g.blob_allocator = top_blob.allocator;
+
+                    Mat top_blob_g = top_blob.channel(g);
+                    dequantize_int32_to_float32(top_blob_g, scale_in, &bias_data[g], bias_term ? 0 : 1, opt_g);
+                }
+
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
+
+                return 0;
+            }
         }
     }
 
+    // group convolution
+    const int channels_g = channels / group;
+    const int num_output_g = num_output / group;
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int g=0; g<group; g++)
+    {
+        const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
+        Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
+
+        const ncnn::Layer* op = group_ops[g];
+
+        Option opt_g = opt;
+        opt_g.blob_allocator = top_blob.allocator;
+
+        // forward
+        op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
+    }
+
     return 0;
 }
 

src/layer/arm/convolutiondepthwise_arm.h

@@ -27,9 +27,11 @@ public:
     virtual int create_pipeline(const Option& opt);
     virtual int destroy_pipeline(const Option& opt);
 
-    virtual int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+protected:
+    int forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
 public:
     Layer* activation;
     std::vector<ncnn::Layer*> group_ops;

src/layer/arm/innerproduct_arm.cpp

@@ -30,7 +30,6 @@ InnerProduct_arm::InnerProduct_arm()
 {
 #if __ARM_NEON
     support_packing = true;
-    use_fp32_packing_inference = false;
 #endif // __ARM_NEON
 
     flatten = 0;
@@ -39,18 +38,7 @@ InnerProduct_arm::InnerProduct_arm()
 int InnerProduct_arm::create_pipeline(const Option& opt)
 {
 #if __ARM_NEON
-    bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u);
-
-    use_fp32_packing_inference = opt.use_packing_layout && weight_data_is_float32 && !use_int8_inference;
-
-    if (use_int8_inference)
-    {
-        support_packing = false;
-    }
-
-    if (use_fp32_packing_inference)
-    {
-
+    if (opt.use_packing_layout)
     {
         flatten = ncnn::create_layer(ncnn::LayerType::Flatten);
 
@@ -60,8 +48,6 @@ int InnerProduct_arm::create_pipeline(const Option& opt)
 
         flatten->create_pipeline(opt);
     }
-
-    } // opt.use_packing_layout
 #endif // __ARM_NEON
 
     return 0;
@@ -81,7 +67,7 @@ int InnerProduct_arm::destroy_pipeline(const Option& opt)
 
 int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
-    if (use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
         // TODO
         return InnerProduct::forward(bottom_blob, top_blob, opt);
@@ -95,34 +81,27 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
     int size = w * h;
 
 #if __ARM_NEON
-    if (use_fp32_packing_inference)
-    {
-
     if (elempack == 4)
     {
+        // flatten
+        Mat bottom_blob_flattened = bottom_blob;
+        if (bottom_blob.dims != 1)
+        {
+            Option opt_flatten = opt;
+            opt_flatten.blob_allocator = opt.workspace_allocator;
 
-    // flatten
-    Mat bottom_blob_flattened = bottom_blob;
-    if (bottom_blob.dims != 1)
-    {
-        Option opt_flatten = opt;
-        opt_flatten.blob_allocator = opt.workspace_allocator;
-
-        flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);
-    }
-
-    // pack1
-    {
-        bottom_blob_flattened.w *= bottom_blob_flattened.elempack;
-        bottom_blob_flattened.elemsize = 4u;
-        bottom_blob_flattened.elempack = 1;
-    }
+            flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten);
+        }
 
-    return forward(bottom_blob_flattened, top_blob, opt);
+        // pack1
+        {
+            bottom_blob_flattened.w *= bottom_blob_flattened.elempack;
+            bottom_blob_flattened.elemsize = 4u;
+            bottom_blob_flattened.elempack = 1;
+        }
 
+        return forward(bottom_blob_flattened, top_blob, opt);
     }
-
-    } // opt.use_packing_layout
 #endif // __ARM_NEON
 
     top_blob.create(num_output, elemsize, opt.blob_allocator);

src/layer/arm/innerproduct_arm.h

@@ -30,8 +30,6 @@ public:
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
-    bool use_fp32_packing_inference;
-
     ncnn::Layer* flatten;
 };
 

src/layer/convolution.cpp

@@ -24,9 +24,8 @@ Convolution::Convolution()
 {
     one_blob_only = true;
     support_inplace = false;
-    use_int8_requantize = false;
 
-    quantize = 0;
+    use_int8_requantize = false;
 }
 
 int Convolution::load_param(const ParamDict& pd)
@@ -77,170 +76,28 @@ int Convolution::load_model(const ModelBin& mb)
 
 int Convolution::create_pipeline(const Option& opt)
 {
-    bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u);
-    bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u);
-
-    if (weight_data_is_int8 && !opt.use_int8_inference)
-    {
-        fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n");
-        return -1;
-    }
-
-    use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term));
-
     // runtime quantize the weight data
-    if (weight_data_is_float32 && use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term)
     {
-        // quantize weight to int8
         Mat int8_weight_data(weight_data_size, (size_t)1u);
         if (int8_weight_data.empty())
             return -100;
 
         const int weight_data_size_output = weight_data_size / num_output;
 
-        for (int n=0; n<num_output; n++)
+        for (int p=0; p<num_output; p++)
         {
-            Layer* op = ncnn::create_layer(ncnn::LayerType::Quantize);
-
-            ncnn::ParamDict pd;
-            pd.set(0, weight_data_int8_scales[n]);// scale
-
-            op->load_param(pd);
-
-            op->create_pipeline(opt);
-
             Option opt_q = opt;
             opt_q.blob_allocator = int8_weight_data.allocator;
 
-            const Mat weight_data_n = weight_data.range(weight_data_size_output * n, weight_data_size_output);
-            Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * n, weight_data_size_output);
-            op->forward(weight_data_n, int8_weight_data_n, opt_q);
-
-            delete op;
+            const Mat weight_data_n = weight_data.range(weight_data_size_output * p, weight_data_size_output);
+            Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * p, weight_data_size_output);
+            quantize_float32_to_int8(weight_data_n, int8_weight_data_n, weight_data_int8_scales[p], opt_q);
         }
 
         weight_data = int8_weight_data;
     }
 
-    // initial the quantize,dequantize op layer
-    if (use_int8_inference)
-    {
-        quantize = ncnn::create_layer(ncnn::LayerType::Quantize);
-        {
-            ncnn::ParamDict pd;
-            pd.set(0, bottom_blob_int8_scale);// scale
-
-            quantize->load_param(pd);
-
-            quantize->create_pipeline(opt);
-        }
-
-        dequantize_ops.resize(num_output);
-        for (int n=0; n<num_output; n++)
-        {
-            dequantize_ops[n] = ncnn::create_layer(ncnn::LayerType::Dequantize);
-
-            float top_rescale = 1.f;
-
-            if (weight_data_int8_scales[n] == 0)
-                top_rescale = 0;
-            else
-                top_rescale = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[n]);
-
-            ncnn::ParamDict pd;
-            pd.set(0, top_rescale);// scale
-            pd.set(1, bias_term);  // bias_term
-            pd.set(2, 1);          // bias_data_size
-
-            dequantize_ops[n]->load_param(pd);
-
-            dequantize_ops[n]->create_pipeline(opt);
-
-            ncnn::Mat weights[1];
-            weights[0] = bias_data.range(n, 1);
-
-            dequantize_ops[n]->load_model(ModelBinFromMatArray(weights));
-
-            dequantize_scales.push_back(top_rescale);
-        }
-    }
-
-    return 0;
-}
-
-int Convolution::destroy_pipeline(const Option& opt)
-{
-    if (quantize)
-    {
-        quantize->destroy_pipeline(opt);
-        delete quantize;
-        quantize = 0;
-    }
-
-    for (int i=0; i<(int)dequantize_ops.size(); i++)
-    {
-        dequantize_ops[i]->destroy_pipeline(opt);
-        delete dequantize_ops[i];
-    }
-    dequantize_ops.clear();
-
-    for (int i=0; i<(int)requantize_ops.size(); i++)
-    {
-        requantize_ops[i]->destroy_pipeline(opt);
-        delete requantize_ops[i];
-    }
-    requantize_ops.clear();
-
-    dequantize_scales.clear();
-    requantize_scales.clear();
-
-    return 0;
-}
-
-int Convolution::create_requantize_op(void)
-{
-    if (!use_int8_requantize)
-    {
-        fprintf(stderr, "requantized op set but use_int8_requantize disabled\n");
-        return -1;
-    }
-
-    requantize_ops.resize(num_output);
-    for (int n=0; n<num_output; n++)
-    {
-        requantize_ops[n] = ncnn::create_layer(ncnn::LayerType::Requantize);
-
-        float scale_in = 1.f;
-        float scale_out = 1.f;
-
-        if (weight_data_int8_scales[n] == 0)
-        {
-            scale_in = 0;
-        }
-        else
-        {
-            scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[n]);
-        }
-
-        scale_out = top_blob_int8_scale;
-
-        ncnn::ParamDict pd;
-        pd.set(0, scale_in);   // scale in
-        pd.set(1, scale_out);  // scale_out
-        pd.set(2, bias_term);  // bias_term
-        pd.set(3, 1);          // bias_data_size
-
-        requantize_ops[n]->load_param(pd);
-
-        ncnn::Mat weights[1];
-        weights[0] = bias_data.range(n, 1);
-
-        requantize_ops[n]->load_model(ModelBinFromMatArray(weights));
-
-        requantize_scales.push_back(scale_in);
-        requantize_scales.push_back(scale_out);
-    }
-
     return 0;
 }
 
@@ -249,6 +106,11 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
     // convolv with NxN kernel
     // value = value + bias
 
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return forward_int8(bottom_blob, top_blob, opt);
+    }
+
     // flattened blob, implement as InnerProduct
     if (bottom_blob.dims == 1 && kernel_w == 1 && kernel_h == 1)
     {
@@ -301,31 +163,12 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
     const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
 
-    Mat bottom_blob_unbordered = bottom_blob;
-    if (use_int8_inference && elemsize != 1)
-    {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
-            return -100;
-
-        // quantize, scale and round to nearest
-        {
-            Option opt_g = opt;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
-
-            quantize->forward(bottom_blob, bottom_blob_int8, opt_g);
-        }
-
-        bottom_blob_unbordered = bottom_blob_int8;
-    }
-
-    Mat bottom_blob_bordered = bottom_blob_unbordered;
+    Mat bottom_blob_bordered = bottom_blob;
     if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
     {
         Option opt_b = opt;
         opt_b.blob_allocator = opt.workspace_allocator;
-        copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
+        copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
     }
     else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
@@ -336,7 +179,7 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
-            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
         }
     }
     else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
@@ -348,7 +191,7 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
-            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
         }
     }
     if (bottom_blob_bordered.empty())
@@ -381,148 +224,6 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
         }
     }
 
-    // int8
-    if (use_int8_inference)
-    {
-        if (use_int8_requantize == true)
-        {
-            Mat top_blob_tm;
-            top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
-            if (top_blob_tm.empty())
-                return -100;
-            
-            top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100; 
-
-            // num_output
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int p=0; p<num_output; p++)
-            {
-                int* outptr = top_blob_tm.channel(p);
-
-                for (int i = 0; i < outh; i++)
-                {
-                    for (int j = 0; j < outw; j++)
-                    {
-                        int sum = 0;
-
-                        const signed char* kptr = (const signed char*)weight_data + maxk * channels * p;
-
-                        // channels
-                        for (int q=0; q<channels; q++)
-                        {
-                            const Mat m = bottom_blob_bordered.channel(q);
-                            const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
-
-                            for (int k = 0; k < maxk; k++)
-                            {
-                                int val = sptr[ space_ofs[k] ];
-                                int w = kptr[k];
-                                sum += val * w;
-                            }
-
-                            kptr += maxk;
-                        }
-
-                        outptr[j] = sum;
-                    }
-
-                    outptr += outw;
-                }
-
-                // requantize, reverse scale inplace
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
-
-                    Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1);
-                    Mat top_blob_g = top_blob.channel_range(p, 1);
-                    requantize_ops[p]->forward(top_blob_tm_g, top_blob_g, opt_g);
-                }       
-
-                // activation relu
-                if (activation_type == 1)
-                {
-                    signed char* outptr_s8 = top_blob.channel(p);
-
-                    for (int i = 0; i < outh*outw; i++)
-                    {
-                        if (outptr_s8[i] < 0)
-                            outptr_s8[i] = 0;
-                    }
-                }                                 
-            }
-        }
-        else
-        {
-            top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100;
-      
-            // num_output
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int p=0; p<num_output; p++)
-            {
-                int* outptr = top_blob.channel(p);
-
-                for (int i = 0; i < outh; i++)
-                {
-                    for (int j = 0; j < outw; j++)
-                    {
-                        int sum = 0;
-
-                        const signed char* kptr = (const signed char*)weight_data + maxk * channels * p;
-
-                        // channels
-                        for (int q=0; q<channels; q++)
-                        {
-                            const Mat m = bottom_blob_bordered.channel(q);
-                            const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
-
-                            for (int k = 0; k < maxk; k++)
-                            {
-                                int val = sptr[ space_ofs[k] ];
-                                int w = kptr[k];
-                                sum += val * w;
-                            }
-
-                            kptr += maxk;
-                        }
-
-                        outptr[j] = sum;
-                    }
-
-                    outptr += outw;
-                }
-
-                // dequantize, reverse scale inplace
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
-
-                    Mat top_blob_g = top_blob.channel_range(p, 1);
-                    dequantize_ops[p]->forward_inplace(top_blob_g, opt_g);
-                }
-
-                // activation relu
-                if (activation_type == 1)
-                {
-                    float* outptr_fp32 = top_blob.channel(p);
-
-                    for (int i = 0; i < outh*outw; i++)
-                    {
-                        outptr_fp32[i] = std::max(outptr_fp32[i], 0.f);
-                    }
-                }
-            }   
-        }        
-
-        return 0;
-    }
-
     // float32
     top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
     if (top_blob.empty())
@@ -594,4 +295,186 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op
     return 0;
 }
 
+static inline signed char float2int8(float v)
+{
+    int int32 = static_cast<int>(round(v));
+    if (int32 > 127) return 127;
+    if (int32 < -127) return -127;
+    return (signed char)int32;
+}
+
+int Convolution::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+
+//     fprintf(stderr, "Convolution input %d x %d  ksize=%d %d  stride=%d %d\n", w, h, kernel_w, kernel_h, stride_w, stride_h);
+
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
+    Mat bottom_blob_unbordered = bottom_blob;
+    if (elemsize != 1)
+    {
+        Option opt_g = opt;
+        opt_g.blob_allocator = opt.workspace_allocator;
+
+        quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g);
+    }
+
+    Mat bottom_blob_bordered = bottom_blob_unbordered;
+    if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
+    {
+        Option opt_b = opt;
+        opt_b.blob_allocator = opt.workspace_allocator;
+        copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
+    }
+    else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
+    {
+        // tensorflow padding=SAME or onnx padding=SAME_UPPER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
+    {
+        // onnx padding=SAME_LOWER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    if (bottom_blob_bordered.empty())
+        return -100;
+
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
+
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
+
+    const int maxk = kernel_w * kernel_h;
+
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
+        }
+    }
+
+    // int8
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
+
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p=0; p<num_output; p++)
+    {
+        signed char* outptr = top_blob.channel(p);
+
+        for (int i = 0; i < outh; i++)
+        {
+            for (int j = 0; j < outw; j++)
+            {
+                int sum = 0;
+
+                const signed char* kptr = (const signed char*)weight_data + maxk * channels * p;
+
+                // channels
+                for (int q=0; q<channels; q++)
+                {
+                    const Mat m = bottom_blob_bordered.channel(q);
+                    const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
+
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        int val = sptr[ space_ofs[k] ];
+                        int w = kptr[k];
+                        sum += val * w;
+                    }
+
+                    kptr += maxk;
+                }
+
+                if (use_int8_requantize)
+                {
+                    // requantize and relu
+                    float scale_in;
+                    if (weight_data_int8_scales[p] == 0)
+                        scale_in = 0;
+                    else
+                        scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                    float sumfp32 = sum * scale_in;
+
+                    if (bias_term)
+                        sumfp32 += bias_data[p];
+
+                    float scale_out = top_blob_int8_scale;//FIXME load param
+
+                    signed char sums8 = float2int8(sumfp32 * scale_out);
+
+                    if (activation_type == 1)
+                    {
+                        sums8 = std::max(sums8, (signed char)0);
+                    }
+
+                    outptr[0] = sums8;
+                    outptr += 1;
+                }
+                else
+                {
+                    // dequantize and relu
+                    float scale_in;
+                    if (weight_data_int8_scales[p] == 0)
+                        scale_in = 0;
+                    else
+                        scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                    float sumfp32 = sum * scale_in;
+
+                    if (bias_term)
+                        sumfp32 += bias_data[p];
+
+                    if (activation_type == 1)
+                    {
+                        sumfp32 = std::max(sumfp32, 0.f);
+                    }
+
+                    ((float*)outptr)[0] = sumfp32;
+                    outptr += 4;
+                }
+            }
+        }
+    }
+
+    return 0;
+}
+
 } // namespace ncnn

src/layer/convolution.h

@@ -29,12 +29,12 @@ public:
     virtual int load_model(const ModelBin& mb);
 
     virtual int create_pipeline(const Option& opt);
-    virtual int destroy_pipeline(const Option& opt);
-
-    virtual int create_requantize_op(void);
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+protected:
+    int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
 public:
     // param
     int num_output;
@@ -65,19 +65,10 @@ public:
 
     Mat weight_data_int8_scales;
     float bottom_blob_int8_scale;
-    float top_blob_int8_scale;
+    float top_blob_int8_scale;// TODO load param
 
-    bool use_int8_inference;
     bool use_int8_requantize;
 
-    ncnn::Layer* quantize;
-    std::vector<ncnn::Layer*> dequantize_ops;
-    std::vector<ncnn::Layer*> requantize_ops;
-
-    // merge de/requantize op into convolution op
-    std::vector<float> dequantize_scales;
-    std::vector<float> requantize_scales;    
-
     // implementation type, 0 means do not use auto pack model 
     int impl_type;
 };

src/layer/convolutiondepthwise.cpp

@@ -100,20 +100,9 @@ int ConvolutionDepthWise::load_model(const ModelBin& mb)
 
 int ConvolutionDepthWise::create_pipeline(const Option& opt)
 {
-    bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u);
-    bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u);
-
-    if (weight_data_is_int8 && !opt.use_int8_inference)
+    // runtime quantize the weight data
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term)
     {
-        fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n");
-        return -1;
-    }
-
-    use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term));
-
-    if (weight_data_is_float32 && use_int8_inference)
-    {
-        // quantize weight to int8
         Mat int8_weight_data(weight_data_size, (size_t)1u);
         if (int8_weight_data.empty())
             return -100;
@@ -122,153 +111,259 @@ int ConvolutionDepthWise::create_pipeline(const Option& opt)
 
         for (int g=0; g<group; g++)
         {
-            Layer* op = ncnn::create_layer(ncnn::LayerType::Quantize);
-
-            ncnn::ParamDict pd;
-            pd.set(0, weight_data_int8_scales[g]);// scale
-
-            op->load_param(pd);
-
-            op->create_pipeline(opt);
-
             Option opt_q = opt;
             opt_q.blob_allocator = int8_weight_data.allocator;
 
             const Mat weight_data_g = weight_data.range(weight_data_size_g * g, weight_data_size_g);
             Mat int8_weight_data_g = int8_weight_data.range(weight_data_size_g * g, weight_data_size_g);
-            op->forward(weight_data_g, int8_weight_data_g, opt_q);
-
-            delete op;
+            quantize_float32_to_int8(weight_data_g, int8_weight_data_g, weight_data_int8_scales[g], opt_q);
         }
 
         weight_data = int8_weight_data;
     }
 
-    if (use_int8_inference)
-    {
-        quantize_ops.resize(group);
-        dequantize_ops.resize(group);
-
-        for (int g=0; g<group; g++)
-        {
-            quantize_ops[g] = ncnn::create_layer(ncnn::LayerType::Quantize);
+    return 0;
+}
 
-            ncnn::ParamDict pd;
-            pd.set(0, bottom_blob_int8_scales[g]);// scale
+int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    // convolv with NxN kernel
+    // value = value + bias
 
-            quantize_ops[g]->load_param(pd);
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return forward_int8(bottom_blob, top_blob, opt);
+    }
 
-            quantize_ops[g]->create_pipeline(opt);
-        }
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
 
-        for (int g=0; g<group; g++)
-        {
-            dequantize_ops[g] = ncnn::create_layer(ncnn::LayerType::Dequantize);
+    if (channels % group != 0 || num_output % group != 0)
+    {
+        // reject invalid group
+        return -100;
+    }
 
-            float top_rescale = 1.f;
-            if (weight_data_int8_scales[g] == 0)
-                top_rescale = 0;
-            else
-                top_rescale = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+//     fprintf(stderr, "ConvolutionDepthWise input %d x %d  pad = %d %d  ksize=%d %d  stride=%d %d\n", w, h, pad_w, pad_h, kernel_w, kernel_h, stride_w, stride_h);
 
-            ncnn::ParamDict pd;
-            pd.set(0, top_rescale);// scale
-            pd.set(1, bias_term);// bias_term
-            pd.set(2, 1);// bias_data_size
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
 
-            dequantize_ops[g]->load_param(pd);
+    Mat bottom_blob_bordered = bottom_blob;
+    if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
+    {
+        Option opt_b = opt;
+        opt_b.blob_allocator = opt.workspace_allocator;
+        copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
+    }
+    else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
+    {
+        // tensorflow padding=SAME or onnx padding=SAME_UPPER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
+    {
+        // onnx padding=SAME_LOWER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    if (bottom_blob_bordered.empty())
+        return -100;
 
-            ncnn::Mat weights[1];
-            weights[0] = bias_data.range(g, 1);
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
 
-            dequantize_ops[g]->load_model(ModelBinFromMatArray(weights));
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
 
-            dequantize_ops[g]->create_pipeline(opt);
+    const int maxk = kernel_w * kernel_h;
 
-            dequantize_scales.push_back(top_rescale);
+    // kernel offsets
+    std::vector<int> _space_ofs(maxk);
+    int* space_ofs = &_space_ofs[0];
+    {
+        int p1 = 0;
+        int p2 = 0;
+        int gap = w * dilation_h - kernel_w * dilation_w;
+        for (int i = 0; i < kernel_h; i++)
+        {
+            for (int j = 0; j < kernel_w; j++)
+            {
+                space_ofs[p1] = p2;
+                p1++;
+                p2 += dilation_w;
+            }
+            p2 += gap;
         }
     }
 
-    return 0;
-}
-
-int ConvolutionDepthWise::destroy_pipeline(const Option& opt)
-{
-    for (int i=0; i<(int)quantize_ops.size(); i++)
-    {
-        quantize_ops[i]->destroy_pipeline(opt);
-        delete quantize_ops[i];
-    }
-    quantize_ops.clear();
+    // float32
+    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
 
-    for (int i=0; i<(int)dequantize_ops.size(); i++)
+    // depth-wise
+    if (channels == group && group == num_output)
     {
-        dequantize_ops[i]->destroy_pipeline(opt);
-        delete dequantize_ops[i];
-    }
-    dequantize_ops.clear();
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int g=0; g<group; g++)
+        {
+            float* outptr = top_blob.channel(g);
+            const float* kptr = (const float*)weight_data + maxk * g;
+            const Mat m = bottom_blob_bordered.channel(g);
 
-    for (int i=0; i<(int)requantize_ops.size(); i++)
-    {
-        requantize_ops[i]->destroy_pipeline(opt);
-        delete requantize_ops[i];
-    }
-    requantize_ops.clear();
+            for (int i = 0; i < outh; i++)
+            {
+                for (int j = 0; j < outw; j++)
+                {
+                    float sum = 0.f;
 
-    dequantize_scales.clear();
-    requantize_scales.clear();
+                    if (bias_term)
+                        sum = bias_data[g];
 
-    return 0;
-}
+                    const float* sptr = m.row(i*stride_h) + j*stride_w;
 
-int ConvolutionDepthWise::create_requantize_op(void)
-{
-    if (!use_int8_requantize)
-    {
-        fprintf(stderr, "requantized op set but use_int8_requantize disabled\n");
-        return -1;
-    }
+                    for (int k = 0; k < maxk; k++)
+                    {
+                        float val = sptr[ space_ofs[k] ];
+                        float w = kptr[k];
+                        sum += val * w;
+                    }
 
-    requantize_ops.resize(group);
-    for (int g=0; g<group; g++)
-    {
-        requantize_ops[g] = ncnn::create_layer(ncnn::LayerType::Requantize);
+                    if (activation_type == 1)
+                    {
+                        sum = std::max(sum, 0.f);
+                    }
+                    else if (activation_type == 2)
+                    {
+                        float slope = activation_params[0];
+                        sum = sum > 0.f ? sum : sum * slope;
+                    }
+                    else if (activation_type == 3)
+                    {
+                        float min = activation_params[0];
+                        float max = activation_params[1];
+                        if (sum < min)
+                            sum = min;
+                        if (sum > max)
+                            sum = max;
+                    }
+                    else if (activation_type == 4)
+                    {
+                        sum = static_cast<float>(1.f / (1.f + exp(-sum)));
+                    }
 
-        float scale_in = 1.f;
-        float scale_out = 1.f;
+                    outptr[j] = sum;
+                }
 
-        if (weight_data_int8_scales[g] == 0)
-        {
-            scale_in = 0;
+                outptr += outw;
+            }
         }
-        else
+    }
+    else
+    {
+        // group convolution
+        const int channels_g = channels / group;
+        const int num_output_g = num_output / group;
+
+#ifdef _WIN32
+        #pragma omp parallel for num_threads(opt.num_threads)
+#else // _WIN32
+        #pragma omp parallel for collapse(2) num_threads(opt.num_threads)
+#endif // _WIN32
+        for (int g=0; g<group; g++)
         {
-            scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
-        }
+            for (int p=0; p<num_output_g; p++)
+            {
+                float* outptr = top_blob.channel(g * num_output_g + p);
+                const float* weight_data_ptr = (const float*)weight_data + maxk * channels_g * num_output_g * g;
+
+                for (int i = 0; i < outh; i++)
+                {
+                    for (int j = 0; j < outw; j++)
+                    {
+                        float sum = 0.f;
+
+                        if (bias_term)
+                            sum = bias_data[num_output_g * g + p];
 
-        scale_out = top_blob_int8_scale;
+                        const float* kptr = weight_data_ptr + maxk * channels_g * p;
 
-        ncnn::ParamDict pd;
-        pd.set(0, scale_in);   // scale in
-        pd.set(1, scale_out);  // scale_out
-        pd.set(2, bias_term);  // bias_term
-        pd.set(3, 1);          // bias_data_size
+                        // channels_g
+                        for (int q=0; q<channels_g; q++)
+                        {
+                            const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
+                            const float* sptr = m.row(i*stride_h) + j*stride_w;
 
-        requantize_ops[g]->load_param(pd);
+                            for (int k = 0; k < maxk; k++)
+                            {
+                                float val = sptr[ space_ofs[k] ];
+                                float w = kptr[k];
+                                sum += val * w;
+                            }
+
+                            kptr += maxk;
+                        }
 
-        ncnn::Mat weights[1];
-        weights[0] = bias_data.range(g, 1);
+                        if (activation_type == 1)
+                        {
+                            sum = std::max(sum, 0.f);
+                        }
+                        else if (activation_type == 2)
+                        {
+                            float slope = activation_params[0];
+                            sum = sum > 0.f ? sum : sum * slope;
+                        }
+                        else if (activation_type == 3)
+                        {
+                            float min = activation_params[0];
+                            float max = activation_params[1];
+                            if (sum < min)
+                                sum = min;
+                            if (sum > max)
+                                sum = max;
+                        }
+                        else if (activation_type == 4)
+                        {
+                            sum = static_cast<float>(1.f / (1.f + exp(-sum)));
+                        }
 
-        requantize_ops[g]->load_model(ModelBinFromMatArray(weights));
+                        outptr[j] = sum;
+                    }
 
-        requantize_scales.push_back(scale_in);
-        requantize_scales.push_back(scale_out);
+                    outptr += outw;
+                }
+            }
+        }
     }
 
     return 0;
 }
 
-int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+static inline signed char float2int8(float v)
+{
+    int int32 = static_cast<int>(round(v));
+    if (int32 > 127) return 127;
+    if (int32 < -127) return -127;
+    return (signed char)int32;
+}
+
+int ConvolutionDepthWise::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
     // convolv with NxN kernel
     // value = value + bias
@@ -290,11 +385,10 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O
     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
 
     Mat bottom_blob_unbordered = bottom_blob;
-    if (use_int8_inference && elemsize != 1)
+    if (elemsize != 1)
     {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
+        bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
+        if (bottom_blob_unbordered.empty())
             return -100;
 
         const int channels_g = channels / group;
@@ -305,14 +399,13 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O
         {
             Option opt_g = opt;
             opt_g.num_threads = 1;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
+            opt_g.blob_allocator = bottom_blob_unbordered.allocator;
 
             const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g);
-            Mat bottom_blob_int8_g = bottom_blob_int8.channel_range(channels_g * g, channels_g);
-            quantize_ops[g]->forward(bottom_blob_g, bottom_blob_int8_g, opt_g);
-        }
+            Mat bottom_blob_int8_g = bottom_blob_unbordered.channel_range(channels_g * g, channels_g);
 
-        bottom_blob_unbordered = bottom_blob_int8;
+            quantize_float32_to_int8(bottom_blob_g, bottom_blob_int8_g, bottom_blob_int8_scales[g], opt_g);
+        }
     }
 
     Mat bottom_blob_bordered = bottom_blob_unbordered;
@@ -377,173 +470,119 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O
     }
 
     // int8
-    if (use_int8_inference)
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
+
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // depth-wise
+    if (channels == group && group == num_output)
     {
-        if (use_int8_requantize == true)
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int g=0; g<group; g++)
         {
-            Mat top_blob_tm;
-            top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
-            if (top_blob_tm.empty())
-                return -100;
-            
-            top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100; 
-
-            // depth-wise
-            if (channels == group && group == num_output)
+            signed char* outptr = top_blob.channel(g);
+            const signed char* kptr = (const signed char*)weight_data + maxk * g;
+            const Mat m = bottom_blob_bordered.channel(g);
+
+            for (int i = 0; i < outh; i++)
             {
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
+                for (int j = 0; j < outw; j++)
                 {
-                    int* outptr = top_blob_tm.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data + maxk * g;
-                    const Mat m = bottom_blob_bordered.channel(g);
+                    int sum = 0;
 
-                    for (int i = 0; i < outh; i++)
-                    {
-                        for (int j = 0; j < outw; j++)
-                        {
-                            int sum = 0;
+                    const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
 
-                            const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
-
-                            for (int k = 0; k < maxk; k++)
-                            {
-                                signed char val = sptr[ space_ofs[k] ];
-                                signed char w = kptr[k];
-                                sum += val * w;
-                            }
-
-                            outptr[j] = sum;
-                        }
-
-                        outptr += outw;
-                    }
-
-                    // requantize, reverse scale inplace
+                    for (int k = 0; k < maxk; k++)
                     {
-                        Option opt_g = opt;
-                        opt_g.num_threads = 1;
-                        opt_g.blob_allocator = top_blob.allocator;
-
-                        Mat top_blob_tm_g = top_blob_tm.channel_range(g, 1);
-                        Mat top_blob_g = top_blob.channel_range(g, 1);
-                        requantize_ops[g]->forward(top_blob_tm_g, top_blob_g, opt_g);
+                        signed char val = sptr[ space_ofs[k] ];
+                        signed char w = kptr[k];
+                        sum += val * w;
                     }
 
-                    // activation relu
-                    if (activation_type == 1)
-                    {
-                        signed char* outptr_s8 = top_blob.channel(g);
-
-                        for (int i = 0; i < outh*outw; i++)
-                        {
-                            if (outptr_s8[i] < 0)
-                                outptr_s8[i] = 0;
-                        }
-                    }                       
-                }
-            }
-            else
-            {
-                const int channels_g = channels / group;
-                const int num_output_g = num_output / group;
-
-#ifdef _WIN32
-                #pragma omp parallel for num_threads(opt.num_threads)
-#else // _WIN32
-                #pragma omp parallel for collapse(2) num_threads(opt.num_threads)
-#endif // _WIN32
-                for (int g=0; g<group; g++)
-                {
-                    for (int p=0; p<num_output_g; p++)
+                    if (use_int8_requantize)
                     {
-                        int* outptr = top_blob_tm.channel(g * num_output_g + p);
-                        const signed char* weight_data_ptr = (const signed char*)weight_data + maxk * channels_g * num_output_g * g;
+                        // requantize and relu
+                        float scale_in;
+                        if (weight_data_int8_scales[g] == 0)
+                            scale_in = 0;
+                        else
+                            scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                        for (int i = 0; i < outh; i++)
-                        {
-                            for (int j = 0; j < outw; j++)
-                            {
-                                int sum = 0;
-
-                                const signed char* kptr = weight_data_ptr + maxk * channels_g * p;
+                        float sumfp32 = sum * scale_in;
 
-                                // channels_g
-                                for (int q=0; q<channels_g; q++)
-                                {
-                                    const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
-                                    const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
+                        if (bias_term)
+                            sumfp32 += bias_data[g];
 
-                                    for (int k = 0; k < maxk; k++)
-                                    {
-                                        signed char val = sptr[ space_ofs[k] ];
-                                        signed char w = kptr[k];
-                                        sum += val * w;
-                                    }
+                        float scale_out = top_blob_int8_scale;//FIXME load param
 
-                                    kptr += maxk;
-                                }
+                        signed char sums8 = float2int8(sumfp32 * scale_out);
 
-                                outptr[j] = sum;
-                            }
-
-                            outptr += outw;
+                        if (activation_type == 1)
+                        {
+                            sums8 = std::max(sums8, (signed char)0);
                         }
-                    }
-                }
 
-                // requantize, reverse scale inplace
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                        outptr[0] = sums8;
+                        outptr += 1;
+                    }
+                    else
+                    {
+                        // dequantize and relu
+                        float scale_in;
+                        if (weight_data_int8_scales[g] == 0)
+                            scale_in = 0;
+                        else
+                            scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                    Mat top_blob_tm_g = top_blob_tm.channel_range(num_output_g * g, num_output_g);
-                    Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
-                    requantize_ops[g]->forward(top_blob_tm_g, top_blob_g, opt_g);   
+                        float sumfp32 = sum * scale_in;
 
-                    if (activation_type == 1)
-                    {
-                        for (int p=0; p<num_output_g; p++)
-                        { 
-                            signed char* outptr_s8 = top_blob.channel(g * num_output_g + p);   
+                        if (bias_term)
+                            sumfp32 += bias_data[g];
 
-                            for (int i = 0; i < outh*outw; i++)
-                            {
-                                if (outptr_s8[i] < 0)
-                                    outptr_s8[i] = 0;
-                            }
+                        if (activation_type == 1)
+                        {
+                            sumfp32 = std::max(sumfp32, 0.f);
                         }
+
+                        ((float*)outptr)[0] = sumfp32;
+                        outptr += 4;
                     }
                 }
             }
         }
-        else
-        {
-            top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100;
+    }
+    else
+    {
+        // group convolution
+        const int channels_g = channels / group;
+        const int num_output_g = num_output / group;
 
-            // depth-wise
-            if (channels == group && group == num_output)
+#ifdef _WIN32
+        #pragma omp parallel for num_threads(opt.num_threads)
+#else // _WIN32
+        #pragma omp parallel for collapse(2) num_threads(opt.num_threads)
+#endif // _WIN32
+        for (int g=0; g<group; g++)
+        {
+            for (int p=0; p<num_output_g; p++)
             {
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
-                {
-                    int* outptr = top_blob.channel(g);
-                    const signed char* kptr = (const signed char*)weight_data + maxk * g;
-                    const Mat m = bottom_blob_bordered.channel(g);
+                signed char* outptr = top_blob.channel(g * num_output_g + p);
+                const signed char* weight_data_ptr = (const signed char*)weight_data + maxk * channels_g * num_output_g * g;
 
-                    for (int i = 0; i < outh; i++)
+                for (int i = 0; i < outh; i++)
+                {
+                    for (int j = 0; j < outw; j++)
                     {
-                        for (int j = 0; j < outw; j++)
-                        {
-                            int sum = 0;
+                        int sum = 0;
 
+                        const signed char* kptr = weight_data_ptr + maxk * channels_g * p;
+
+                        // channels_g
+                        for (int q=0; q<channels_g; q++)
+                        {
+                            const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
                             const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
 
                             for (int k = 0; k < maxk; k++)
@@ -553,248 +592,59 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O
                                 sum += val * w;
                             }
 
-                            outptr[j] = sum;
-                        }
-
-                        outptr += outw;
-                    }
-
-                    // dequantize, reverse scale inplace
-                    {
-                        Option opt_g = opt;
-                        opt_g.num_threads = 1;
-                        opt_g.blob_allocator = top_blob.allocator;
-
-                        Mat top_blob_g = top_blob.channel_range(g, 1);
-                        dequantize_ops[g]->forward_inplace(top_blob_g, opt_g);
-                    }
-
-                    // activation relu
-                    if (activation_type == 1)
-                    {
-                        float* outptr_fp32 = top_blob.channel(g);
-
-                        for (int i = 0; i < outh*outw; i++)
-                        {
-                            outptr_fp32[i] = std::max(outptr_fp32[i], 0.f);
+                            kptr += maxk;
                         }
-                    }                    
-                }
-            }
-            else
-            {
-                const int channels_g = channels / group;
-                const int num_output_g = num_output / group;
-
-#ifdef _WIN32
-                #pragma omp parallel for num_threads(opt.num_threads)
-#else // _WIN32
-                #pragma omp parallel for collapse(2) num_threads(opt.num_threads)
-#endif // _WIN32
-                for (int g=0; g<group; g++)
-                {
-                    for (int p=0; p<num_output_g; p++)
-                    {
-                        int* outptr = top_blob.channel(g * num_output_g + p);
-                        const signed char* weight_data_ptr = (const signed char*)weight_data + maxk * channels_g * num_output_g * g;
 
-                        for (int i = 0; i < outh; i++)
+                        if (use_int8_requantize)
                         {
-                            for (int j = 0; j < outw; j++)
-                            {
-                                int sum = 0;
+                            // requantize and relu
+                            float scale_in;
+                            if (weight_data_int8_scales[g] == 0)
+                                scale_in = 0;
+                            else
+                                scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                                const signed char* kptr = weight_data_ptr + maxk * channels_g * p;
+                            float sumfp32 = sum * scale_in;
 
-                                // channels_g
-                                for (int q=0; q<channels_g; q++)
-                                {
-                                    const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
-                                    const signed char* sptr = m.row<signed char>(i*stride_h) + j*stride_w;
+                            if (bias_term)
+                                sumfp32 += bias_data[g * num_output_g + p];
 
-                                    for (int k = 0; k < maxk; k++)
-                                    {
-                                        signed char val = sptr[ space_ofs[k] ];
-                                        signed char w = kptr[k];
-                                        sum += val * w;
-                                    }
+                            float scale_out = top_blob_int8_scale;//FIXME load param
 
-                                    kptr += maxk;
-                                }
+                            signed char sums8 = float2int8(sumfp32 * scale_out);
 
-                                outptr[j] = sum;
+                            if (activation_type == 1)
+                            {
+                                sums8 = std::max(sums8, (signed char)0);
                             }
 
-                            outptr += outw;
+                            outptr[0] = sums8;
+                            outptr += 1;
                         }
-                    }
-                }
-
-                // dequantize, reverse scale inplace
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                        else
+                        {
+                            // dequantize and relu
+                            float scale_in;
+                            if (weight_data_int8_scales[g] == 0)
+                                scale_in = 0;
+                            else
+                                scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                    Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
-                    dequantize_ops[g]->forward_inplace(top_blob_g, opt_g);
+                            float sumfp32 = sum * scale_in;
 
-                    if (activation_type == 1)
-                    {
-                        for (int p=0; p<num_output_g; p++)
-                        { 
-                            float* outptr_fp32 = top_blob.channel(g * num_output_g + p);   
+                            if (bias_term)
+                                sumfp32 += bias_data[g * num_output_g + p];
 
-                            for (int i = 0; i < outh*outw; i++)
+                            if (activation_type == 1)
                             {
-                                outptr_fp32[i] = std::max(outptr_fp32[i], 0.f);
+                                sumfp32 = std::max(sumfp32, 0.f);
                             }
-                        }
-                    }                    
-                }
-            }
-        }
-
-        return 0;
-    }
-
-    // float32
-    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
-    if (top_blob.empty())
-        return -100;
-    
-    // depth-wise
-    if (channels == group && group == num_output)
-    {
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int g=0; g<group; g++)
-        {
-            float* outptr = top_blob.channel(g);
-            const float* kptr = (const float*)weight_data + maxk * g;
-            const Mat m = bottom_blob_bordered.channel(g);
-
-            for (int i = 0; i < outh; i++)
-            {
-                for (int j = 0; j < outw; j++)
-                {
-                    float sum = 0.f;
-
-                    if (bias_term)
-                        sum = bias_data[g];
-
-                    const float* sptr = m.row(i*stride_h) + j*stride_w;
-
-                    for (int k = 0; k < maxk; k++)
-                    {
-                        float val = sptr[ space_ofs[k] ];
-                        float w = kptr[k];
-                        sum += val * w;
-                    }
-
-                    if (activation_type == 1)
-                    {
-                        sum = std::max(sum, 0.f);
-                    }
-                    else if (activation_type == 2)
-                    {
-                        float slope = activation_params[0];
-                        sum = sum > 0.f ? sum : sum * slope;
-                    }
-                    else if (activation_type == 3)
-                    {
-                        float min = activation_params[0];
-                        float max = activation_params[1];
-                        if (sum < min)
-                            sum = min;
-                        if (sum > max)
-                            sum = max;
-                    }
-                    else if (activation_type == 4)
-                    {
-                        sum = static_cast<float>(1.f / (1.f + exp(-sum)));
-                    }
-
-                    outptr[j] = sum;
-                }
-
-                outptr += outw;
-            }
-        }
-
-        return 0;
-    }
-
-    const int channels_g = channels / group;
-    const int num_output_g = num_output / group;
-
-#ifdef _WIN32
-    #pragma omp parallel for num_threads(opt.num_threads)
-#else // _WIN32
-    #pragma omp parallel for collapse(2) num_threads(opt.num_threads)
-#endif // _WIN32
-    for (int g=0; g<group; g++)
-    {
-        for (int p=0; p<num_output_g; p++)
-        {
-            float* outptr = top_blob.channel(g * num_output_g + p);
-            const float* weight_data_ptr = (const float*)weight_data + maxk * channels_g * num_output_g * g;
-
-            for (int i = 0; i < outh; i++)
-            {
-                for (int j = 0; j < outw; j++)
-                {
-                    float sum = 0.f;
-
-                    if (bias_term)
-                        sum = bias_data[num_output_g * g + p];
 
-                    const float* kptr = weight_data_ptr + maxk * channels_g * p;
-
-                    // channels_g
-                    for (int q=0; q<channels_g; q++)
-                    {
-                        const Mat m = bottom_blob_bordered.channel(channels_g * g + q);
-                        const float* sptr = m.row(i*stride_h) + j*stride_w;
-
-                        for (int k = 0; k < maxk; k++)
-                        {
-                            float val = sptr[ space_ofs[k] ];
-                            float w = kptr[k];
-                            sum += val * w;
+                            ((float*)outptr)[0] = sumfp32;
+                            outptr += 4;
                         }
-
-                        kptr += maxk;
-                    }
-
-                    if (activation_type == 1)
-                    {
-                        sum = std::max(sum, 0.f);
-                    }
-                    else if (activation_type == 2)
-                    {
-                        float slope = activation_params[0];
-                        sum = sum > 0.f ? sum : sum * slope;
-                    }
-                    else if (activation_type == 3)
-                    {
-                        float min = activation_params[0];
-                        float max = activation_params[1];
-                        if (sum < min)
-                            sum = min;
-                        if (sum > max)
-                            sum = max;
                     }
-                    else if (activation_type == 4)
-                    {
-                        sum = static_cast<float>(1.f / (1.f + exp(-sum)));
-                    }
-
-                    outptr[j] = sum;
                 }
-
-                outptr += outw;
             }
         }
     }

src/layer/convolutiondepthwise.h

@@ -29,12 +29,12 @@ public:
     virtual int load_model(const ModelBin& mb);
 
     virtual int create_pipeline(const Option& opt);
-    virtual int destroy_pipeline(const Option& opt);
-
-    virtual int create_requantize_op(void);
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+protected:
+    int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
 public:
     // param
     int num_output;
@@ -68,16 +68,7 @@ public:
     Mat bottom_blob_int8_scales;
     float top_blob_int8_scale;
 
-    bool use_int8_inference;
     bool use_int8_requantize;
-
-    std::vector<ncnn::Layer*> quantize_ops;
-    std::vector<ncnn::Layer*> dequantize_ops;
-    std::vector<ncnn::Layer*> requantize_ops;
-
-    // merge de/requantize op into convolutiondepthwise op
-    std::vector<float> dequantize_scales;
-    std::vector<float> requantize_scales;
 };
 
 } // namespace ncnn

src/layer/innerproduct.cpp

@@ -24,8 +24,6 @@ InnerProduct::InnerProduct()
 {
     one_blob_only = true;
     support_inplace = false;
-
-    quantize = 0;
 }
 
 int InnerProduct::load_param(const ParamDict& pd)
@@ -64,87 +62,23 @@ int InnerProduct::load_model(const ModelBin& mb)
 
 int InnerProduct::create_pipeline(const Option& opt)
 {
-    bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u);
-    bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u);
-
-    if (weight_data_is_int8 && !opt.use_int8_inference)
-    {
-        fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n");
-        return -1;
-    }
-
-    use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term));
-
-    // initial the quantize,dequantize op layer
-    if (use_int8_inference)
-    {
-        quantize = ncnn::create_layer(ncnn::LayerType::Quantize);
-        {
-            ncnn::ParamDict pd;
-            pd.set(0, bottom_blob_int8_scale);// scale
-
-            quantize->load_param(pd);
-
-            quantize->create_pipeline(opt);
-        }
-
-        dequantize_ops.resize(num_output);
-        for (int n=0; n<num_output; n++)
-        {
-            dequantize_ops[n] = ncnn::create_layer(ncnn::LayerType::Dequantize);
-
-            float top_rescale = 1.f;
-
-            if (weight_data_int8_scales[n] == 0)
-                top_rescale = 0;
-            else
-                top_rescale = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[n]);
-
-            ncnn::ParamDict pd;
-            pd.set(0, top_rescale);// scale
-            pd.set(1, bias_term);  // bias_term
-            pd.set(2, 1);          // bias_data_size
-
-            dequantize_ops[n]->load_param(pd);
-
-            ncnn::Mat weights[1];
-            weights[0] = bias_data.range(n, 1);
-
-            dequantize_ops[n]->load_model(ModelBinFromMatArray(weights));
-
-            dequantize_ops[n]->create_pipeline(opt);
-        }
-    }
-
     // runtime quantize the weight data
-    if (weight_data_is_float32 && use_int8_inference)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term)
     {
-        // quantize weight to int8
         Mat int8_weight_data(weight_data_size, (size_t)1u);
         if (int8_weight_data.empty())
             return -100;
 
         const int weight_data_size_output = weight_data_size / num_output;
 
-        for (int n=0; n<num_output; n++)
+        for (int p=0; p<num_output; p++)
         {
-            Layer* op = ncnn::create_layer(ncnn::LayerType::Quantize);
-
-            ncnn::ParamDict pd;
-            pd.set(0, weight_data_int8_scales[n]);// scale
-
-            op->load_param(pd);
-
-            op->create_pipeline(opt);
-
             Option opt_q = opt;
             opt_q.blob_allocator = int8_weight_data.allocator;
 
-            const Mat weight_data_n = weight_data.range(weight_data_size_output * n, weight_data_size_output);
-            Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * n, weight_data_size_output);
-            op->forward(weight_data_n, int8_weight_data_n, opt_q);
-
-            delete op;
+            const Mat weight_data_n = weight_data.range(weight_data_size_output * p, weight_data_size_output);
+            Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * p, weight_data_size_output);
+            quantize_float32_to_int8(weight_data_n, int8_weight_data_n, weight_data_int8_scales[p], opt_q);
         }
 
         weight_data = int8_weight_data;
@@ -153,27 +87,13 @@ int InnerProduct::create_pipeline(const Option& opt)
     return 0;
 }
 
-int InnerProduct::destroy_pipeline(const Option& opt)
+int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
-    if (quantize)
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
-        quantize->destroy_pipeline(opt);
-        delete quantize;
-        quantize = 0;
+        return forward_int8(bottom_blob, top_blob, opt);
     }
 
-    for (int i=0; i<(int)dequantize_ops.size(); i++)
-    {
-        dequantize_ops[i]->destroy_pipeline(opt);
-        delete dequantize_ops[i];
-    }
-    dequantize_ops.clear();
-
-    return 0;
-}
-
-int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
-{
     int w = bottom_blob.w;
     int h = bottom_blob.h;
     int channels = bottom_blob.c;
@@ -184,74 +104,6 @@ int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o
     if (top_blob.empty())
         return -100;
 
-    if (use_int8_inference)
-    {
-        Mat bottom_blob_tm = bottom_blob;
-        if (elemsize != 1)
-        {
-            Mat bottom_blob_int8;
-            bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-            if (bottom_blob_int8.empty())
-                return -100;
-
-            // quantize, scale and round to nearest
-            {
-                Option opt_g = opt;
-                opt_g.blob_allocator = bottom_blob_int8.allocator;
-
-                quantize->forward(bottom_blob, bottom_blob_int8, opt_g);
-            }
-
-            bottom_blob_tm = bottom_blob_int8;
-        }
-
-        // num_output
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int p=0; p<num_output; p++)
-        {
-            int sum = 0;
-            int* out = top_blob;
-
-            // channels
-            for (int q=0; q<channels; q++)
-            {
-                const signed char* w = (const signed char*)weight_data + size * channels * p + size * q;
-                const signed char* m = bottom_blob_tm.channel(q);
-
-                for (int i = 0; i < size; i++)
-                {
-                    sum += m[i] * w[i];
-                }
-            }
-
-            out[p] = sum;       
-        }
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int p=0; p<num_output; p++)
-        {
-            int* out_s32 = top_blob;
-            float* out_f32 = top_blob;
-            float top_rescale = 1.f;
-            if (weight_data_int8_scales[p] == 0)
-                top_rescale = 0;
-            else
-                top_rescale = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
-
-            if (bias_term)
-                out_f32[p] = out_s32[p] * top_rescale + bias_data[p];
-            else
-                out_f32[p] = out_s32[p] * top_rescale;
-
-            if (activation_type == 1)
-            {
-                out_f32[p] = std::max(out_f32[p], 0.f);
-            }                
-        }
-
-        return 0;
-    }
-
     // num_output
     #pragma omp parallel for num_threads(opt.num_threads)
     for (int p=0; p<num_output; p++)
@@ -302,4 +154,68 @@ int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o
     return 0;
 }
 
+int InnerProduct::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+    int size = w * h;
+
+    Mat bottom_blob_tm = bottom_blob;
+    if (elemsize != 1)
+    {
+        Option opt_g = opt;
+        opt_g.blob_allocator = opt.workspace_allocator;
+
+        quantize_float32_to_int8(bottom_blob, bottom_blob_tm, bottom_blob_int8_scale, opt_g);
+    }
+
+    top_blob.create(num_output, elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p=0; p<num_output; p++)
+    {
+        float* outptr = top_blob;
+
+        int sum = 0;
+
+        // channels
+        for (int q=0; q<channels; q++)
+        {
+            const signed char* w = (const signed char*)weight_data + size * channels * p + size * q;
+            const signed char* m = bottom_blob_tm.channel(q);
+
+            for (int i = 0; i < size; i++)
+            {
+                sum += m[i] * w[i];
+            }
+        }
+
+        // dequantize and relu
+        float scale_in;
+        if (weight_data_int8_scales[p] == 0)
+            scale_in = 0;
+        else
+            scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+        float sumfp32 = sum * scale_in;
+
+        if (bias_term)
+            sumfp32 += bias_data[p];
+
+        if (activation_type == 1)
+        {
+            sumfp32 = std::max(sumfp32, 0.f);
+        }
+
+        outptr[p] = sumfp32;
+    }
+
+    return 0;
+}
+
 } // namespace ncnn

src/layer/innerproduct.h

@@ -29,9 +29,9 @@ public:
     virtual int load_model(const ModelBin& mb);
 
     virtual int create_pipeline(const Option& opt);
-    virtual int destroy_pipeline(const Option& opt);
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+    virtual int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
     // param
@@ -52,11 +52,6 @@ public:
 
     Mat weight_data_int8_scales;
     float bottom_blob_int8_scale;
-
-    bool use_int8_inference;
-
-    ncnn::Layer* quantize;
-    std::vector<ncnn::Layer*> dequantize_ops;
 };
 
 } // namespace ncnn

src/layer/x86/convolution_1x1_int8.h

@@ -196,47 +196,3 @@ static void conv1x1s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat
         }
     }
 }
-
-static void conv1x1s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 1;
-    int kernel_h = 1;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv1x1s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 1;
-    int kernel_h = 1;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv1x1s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 1;
-    int kernel_h = 1;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}
-
-static void conv1x1s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 1;
-    int kernel_h = 1;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}

src/layer/x86/convolution_3x3_int8.h

@@ -909,47 +909,3 @@ static void conv3x3s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat
         }
     }
 }
-
-static void conv3x3s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv3x3s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv3x3s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}
-
-static void conv3x3s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 3;
-    int kernel_h = 3;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}

src/layer/x86/convolution_5x5_int8.h

@@ -1,79 +0,0 @@
-// BUG1989 is pleased to support the open source community by supporting ncnn available.
-//
-// Copyright (C) 2019 BUG1989. All rights reserved.
-//
-// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// https://opensource.org/licenses/BSD-3-Clause
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-static void conv5x5s1_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv5x5s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv5x5s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv5x5s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv5x5s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}
-
-static void conv5x5s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 5;
-    int kernel_h = 5;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}

src/layer/x86/convolution_7x7_int8.h

@@ -1,79 +0,0 @@
-// BUG1989 is pleased to support the open source community by supporting ncnn available.
-//
-// Copyright (C) 2019 BUG1989. All rights reserved.
-//
-// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
-// in compliance with the License. You may obtain a copy of the License at
-//
-// https://opensource.org/licenses/BSD-3-Clause
-//
-// Unless required by applicable law or agreed to in writing, software distributed
-// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-// CONDITIONS OF ANY KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations under the License.
-
-static void conv7x7s1_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv7x7s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt);
-}
-
-static void conv7x7s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv7x7s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_dequant, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt);
-}
-
-static void conv7x7s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 1;
-    int stride_h = 1;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}
-
-static void conv7x7s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector<float> scales_requant, const Option& opt)
-{
-    int kernel_w = 7;
-    int kernel_h = 7;
-
-    int stride_w = 2;
-    int stride_h = 2;
-
-    conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt);
-}

src/layer/x86/convolution_x86.cpp

@@ -35,14 +35,13 @@ namespace ncnn {
 #include "convolution_sgemm_int8.h"
 #include "convolution_1x1_int8.h"
 #include "convolution_3x3_int8.h"
-#include "convolution_5x5_int8.h"
-#include "convolution_7x7_int8.h"
 
 DEFINE_LAYER_CREATOR(Convolution_x86)
 
 Convolution_x86::Convolution_x86()
 {
     activation = 0;
+    convolution_dilation1 = 0;
 }
 
 int Convolution_x86::create_pipeline(const Option& opt)
@@ -84,35 +83,71 @@ int Convolution_x86::create_pipeline(const Option& opt)
         activation->create_pipeline(opt);
     }
 
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return create_pipeline_int8_x86(opt);
+    }
+
+    int kernel_size = kernel_w * kernel_h;
+    int num_input = weight_data_size / kernel_size / num_output;
+
     use_winograd3x3 = false;
 
-    if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+    if (kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
-        int num_input = weight_data_size / 9 / num_output;
-        // winograd is slow on small channel count
-        if(num_input >= 16 && num_output >= 16)
-            use_winograd3x3 = true;
-    }           
+        convolution_dilation1 = ncnn::create_layer(ncnn::LayerType::Convolution);
 
-    if (use_winograd3x3)
-    {
-        int num_input = weight_data_size / 9 / num_output;
+        // set param
+        ncnn::ParamDict pd;
+        pd.set(0, num_output);// num_output
+        pd.set(1, kernel_w);
+        pd.set(11, kernel_h);
+        pd.set(2, 1);
+        pd.set(12, 1);
+        pd.set(3, 1);// stride_w
+        pd.set(13, 1);// stride_h
+        pd.set(4, 0);// pad_w
+        pd.set(14, 0);// pad_h
+        pd.set(5, bias_term);
+        pd.set(6, weight_data_size);
+
+        convolution_dilation1->load_param(pd);
+
+        // set weights
+        if (bias_term)
+        {
+            ncnn::Mat weights[2];
+            weights[0] = weight_data;
+            weights[1] = bias_data;
 
-        if (use_int8_inference)
-            // conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output);
-            conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output);
+            convolution_dilation1->load_model(ModelBinFromMatArray(weights));
+        }
         else
-            conv3x3s1_winograd23_transform_kernel_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output);
-//             conv3x3s1_winograd43_transform_kernel_sse(weight_data, weight_3x3_winograd43_data, num_input, num_output);
-    }
+        {
+            ncnn::Mat weights[1];
+            weights[0] = weight_data;
+
+            convolution_dilation1->load_model(ModelBinFromMatArray(weights));
+        }
 
-    if (use_int8_inference == false)
+        convolution_dilation1->create_pipeline(opt);
+    }
+    else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1
+        && num_input >= 16 && num_output >= 16)
     {
-        int kernel_size = kernel_w * kernel_h;
-        int num_input = weight_data_size / kernel_size / num_output;
+        // winograd is slow on small channel count
+        use_winograd3x3 = true;
 
+        conv3x3s1_winograd23_transform_kernel_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output);
+//         conv3x3s1_winograd43_transform_kernel_sse(weight_data, weight_3x3_winograd43_data, num_input, num_output);
+
+        // for small size
+        conv_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_size);
+    }
+    else
+    {
         conv_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_size);
-    }       
+    }
 
     return 0;
 }
@@ -126,19 +161,43 @@ int Convolution_x86::destroy_pipeline(const Option& opt)
         activation = 0;
     }
 
+    if (convolution_dilation1)
+    {
+        convolution_dilation1->destroy_pipeline(opt);
+        delete convolution_dilation1;
+        convolution_dilation1 = 0;
+    }
+
     return 0;
 }
 
-int Convolution_x86::forwardDilation(const Mat& bottom_blob, Mat& top_blob, conv_func conv, const Option& opt) const
+int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
+    // convolv with NxN kernel
+    // value = value + bias
+
+    if (bottom_blob.dims != 3)
+    {
+        return Convolution::forward(bottom_blob, top_blob, opt);
+    }
+
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return forward_int8_x86(bottom_blob, top_blob, opt);
+    }
+
+    if ((dilation_w > 1 || dilation_h > 1) && (stride_w > 1 || stride_h > 1))
+    {
+        return Convolution::forward(bottom_blob, top_blob, opt);
+    }
+
     int w = bottom_blob.w;
     int h = bottom_blob.h;
+    int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
 
-    const int kernel_size = kernel_w;
-    const int stride = stride_w;
-    const int dilation = dilation_w;
-    const int kernel_extent = dilation * (kernel_size - 1) + 1;
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
 
     Mat bottom_blob_bordered = bottom_blob;
     if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
@@ -146,319 +205,113 @@ int Convolution_x86::forwardDilation(const Mat& bottom_blob, Mat& top_blob, conv
         Option opt_b = opt;
         opt_b.blob_allocator = opt.workspace_allocator;
         copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
-        if (bottom_blob_bordered.empty())
-            return -100;
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
-        int wpad = kernel_extent + (w - 1) / stride * stride - w;
-        int hpad = kernel_extent + (h - 1) / stride * stride - h;
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
     {
-        int wpad = kernel_extent + (w - 1) / stride * stride - w;
-        int hpad = kernel_extent + (h - 1) / stride * stride - h;
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
+    if (bottom_blob_bordered.empty())
+        return -100;
 
-    int outw = (w - kernel_extent) / stride + 1;
-    int outh = (h - kernel_extent) / stride + 1;
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
+
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
 
+    // float32
     top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
     if (top_blob.empty())
         return -100;
 
-    // Make (dilation * dilation) batches
-    Mat inner_bottom_blob;
-    Mat inner_top_blob;
-    for (int x = 0; x < dilation; x ++)
+    if (kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
-        for (int y = 0; y < dilation; y ++)
-        {
-            int inner_w = (w - y + dilation - 1) / dilation;
-            int inner_h = (h - x + dilation - 1) / dilation;
-
-            int inner_outw = (inner_w - kernel_size) / stride + 1;
-            int inner_outh = (inner_h - kernel_size) / stride + 1;
-
-            inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator);
-            if (inner_bottom_blob.empty())
-                return -100;
-
-            inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator);
-            if (inner_top_blob.empty())
-                return -100;
+        return forwardDilation_x86(bottom_blob_bordered, top_blob, opt);
+    }
 
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int c = 0; c < bottom_blob.c; c ++)
-            {
-                float *outptr = inner_bottom_blob.channel(c);
+    if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+    {
+        if (use_winograd3x3 && outw >= 8 && outh >=8)
+        {
+            conv3x3s1_winograd23_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, bias_data, opt);
+//             conv3x3s1_winograd43_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd43_data, bias_data, opt);
+        }
+        else
+        {
+            conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);
+        }
 
-                for (int i = 0; i < inner_h; i ++)
-                {
-                    const float* ptr = (const float *)bottom_blob_bordered.channel(c) + dilation * i * w + x * w + y;
-                    for (int j = 0; j < inner_w; j ++)
-                    {
-                        outptr[j] = ptr[j*dilation];
-                    }
-                    outptr += inner_w;
-                }
-            }
+        if (activation)
+        {
+            activation->forward_inplace(top_blob, opt);
+        }
+    }
+    else
+    {
+//         conv1x1s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+//         conv1x1s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+//         conv3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+//         conv3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+//         conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
 
-            Option opt_g = opt;
-            opt_g.blob_allocator = inner_top_blob.allocator;
-            if (kernel_size == 7)
-            {
-            // FIXME conv7x7s1_sse use sgemm
-            conv(inner_bottom_blob, inner_top_blob, weight_sgemm_data, bias_data, opt_g);
-            }
-            else
-            {
-            conv(inner_bottom_blob, inner_top_blob, weight_data, bias_data, opt_g);
-            }
+        conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);
 
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int c = 0; c < num_output; c ++)
-            {
-                float *outptr = (float *)top_blob.channel(c) + x * outw + y;
-                for (int i = 0; i < inner_outh; i ++)
-                {
-                    const float* ptr = (const float *)inner_top_blob.channel(c) + i * inner_outw;
-                    for (int j = 0; j < inner_outw; j ++)
-                    {
-                        outptr[j*dilation] = ptr[j];
-                    }
-                    outptr += dilation * outw;
-                }
-            }
+        if (activation)
+        {
+            activation->forward_inplace(top_blob, opt);
         }
     }
 
     return 0;
 }
 
-int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+int Convolution_x86::create_pipeline_int8_x86(const Option& opt)
 {
-    // convolv with NxN kernel
-    // value = value + bias
-
-    if (bottom_blob.dims != 3)
-    {
-        return Convolution::forward(bottom_blob, top_blob, opt);
-    }
-
-    if (kernel_w != kernel_h || stride_w != stride_h)
-    {
-        return Convolution::forward(bottom_blob, top_blob, opt);
-    }
+    int kernel_size = kernel_w * kernel_h;
+    int num_input = weight_data_size / kernel_size / num_output;
 
-    const int kernel_size = kernel_w;
-    const int stride = stride_w;  
-
-    if (kernel_size > 7 || stride > 7 || dilation_w != dilation_h)
-    {
-        return Convolution::forward(bottom_blob, top_blob, opt);
-    }
-
-    typedef void (*conv_func)(const Mat&, Mat&, const Mat&, const Mat&, const Option&);
-
-    // kernel_size x stride
-    conv_func conv_func_table[7][4] =
-    {
-        {
-            conv1x1s1_sse,
-            conv1x1s2_sse,
-            0,
-            0
-        }, // kernel_size = 1
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 2
-        {
-            conv3x3s1_sse,
-            conv3x3s2_sse,
-            0,
-            0
-        }, // kernel_size = 3
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 4
-        {
-            conv5x5s1_sse,
-            conv5x5s2_sse,
-            0,
-            0
-        }, // kernel_size = 5
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 6
-        {
-            conv7x7s1_sse,          
-            conv7x7s2_sse,          
-            0,
-            0
-        }  // kernel_size = 7        
-    };
-
-    typedef void (*conv_int8_dequant_func)(const Mat&, Mat&, const Mat&, const Mat&, std::vector<float>, const Option&);
-    typedef void (*conv_int8_requant_func)(const Mat&, Mat&, const Mat&, const Mat&, std::vector<float>, const Option&);
-
-    // kernel_size x stride
-    conv_int8_dequant_func conv_int8_dequant_func_table[7][4] =
-    {
-        {
-            conv1x1s1_int8_dequant_sse,
-            conv1x1s2_int8_dequant_sse,
-            0,
-            0
-        }, // kernel_size = 1
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 2
-        {
-            conv3x3s1_int8_dequant_sse,
-            conv3x3s2_int8_dequant_sse,
-            0,
-            0,
-        }, // kernel_size = 3
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 4
-        {        
-            conv5x5s1_int8_dequant_sse,
-            conv5x5s2_int8_dequant_sse,    
-            0,
-            0
-        }, // kernel_size = 5
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 6
-        {
-            conv7x7s1_int8_dequant_sse,          
-            conv7x7s2_int8_dequant_sse, 
-            0,
-            0
-        }  // kernel_size = 7
-    };
+    use_winograd3x3_int8 = false;
 
-    conv_int8_requant_func conv_int8_requant_func_table[7][4] =
+    if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1
+        && num_input >= 16 && num_output >= 16)
     {
-        {
-            conv1x1s1_int8_requant_sse,
-            conv1x1s2_int8_requant_sse,
-            0,
-            0
-        }, // kernel_size = 1
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 2
-        {
-            conv3x3s1_int8_requant_sse,
-            conv3x3s2_int8_requant_sse,
-            0,
-            0,
-        }, // kernel_size = 3
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 4
-        {        
-            conv5x5s1_int8_requant_sse,
-            conv5x5s2_int8_requant_sse,    
-            0,
-            0
-        }, // kernel_size = 5
-        {
-            0,
-            0,
-            0,
-            0
-        }, // kernel_size = 6
-        {
-            conv7x7s1_int8_requant_sse,          
-            conv7x7s2_int8_requant_sse, 
-            0,
-            0
-        }  // kernel_size = 7
-    };
-
-    conv_func conv = 0;
-    conv_int8_dequant_func conv_int8_dequant = 0;
-    conv_int8_requant_func conv_int8_requant = 0;
+        // winograd is slow on small channel count
+        use_winograd3x3_int8 = true;
 
-    if (use_int8_inference)
-    {
-        if (use_int8_requantize)
-            conv_int8_requant = conv_int8_requant_func_table[kernel_size-1][stride-1];
-        else
-            conv_int8_dequant = conv_int8_dequant_func_table[kernel_size-1][stride-1];  
-        if ((!conv_int8_requant) && (!conv_int8_dequant))
-        {
-            return Convolution::forward(bottom_blob, top_blob, opt);
-        }
+        conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);
+//         conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output);
     }
     else
     {
-        conv = conv_func_table[kernel_size-1][stride-1];
-        if (!conv)
-        {
-            return Convolution::forward(bottom_blob, top_blob, opt);
-        }
+        // TODO offline transform weight
+    }
 
-        if (dilation_w != 1)
-        {
-            if (stride != 1)
-                return Convolution::forward(bottom_blob, top_blob, opt);
+    return 0;
+}
 
-            return forwardDilation(bottom_blob, top_blob, conv, opt);
-        }
+int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    if (dilation_w > 1 || dilation_h > 1)
+    {
+        return Convolution::forward(bottom_blob, top_blob, opt);
     }
 
     int w = bottom_blob.w;
@@ -466,23 +319,16 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
     int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
 
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
     Mat bottom_blob_unbordered = bottom_blob;
-    if (use_int8_inference && elemsize != 1)
+    if (elemsize != 1)
     {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
-            return -100;
-
-        // quantize, scale and round to nearest
-        {
-            Option opt_g = opt;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
+        Option opt_g = opt;
+        opt_g.blob_allocator = opt.workspace_allocator;
 
-            quantize->forward(bottom_blob, bottom_blob_int8, opt_g);
-        }
-
-        bottom_blob_unbordered = bottom_blob_int8;
+        quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g);
     }
 
     Mat bottom_blob_bordered = bottom_blob_unbordered;
@@ -491,131 +337,231 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
         Option opt_b = opt;
         opt_b.blob_allocator = opt.workspace_allocator;
         copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
-        if (bottom_blob_bordered.empty())
-            return -100;
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
-        int wpad = kernel_size + (w - 1) / stride * stride - w;
-        int hpad = kernel_size + (h - 1) / stride * stride - h;
+        // tensorflow padding=SAME or onnx padding=SAME_UPPER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
     {
-        int wpad = kernel_size + (w - 1) / stride * stride - w;
-        int hpad = kernel_size + (h - 1) / stride * stride - h;
+        // onnx padding=SAME_LOWER
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
         if (wpad > 0 || hpad > 0)
         {
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
+    if (bottom_blob_bordered.empty())
+        return -100;
+
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
+
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
+
+    // int8
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
 
-    int outw = (w - kernel_size) / stride + 1;
-    int outh = (h - kernel_size) / stride + 1;
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
 
     // int8
-    if (use_int8_inference)
-    {         
-        if (use_int8_requantize == true)
+    if (use_int8_requantize)
+    {
+        Mat top_blob_tm;
+        top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
+        if (top_blob_tm.empty())
+            return -100;
+
+        if (use_winograd3x3_int8)
         {
-            Mat top_blob_tm;
-            top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
-            if (top_blob_tm.empty())
-                return -100;
-            
-            top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100; 
+            conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);
+//             conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt);
 
-            if (use_winograd3x3)
+            // requantize, reverse scale inplace
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int p=0; p<num_output; p++)
             {
-                // conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data, opt);
-                conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data, opt);
+                Option opt_g = opt;
+                opt_g.num_threads = 1;
+                opt_g.blob_allocator = top_blob.allocator;
 
-                // requantize, reverse scale inplace
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int p=0; p<num_output; p++)
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1);
+                Mat top_blob_g = top_blob.channel_range(p, 1);
 
-                    Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1);
-                    Mat top_blob_g = top_blob.channel_range(p, 1);
-                    requantize_ops[p]->forward(top_blob_tm_g, top_blob_g, opt_g);
-                }
+                // requantize and relu
+                float scale_in;
+                if (weight_data_int8_scales[p] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                float scale_out = top_blob_int8_scale;//FIXME load param
+
+                requantize_int8_to_int8(top_blob_tm, top_blob, scale_in, scale_out, &bias_data[p], bias_term ? 1 : 0, 0, opt_g);
             }
-            else
-                conv_int8_requant(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
         }
         else
         {
-            top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100;
-
-            if (use_winograd3x3)
+            std::vector<float> requantize_scales;
+            for (int p=0; p<num_output; p++)
             {
-                // conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, opt);
-                conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, opt);
+                float scale_in;
+                if (weight_data_int8_scales[p] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
 
-                // dequantize, reverse scale inplace
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int p=0; p<num_output; p++)
-                {
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                float scale_out = top_blob_int8_scale;
 
-                    Mat top_blob_g = top_blob.channel_range(p, 1);
-                    dequantize_ops[p]->forward_inplace(top_blob_g, opt_g);
-                }
+                requantize_scales.push_back(scale_in);
+                requantize_scales.push_back(scale_out);
             }
-            else
-                conv_int8_dequant(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);     
+
+            conv_im2col_sgemm_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, kernel_w, kernel_h, stride_w, stride_h, bias_data, requantize_scales, opt);
         }
+    }
+    else
+    {
+        if (use_winograd3x3_int8)
+        {
+            conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);
+//             conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt);
 
-        if (activation)
+            // dequantize, reverse scale inplace
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int p=0; p<num_output; p++)
+            {
+                Option opt_g = opt;
+                opt_g.num_threads = 1;
+                opt_g.blob_allocator = top_blob.allocator;
+
+                Mat top_blob_g = top_blob.channel_range(p, 1);
+
+                // dequantize
+                float scale_in;
+                if (weight_data_int8_scales[p] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                dequantize_int32_to_float32(top_blob_g, scale_in, &bias_data[p], bias_term ? 1 : 0, opt_g);
+            }
+        }
+        else
         {
-            activation->forward_inplace(top_blob, opt);
-        }        
+            std::vector<float> dequantize_scales;
+            for (int p=0; p<num_output; p++)
+            {
+                float scale_in;
+                if (weight_data_int8_scales[p] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]);
+
+                dequantize_scales.push_back(scale_in);
+            }
 
-        return 0;
+            conv_im2col_sgemm_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, kernel_w, kernel_h, stride_w, stride_h, bias_data, dequantize_scales, opt);
+        }
     }
 
-    // float32
+    if (activation)
+    {
+        activation->forward_inplace(top_blob, opt);
+    }
+
+    return 0;
+}
+
+int Convolution_x86::forwardDilation_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    size_t elemsize = bottom_blob.elemsize;
+
+    const int kernel_size = kernel_w;
+    const int stride = stride_w;
+    const int dilation = dilation_w;
+    const int kernel_extent = dilation * (kernel_size - 1) + 1;
+
+    int outw = (w - kernel_extent) / stride + 1;
+    int outh = (h - kernel_extent) / stride + 1;
+
     top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
     if (top_blob.empty())
-        return -100;    
+        return -100;
 
-    if (use_winograd3x3 && outw >= 8 && outh >=8)
+    // Make (dilation * dilation) batches
+    Mat inner_bottom_blob;
+    Mat inner_top_blob;
+    for (int x = 0; x < dilation; x ++)
     {
-        conv3x3s1_winograd23_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, bias_data, opt);
-//         conv3x3s1_winograd43_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd43_data, bias_data, opt);
+        for (int y = 0; y < dilation; y ++)
+        {
+            int inner_w = (w - y + dilation - 1) / dilation;
+            int inner_h = (h - x + dilation - 1) / dilation;
+
+            int inner_outw = (inner_w - kernel_size) / stride + 1;
+            int inner_outh = (inner_h - kernel_size) / stride + 1;
+
+            inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator);
+            if (inner_bottom_blob.empty())
+                return -100;
+
+            inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator);
+            if (inner_top_blob.empty())
+                return -100;
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int c = 0; c < bottom_blob.c; c ++)
+            {
+                float *outptr = inner_bottom_blob.channel(c);
+
+                for (int i = 0; i < inner_h; i ++)
+                {
+                    const float* ptr = (const float *)bottom_blob.channel(c) + dilation * i * w + x * w + y;
+                    for (int j = 0; j < inner_w; j ++)
+                    {
+                        outptr[j] = ptr[j*dilation];
+                    }
+                    outptr += inner_w;
+                }
+            }
+
+            Option opt_g = opt;
+            opt_g.blob_allocator = inner_top_blob.allocator;
+            convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g);
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int c = 0; c < num_output; c ++)
+            {
+                float *outptr = (float *)top_blob.channel(c) + x * outw + y;
+                for (int i = 0; i < inner_outh; i ++)
+                {
+                    const float* ptr = (const float *)inner_top_blob.channel(c) + i * inner_outw;
+                    for (int j = 0; j < inner_outw; j ++)
+                    {
+                        outptr[j*dilation] = ptr[j];
+                    }
+                    outptr += dilation * outw;
+                }
+            }
+        }
     }
-    else
-        //conv(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
-        conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt);
 
     if (activation)
     {

src/layer/x86/convolution_x86.h

@@ -19,8 +19,6 @@
 
 namespace ncnn {
 
-typedef void (*conv_func)(const Mat&, Mat&, const Mat&, const Mat&, const Option&);
-
 class Convolution_x86 : virtual public Convolution
 {
 public:
@@ -30,7 +28,11 @@ public:
     virtual int destroy_pipeline(const Option& opt);
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
-    virtual int forwardDilation(const Mat& bottom_blob, Mat &top_blob, conv_func conv, const Option& opt) const;
+
+protected:
+    int create_pipeline_int8_x86(const Option& opt);
+    int forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+    int forwardDilation_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
     Layer* activation;
@@ -38,6 +40,13 @@ public:
     Mat weight_3x3_winograd23_data;
     Mat weight_sgemm_data;
     std::vector<Mat> weight_3x3_winograd43_data;
+
+    // forwardDilation
+    Layer* convolution_dilation1;
+
+    // int8
+    bool use_winograd3x3_int8;
+    Mat weight_3x3_winograd23_data_int8;
 };
 
 } // namespace ncnn

src/layer/x86/convolutiondepthwise_x86.cpp

@@ -14,10 +14,6 @@
 
 #include "convolutiondepthwise_x86.h"
 
-#ifdef _OPENMP
-#include <omp.h>
-#endif
-
 #include "layer_type.h"
 
 namespace ncnn {
@@ -84,14 +80,16 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
     if (channels == group && group == num_output)
     {
         // depth-wise specific
-        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
+        // special path for both int8 and fp32
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
         {
-            if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2))
-            {
-                return 0;
-            }
+            return 0;
         }
-    }    
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            return 0;
+        }
+    }
 
     const int channels_g = channels / group;
     const int num_output_g = num_output / group;
@@ -121,6 +119,8 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
         pd.set(5, bias_term);
         pd.set(6, maxk * channels_g * num_output_g);// weight_data_size
         pd.set(8, int8_scale_term);
+        pd.set(9, activation_type);
+        pd.set(10, activation_params);
 
         op->load_param(pd);
 
@@ -155,6 +155,8 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt)
 
         op->create_pipeline(opt);
 
+//         op->use_int8_requantize = use_int8_requantize; FIXME
+
         group_ops[g] = op;
     }      
 
@@ -185,26 +187,125 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
     // convolv with NxN kernel
     // value = value + bias
 
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return forward_int8_x86(bottom_blob, top_blob, opt);
+    }
+
     int w = bottom_blob.w;
     int h = bottom_blob.h;
     int channels = bottom_blob.c;
     size_t elemsize = bottom_blob.elemsize;
 
-    if (channels % group != 0 || num_output % group != 0)
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
+    Mat bottom_blob_bordered = bottom_blob;
+    if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
+    {
+        Option opt_b = opt;
+        opt_b.blob_allocator = opt.workspace_allocator;
+        copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
+    }
+    else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
-        // reject invalid group
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
+    {
+        int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w;
+        int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h;
+        if (wpad > 0 || hpad > 0)
+        {
+            Option opt_b = opt;
+            opt_b.blob_allocator = opt.workspace_allocator;
+            copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
+        }
+    }
+    if (bottom_blob_bordered.empty())
+        return -100;
+
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
+
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
+
+    // float32
+    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
+    if (top_blob.empty())
         return -100;
+
+    // depth-wise
+    if (channels == group && group == num_output)
+    {
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+        {
+            convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+
+            return 0;
+        }
+        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
+        {
+            convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
+
+            if (activation)
+            {
+                activation->forward_inplace(top_blob, opt);
+            }
+
+            return 0;
+        }
     }
 
+    // group convolution
+    const int channels_g = channels / group;
+    const int num_output_g = num_output / group;
+
+    for (int g=0; g<group; g++)
+    {
+        const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
+        Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
+
+        const ncnn::Layer* op = group_ops[g];
+
+        Option opt_g = opt;
+        opt_g.blob_allocator = top_blob.allocator;
+
+        // forward
+        op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
+    }
+
+    return 0;
+}
+
+int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+
     const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
     const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
 
     Mat bottom_blob_unbordered = bottom_blob;
-    if (use_int8_inference && elemsize != 1)
+    if (elemsize != 1)
     {
-        Mat bottom_blob_int8;
-        bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
-        if (bottom_blob_int8.empty())
+        bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator);
+        if (bottom_blob_unbordered.empty())
             return -100;
 
         const int channels_g = channels / group;
@@ -215,15 +316,14 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         {
             Option opt_g = opt;
             opt_g.num_threads = 1;
-            opt_g.blob_allocator = bottom_blob_int8.allocator;
+            opt_g.blob_allocator = bottom_blob_unbordered.allocator;
 
             const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g);
-            Mat bottom_blob_int8_g = bottom_blob_int8.channel_range(channels_g * g, channels_g);
-            quantize_ops[g]->forward(bottom_blob_g, bottom_blob_int8_g, opt_g);
-        }
+            Mat bottom_blob_int8_g = bottom_blob_unbordered.channel_range(channels_g * g, channels_g);
 
-        bottom_blob_unbordered = bottom_blob_int8;       
-    }     
+            quantize_float32_to_int8(bottom_blob_g, bottom_blob_int8_g, bottom_blob_int8_scales[g], opt_g);
+        }
+    }
 
     Mat bottom_blob_bordered = bottom_blob_unbordered;
     if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0)
@@ -231,11 +331,6 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         Option opt_b = opt;
         opt_b.blob_allocator = opt.workspace_allocator;
         copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b);
-        if (bottom_blob_bordered.empty())
-            return -100;
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233)
     {
@@ -246,12 +341,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
     else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234)
     {
@@ -262,234 +352,107 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
             Option opt_b = opt;
             opt_b.blob_allocator = opt.workspace_allocator;
             copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b);
-            if (bottom_blob_bordered.empty())
-                return -100;
         }
-
-        w = bottom_blob_bordered.w;
-        h = bottom_blob_bordered.h;
     }
+    if (bottom_blob_bordered.empty())
+        return -100;
+
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
 
     int outw = (w - kernel_extent_w) / stride_w + 1;
     int outh = (h - kernel_extent_h) / stride_h + 1;
 
     // int8
-    if (use_int8_inference)
+    size_t out_elemsize = use_int8_requantize ? 1u : 4u;
+
+    top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // depth-wise
+    if (channels == group && group == num_output)
     {
         if (use_int8_requantize)
         {
-            Mat top_blob_tm;
-            top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator);
-            if (top_blob_tm.empty())
-                return -100;
-            
-            top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100;
-
-            // depth-wise
-            if (channels == group && group == num_output)
-            {                
-                if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
-                {
-                    if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2))
-                    {
-                        if (stride_w == 1 && stride_h == 1)
-                        {
-                            convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);                          
-                        }
-                        else if (stride_w == 2 && stride_h == 2)
-                        {
-                            convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);                           
-                        }
-
-                        if (activation)
-                        {
-                            activation->forward_inplace(top_blob, opt);
-                        }
-
-                        return 0;
-                    }
-                }
-
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
-                {
-                    const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(g, 1);
-                    Mat top_blob_tm_g = top_blob_tm.channel_range(g, 1);
+            std::vector<float> requantize_scales;
+            for (int g=0; g<group; g++)
+            {
+                float scale_in;
+                if (weight_data_int8_scales[g] == 0)
+                    scale_in = 0;
+                else
+                    scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                    const ncnn::Layer* op = group_ops[g];
+                float scale_out = top_blob_int8_scale;
 
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                requantize_scales.push_back(scale_in);
+                requantize_scales.push_back(scale_out);
+            }
 
-                    // forward
-                    op->forward(bottom_blob_bordered_g, top_blob_tm_g, opt_g);
-                }
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
+            {
+                convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
 
                 if (activation)
                 {
                     activation->forward_inplace(top_blob, opt);
-                }                
+                }
 
                 return 0;
             }
-
-            const int channels_g = channels / group;
-            const int num_output_g = num_output / group;
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int g=0; g<group; g++)
+            if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
             {
-                const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
-                Mat top_blob_tm_g = top_blob_tm.channel_range(num_output_g * g, num_output_g);
-
-                const ncnn::Layer* op = group_ops[g];
+                convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt);
 
-                Option opt_g = opt;
-                opt_g.blob_allocator = top_blob.allocator;
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
 
-                // forward
-                op->forward(bottom_blob_bordered_g, top_blob_tm_g, opt_g);
-            }                 
+                return 0;
+            }
         }
         else
         {
-            top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator);
-            if (top_blob.empty())
-                return -100;
-
-            // depth-wise
-            if (channels == group && group == num_output)
-            {                
-                if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
-                {
-                    if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2))
-                    {
-                        if (stride_w == 1 && stride_h == 1)
-                        {
-                            convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
-                        }
-                        else if (stride_w == 2 && stride_h == 2)
-                        {
-                            convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);                          
-                        }
-
-                        if (activation)
-                        {
-                            activation->forward_inplace(top_blob, opt);
-                        }                        
-
-                        return 0;
-                    }
-                }
-
-                #pragma omp parallel for num_threads(opt.num_threads)
-                for (int g=0; g<group; g++)
-                {
-                    const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(g, 1);
-                    Mat top_blob_g = top_blob.channel_range(g, 1);
-
-                    const ncnn::Layer* op = group_ops[g];
+            std::vector<float> dequantize_scales;
+            for (int g=0; g<group; g++)
+            {
+                float top_rescale = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
 
-                    Option opt_g = opt;
-                    opt_g.num_threads = 1;
-                    opt_g.blob_allocator = top_blob.allocator;
+                dequantize_scales.push_back(top_rescale);
+            }
 
-                    // forward
-                    op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-                }
+            if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1)
+            {
+                convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
 
                 if (activation)
                 {
                     activation->forward_inplace(top_blob, opt);
-                }                
-              
+                }
+
                 return 0;
             }
-
-            const int channels_g = channels / group;
-            const int num_output_g = num_output / group;
-
-            #pragma omp parallel for num_threads(opt.num_threads)
-            for (int g=0; g<group; g++)
+            if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1)
             {
-                const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
-                Mat top_blob_g = top_blob.channel_range(num_output_g * g, num_output_g);
+                convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt);
 
-                const ncnn::Layer* op = group_ops[g];
-
-                Option opt_g = opt;
-                opt_g.blob_allocator = top_blob.allocator;
-
-                // forward
-                op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-            }                       
-        }
-
-        if (activation)
-        {
-            activation->forward_inplace(top_blob, opt);
-        }          
-
-        return 0;
-    }
-
-    // float32
-    top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator);
-    if (top_blob.empty())
-        return -100;
-    
-    // depth-wise
-    if (channels == group && group == num_output)
-    {
-        if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1)
-        {
-            if (stride_w == 1 && stride_h == 1)
-            {
-                convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
-            }
-            else if (stride_w == 2 && stride_h == 2)
-            {
-                convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt);
-            }
+                if (activation)
+                {
+                    activation->forward_inplace(top_blob, opt);
+                }
 
-            if (activation)
-            {
-                activation->forward_inplace(top_blob, opt);
+                return 0;
             }
-
-            return 0;
-        }
-
-        #pragma omp parallel for num_threads(opt.num_threads)
-        for (int g=0; g<group; g++)
-        {
-            const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(g, 1);
-            Mat top_blob_g = top_blob.channel_range(g, 1);
-
-            const ncnn::Layer* op = group_ops[g];
-
-            Option opt_g = opt;
-            opt_g.num_threads = 1;
-            opt_g.blob_allocator = top_blob.allocator;
-
-         // forward
-            op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
-        }
-
-        if (activation)
-        {
-            activation->forward_inplace(top_blob, opt);
         }
-
-        return 0;
     }
 
+    // group convolution
     const int channels_g = channels / group;
     const int num_output_g = num_output / group;
 
+    #pragma omp parallel for num_threads(opt.num_threads)
     for (int g=0; g<group; g++)
     {
         const Mat bottom_blob_bordered_g = bottom_blob_bordered.channel_range(channels_g * g, channels_g);
@@ -504,11 +467,6 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con
         op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
     }
 
-    if (activation)
-    {
-        activation->forward_inplace(top_blob, opt);
-    }
-
     return 0;
 }
 

src/layer/x86/convolutiondepthwise_x86.h

@@ -29,6 +29,9 @@ public:
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+protected:
+    int forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
 public:
     Layer* activation;
     std::vector<ncnn::Layer*> group_ops;

src/mat.cpp

@@ -24,66 +24,68 @@
 #include "layer_type.h"
 #include "layer.h"
 
+#if NCNN_VULKAN
 #if __ANDROID_API__ >= 26
 #include <android/hardware_buffer.h>
 #endif // __ANDROID_API__ >= 26
+#endif // NCNN_VULKAN
 
 namespace ncnn {
 
 void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_vals)
 {
-    ncnn::Layer* op;
+    Layer* op;
 
     if (mean_vals && !norm_vals)
     {
         // substract mean only
-        op = ncnn::create_layer(ncnn::LayerType::Bias);
+        op = create_layer(LayerType::Bias);
 
-        ncnn::ParamDict pd;
+        ParamDict pd;
         pd.set(0, c);
 
         op->load_param(pd);
 
-        ncnn::Mat weights[1];
+        Mat weights[1];
         weights[0] = Mat(c);
         for (int q=0; q<c; q++)
         {
             weights[0][q] = -mean_vals[q];
         }
 
-        op->load_model(ncnn::ModelBinFromMatArray(weights));
+        op->load_model(ModelBinFromMatArray(weights));
     }
     else if (!mean_vals && norm_vals)
     {
         // normalize only
-        op = ncnn::create_layer(ncnn::LayerType::Scale);
+        op = create_layer(LayerType::Scale);
 
-        ncnn::ParamDict pd;
+        ParamDict pd;
         pd.set(0, c);
 
         op->load_param(pd);
 
-        ncnn::Mat weights[1];
+        Mat weights[1];
         weights[0] = Mat(c);
         for (int q=0; q<c; q++)
         {
             weights[0][q] = norm_vals[q];
         }
 
-        op->load_model(ncnn::ModelBinFromMatArray(weights));
+        op->load_model(ModelBinFromMatArray(weights));
     }
     else if (mean_vals && norm_vals)
     {
         // substract mean and normalize
-        op = ncnn::create_layer(ncnn::LayerType::Scale);
+        op = create_layer(LayerType::Scale);
 
-        ncnn::ParamDict pd;
+        ParamDict pd;
         pd.set(0, c);
         pd.set(1, 1);
 
         op->load_param(pd);
 
-        ncnn::Mat weights[2];
+        Mat weights[2];
         weights[0] = Mat(c);
         weights[1] = Mat(c);
         for (int q=0; q<c; q++)
@@ -92,7 +94,7 @@ void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_val
             weights[1][q] = - mean_vals[q] * norm_vals[q];
         }
 
-        op->load_model(ncnn::ModelBinFromMatArray(weights));
+        op->load_model(ModelBinFromMatArray(weights));
     }
     else // if (!mean_vals && !norm_vals)
     {
@@ -101,8 +103,13 @@ void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_val
 
     Option opt;
     opt.num_threads = 1;// TODO
+
+    op->create_pipeline(opt);
+
     op->forward_inplace(*this, opt);
 
+    op->destroy_pipeline(opt);
+
     delete op;
 }
 
@@ -226,6 +233,7 @@ Mat Mat::from_float16(const unsigned short* data, int size)
     return m;
 }
 
+#if NCNN_VULKAN
 #if __ANDROID_API__ >= 26
 VkImageMat VkImageMat::from_android_hardware_buffer(AHardwareBuffer* hb, VkAndroidHardwareBufferImageAllocator* allocator)
 {
@@ -248,12 +256,13 @@ VkImageMat VkImageMat::from_android_hardware_buffer(AHardwareBuffer* hb, VkAndro
     return m;
 }
 #endif // __ANDROID_API__ >= 26
+#endif // NCNN_VULKAN
 
 void copy_make_border(const Mat& src, Mat& dst, int top, int bottom, int left, int right, int type, float v, const Option& opt)
 {
-    ncnn::Layer* padding = ncnn::create_layer(ncnn::LayerType::Padding);
+    Layer* padding = create_layer(LayerType::Padding);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, top);
     pd.set(1, bottom);
     pd.set(2, left);
@@ -263,16 +272,20 @@ void copy_make_border(const Mat& src, Mat& dst, int top, int bottom, int left, i
 
     padding->load_param(pd);
 
+    padding->create_pipeline(opt);
+
     padding->forward(src, dst, opt);
 
+    padding->destroy_pipeline(opt);
+
     delete padding;
 }
 
 void copy_cut_border(const Mat& src, Mat& dst, int top, int bottom, int left, int right, const Option& opt)
 {
-    ncnn::Layer* crop = ncnn::create_layer(ncnn::LayerType::Crop);
+    Layer* crop = create_layer(LayerType::Crop);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, left);
     pd.set(1, top);
     pd.set(2, 0);
@@ -282,85 +295,179 @@ void copy_cut_border(const Mat& src, Mat& dst, int top, int bottom, int left, in
 
     crop->load_param(pd);
 
+    crop->create_pipeline(opt);
+
     crop->forward(src, dst, opt);
 
+    crop->destroy_pipeline(opt);
+
     delete crop;
 }
 
 void resize_bilinear(const Mat& src, Mat& dst, int w, int h, const Option& opt)
 {
-    ncnn::Layer* interp = ncnn::create_layer(ncnn::LayerType::Interp);
+    Layer* interp = create_layer(LayerType::Interp);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, 2);
     pd.set(3, h);
     pd.set(4, w);
 
     interp->load_param(pd);
 
+    interp->create_pipeline(opt);
+
     interp->forward(src, dst, opt);
 
+    interp->destroy_pipeline(opt);
+
     delete interp;
 }
 
 void resize_bicubic(const Mat& src, Mat& dst, int w, int h, const Option& opt)
 {
-    ncnn::Layer* interp = ncnn::create_layer(ncnn::LayerType::Interp);
+    Layer* interp = create_layer(LayerType::Interp);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, 3);
     pd.set(3, h);
     pd.set(4, w);
 
     interp->load_param(pd);
 
+    interp->create_pipeline(opt);
+
     interp->forward(src, dst, opt);
 
+    interp->destroy_pipeline(opt);
+
     delete interp;
 }
 
 void convert_packing(const Mat& src, Mat& dst, int _elempack, const Option& opt)
 {
-    ncnn::Layer* packing = ncnn::create_layer(ncnn::LayerType::Packing);
+    Layer* packing = create_layer(LayerType::Packing);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, _elempack);
 
     packing->load_param(pd);
 
+    packing->create_pipeline(opt);
+
     packing->forward(src, dst, opt);
 
+    packing->destroy_pipeline(opt);
+
     delete packing;
 }
 
 void cast_float32_to_float16(const Mat& src, Mat& dst, const Option& opt)
 {
-    ncnn::Layer* cast = ncnn::create_layer(ncnn::LayerType::Cast);
+    Layer* cast = create_layer(LayerType::Cast);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, 1);
     pd.set(1, 2);
 
     cast->load_param(pd);
 
+    cast->create_pipeline(opt);
+
     cast->forward(src, dst, opt);
 
+    cast->destroy_pipeline(opt);
+
     delete cast;
 }
 
 void cast_float16_to_float32(const Mat& src, Mat& dst, const Option& opt)
 {
-    ncnn::Layer* cast = ncnn::create_layer(ncnn::LayerType::Cast);
+    Layer* cast = create_layer(LayerType::Cast);
 
-    ncnn::ParamDict pd;
+    ParamDict pd;
     pd.set(0, 2);
     pd.set(1, 1);
 
     cast->load_param(pd);
 
+    cast->create_pipeline(opt);
+
     cast->forward(src, dst, opt);
 
+    cast->destroy_pipeline(opt);
+
     delete cast;
 }
 
+void quantize_float32_to_int8(const Mat& src, Mat& dst, float scale, const Option& opt)
+{
+    Layer* quantize = create_layer(LayerType::Quantize);
+
+    ParamDict pd;
+    pd.set(0, scale);
+
+    quantize->load_param(pd);
+
+    quantize->create_pipeline(opt);
+
+    quantize->forward(src, dst, opt);
+
+    quantize->destroy_pipeline(opt);
+
+    delete quantize;
+}
+
+void dequantize_int32_to_float32(Mat& m, float scale, const float* bias, int bias_data_size, const Option& opt)
+{
+    Layer* dequantize = create_layer(LayerType::Dequantize);
+
+    ParamDict pd;
+    pd.set(0, scale);
+    pd.set(1, bias ? 1 : 0);
+    pd.set(2, bias_data_size);
+
+    dequantize->load_param(pd);
+
+    Mat weights[1];
+    weights[0] = Mat(bias_data_size, (void*)bias);
+
+    dequantize->load_model(ModelBinFromMatArray(weights));
+
+    dequantize->create_pipeline(opt);
+
+    dequantize->forward_inplace(m, opt);
+
+    dequantize->destroy_pipeline(opt);
+
+    delete dequantize;
+}
+
+void requantize_int8_to_int8(const Mat& src, Mat& dst, float scale_in, float scale_out, const float* bias, int bias_data_size, int fusion_relu, const Option& opt)
+{
+    Layer* requantize = create_layer(LayerType::Requantize);
+
+    ParamDict pd;
+    pd.set(0, scale_in);
+    pd.set(1, scale_out);
+    pd.set(2, bias ? 1 : 0);
+    pd.set(3, bias_data_size);
+    pd.set(4, fusion_relu);
+
+    requantize->load_param(pd);
+
+    Mat weights[1];
+    weights[0] = Mat(bias_data_size, (void*)bias);
+
+    requantize->load_model(ModelBinFromMatArray(weights));
+
+    requantize->create_pipeline(opt);
+
+    requantize->forward(src, dst, opt);
+
+    requantize->destroy_pipeline(opt);
+
+    delete requantize;
+}
+
 } // namespace ncnn

src/mat.h

@@ -483,6 +483,9 @@ void resize_bicubic(const Mat& src, Mat& dst, int w, int h, const Option& opt =
 void convert_packing(const Mat& src, Mat& dst, int elempack, const Option& opt = Option());
 void cast_float32_to_float16(const Mat& src, Mat& dst, const Option& opt = Option());
 void cast_float16_to_float32(const Mat& src, Mat& dst, const Option& opt = Option());
+void quantize_float32_to_int8(const Mat& src, Mat& dst, float scale, const Option& opt = Option());
+void dequantize_int32_to_float32(Mat& m, float scale, const float* bias, int bias_data_size, const Option& opt = Option());
+void requantize_int8_to_int8(const Mat& src, Mat& dst, float scale_in, float scale_out, const float* bias, int bias_data_size, int fusion_relu, const Option& opt = Option());
 
 inline Mat::Mat()
     : data(0), refcount(0), elemsize(0), elempack(0), allocator(0), dims(0), w(0), h(0), c(0), cstep(0)

src/net.cpp

@@ -661,25 +661,21 @@ int Net::fuse_network()
                     {
                         ((Convolution*)layer)->use_int8_requantize = true;
                         ((Convolution*)layer)->top_blob_int8_scale = ((Convolution*)layer_next)->bottom_blob_int8_scale;
-                        ((Convolution*)layer)->create_requantize_op();
                     }
                     else if (layer->type == "ConvolutionDepthWise" && layer_next->type == "Convolution")
                     {
                         ((ConvolutionDepthWise*)layer)->use_int8_requantize = true;
                         ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((Convolution*)layer_next)->bottom_blob_int8_scale;
-                        ((ConvolutionDepthWise*)layer)->create_requantize_op();
                     }
                     else if (layer->type == "Convolution" && layer_next->type == "ConvolutionDepthWise")
                     {
                         ((Convolution*)layer)->use_int8_requantize = true;
                         ((Convolution*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next)->bottom_blob_int8_scales[0];
-                        ((Convolution*)layer)->create_requantize_op();
                     }
                     else
                     {
                         ((ConvolutionDepthWise*)layer)->use_int8_requantize = true;
                         ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next)->bottom_blob_int8_scales[0];
-                        ((ConvolutionDepthWise*)layer)->create_requantize_op();
                     }
                 }                  
                 else if (layer_next->type == "ReLU")
@@ -699,25 +695,21 @@ int Net::fuse_network()
                         {
                             ((Convolution*)layer)->use_int8_requantize = true;
                             ((Convolution*)layer)->top_blob_int8_scale = ((Convolution*)layer_next_2)->bottom_blob_int8_scale;
-                            ((Convolution*)layer)->create_requantize_op();
                         }
                         else if (layer->type == "ConvolutionDepthWise" && layer_next_2->type == "Convolution")
                         {
                             ((ConvolutionDepthWise*)layer)->use_int8_requantize = true;
                             ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((Convolution*)layer_next_2)->bottom_blob_int8_scale;
-                            ((ConvolutionDepthWise*)layer)->create_requantize_op();
                         }
                         else if (layer->type == "Convolution" && layer_next_2->type == "ConvolutionDepthWise")
                         {
                             ((Convolution*)layer)->use_int8_requantize = true;
                             ((Convolution*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next_2)->bottom_blob_int8_scales[0];
-                            ((Convolution*)layer)->create_requantize_op();
                         }
                         else
                         {
                             ((ConvolutionDepthWise*)layer)->use_int8_requantize = true;
                             ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next_2)->bottom_blob_int8_scales[0];
-                            ((ConvolutionDepthWise*)layer)->create_requantize_op();
                         }
                     }
                     else if (layer_next_2->type == "Split")
@@ -749,7 +741,6 @@ int Net::fuse_network()
                             }
 
                             ((Convolution*)layer)->use_int8_requantize = true;
-                            ((Convolution*)layer)->create_requantize_op();    
                             // fprintf(stderr, "\n");
                         }
                     }

tests/test_convolution.cpp

@@ -29,13 +29,16 @@ static int test_convolution(int w, int h, int c, int outch, int kernel, int dila
     pd.set(5, bias);// bias_term
     pd.set(6, outch*c*kernel*kernel);
 
-    std::vector<ncnn::Mat> weights(2);
+    std::vector<ncnn::Mat> weights(bias ? 2 : 1);
     weights[0] = RandomMat(outch*c*kernel*kernel);
-    weights[1] = RandomMat(outch);
+    if (bias)
+        weights[1] = RandomMat(outch);
     ncnn::ModelBinFromMatArray mb(weights.data());
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.use_vulkan_compute = true;
+    opt.use_int8_inference = false;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
     opt.use_fp16_arithmetic = false;
@@ -111,9 +114,117 @@ static int test_convolution_0()
     return 0;
 }
 
+static int test_convolution_int8(int w, int h, int c, int outch, int kernel, int dilation, int stride, int pad, int bias)
+{
+    ncnn::Mat a = RandomMat(w, h, c);
+
+    ncnn::ParamDict pd;
+    pd.set(0, outch);// num_output
+    pd.set(1, kernel);// kernel_w
+    pd.set(2, dilation);// dilation_w
+    pd.set(3, stride);// stride_w
+    pd.set(4, pad);// pad_w
+    pd.set(5, bias);// bias_term
+    pd.set(6, outch*c*kernel*kernel);
+    pd.set(8, 1);// int8_scale_term
+
+    std::vector<ncnn::Mat> weights(bias ? 4 : 3);
+    weights[0] = RandomMat(outch*c*kernel*kernel);
+    if (bias)
+    {
+        weights[1] = RandomMat(outch);
+        weights[2] = RandomMat(outch);
+        weights[3] = RandomMat(1);
+    }
+    else
+    {
+        weights[1] = RandomMat(outch);
+        weights[2] = RandomMat(1);
+    }
+    ncnn::ModelBinFromMatArray mb(weights.data());
+
+    ncnn::Option opt;
+    opt.num_threads = 1;
+    opt.use_vulkan_compute = false;
+    opt.use_int8_inference = true;
+    opt.use_fp16_packed = false;
+    opt.use_fp16_storage = false;
+    opt.use_fp16_arithmetic = false;
+    opt.use_int8_storage = false;
+    opt.use_int8_arithmetic = false;
+    opt.use_packing_layout = false;
+
+    int ret = test_layer<ncnn::Convolution>("Convolution", pd, mb, opt, a);
+    if (ret != 0)
+    {
+        fprintf(stderr, "test_convolution_int8 failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias);
+    }
+
+    return 0;
+}
+
+static int test_convolution_1()
+{
+    static const int kdsp[24][4] = {
+        {1, 1, 1, 0},
+        {1, 1, 2, 0},
+        {2, 1, 1, 1},
+        {2, 1, 2, 1},
+        {2, 2, 1, 1},
+        {2, 2, 2, 1},
+        {3, 1, 1, 1},
+        {3, 1, 2, 1},
+        {3, 2, 1, 1},
+        {3, 2, 2, 1},
+        {4, 1, 1, 2},
+        {4, 1, 2, 2},
+        {4, 2, 1, 2},
+        {4, 2, 2, 2},
+        {5, 1, 1, 2},
+        {5, 1, 2, 2},
+        {5, 2, 1, 2},
+        {5, 2, 2, 2},
+        {7, 1, 1, 3},
+        {7, 1, 2, 3},
+        {7, 1, 3, 3},
+        {7, 2, 1, 3},
+        {7, 2, 2, 3},
+        {7, 2, 3, 3},
+    };
+
+    for (int i=0; i<24; i++)
+    {
+        int ret = 0
+        || test_convolution_int8(13, 11, 1, 1, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 2, 2, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 3, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 4, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 7, 7, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 8, 8, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 15, 15, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 16, 16, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+
+        || test_convolution_int8(13, 11, 1, 1, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 2, 2, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 3, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 3, 12, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 4, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 8, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 8, 8, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 16, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        || test_convolution_int8(13, 11, 16, 16, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1)
+        ;
+
+        if (ret != 0)
+            return -1;
+    }
+
+    return 0;
+}
+
 int main()
 {
     SRAND(7767517);
 
-    return test_convolution_0();
+    return test_convolution_0() || test_convolution_1();
 }

tests/test_convolutiondepthwise.cpp

@@ -37,6 +37,8 @@ static int test_convolutiondepthwise(int w, int h, int c, int outch, int kernel,
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.use_vulkan_compute = true;
+    opt.use_int8_inference = false;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
     opt.use_fp16_arithmetic = false;

tests/test_deconvolution.cpp

@@ -36,6 +36,7 @@ static int test_deconvolution(int w, int h, int c, int outch, int kernel, int di
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.use_vulkan_compute = true;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
     opt.use_fp16_arithmetic = false;

tests/test_deconvolutiondepthwise.cpp

@@ -37,6 +37,7 @@ static int test_deconvolutiondepthwise(int w, int h, int c, int outch, int kerne
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.use_vulkan_compute = true;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
     opt.use_fp16_arithmetic = false;

tests/test_relu.cpp

@@ -28,6 +28,7 @@ static int test_relu(float slope, bool use_packing_layout)
 
     ncnn::Option opt;
     opt.num_threads = 1;
+    opt.use_vulkan_compute = true;
     opt.use_fp16_packed = false;
     opt.use_fp16_storage = false;
     opt.use_fp16_arithmetic = false;

tests/testutil.h

@@ -168,7 +168,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     ncnn::VkBlobBufferAllocator g_blob_vkallocator(vkdev);
     ncnn::VkStagingBufferAllocator g_staging_vkallocator(vkdev);
 
-    opt.use_vulkan_compute = true;
     opt.blob_vkallocator = &g_blob_vkallocator;
     opt.workspace_vkallocator = &g_blob_vkallocator;
     opt.staging_vkallocator = &g_staging_vkallocator;
@@ -193,6 +192,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     op->create_pipeline(opt);
 
 #if NCNN_VULKAN
+    if (opt.use_vulkan_compute)
     {
         ncnn::VkTransfer cmd(vkdev);
         cmd.weight_vkallocator = &g_weight_vkallocator;
@@ -240,6 +240,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
 
 #if NCNN_VULKAN
     std::vector<ncnn::Mat> d(top_blob_count);
+    if (opt.use_vulkan_compute)
     {
         // pack
         std::vector<ncnn::Mat> a4(a.size());
@@ -335,7 +336,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     }
 
 #if NCNN_VULKAN
-    if (CompareMat(b, d, epsilon) != 0)
+    if (opt.use_vulkan_compute && CompareMat(b, d, epsilon) != 0)
     {
         fprintf(stderr, "test_layer failed gpu\n");
         return -1;
@@ -362,7 +363,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     ncnn::VkBlobBufferAllocator g_blob_vkallocator(vkdev);
     ncnn::VkStagingBufferAllocator g_staging_vkallocator(vkdev);
 
-    opt.use_vulkan_compute = true;
     opt.blob_vkallocator = &g_blob_vkallocator;
     opt.workspace_vkallocator = &g_blob_vkallocator;
     opt.staging_vkallocator = &g_staging_vkallocator;
@@ -380,6 +380,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     op->create_pipeline(opt);
 
 #if NCNN_VULKAN
+    if (opt.use_vulkan_compute)
     {
         ncnn::VkTransfer cmd(vkdev);
         cmd.weight_vkallocator = &g_weight_vkallocator;
@@ -423,6 +424,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
 
 #if NCNN_VULKAN
     ncnn::Mat d;
+    if (opt.use_vulkan_compute)
     {
         // pack
         ncnn::Mat a4;
@@ -497,7 +499,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m
     }
 
 #if NCNN_VULKAN
-    if (CompareMat(b, d, epsilon) != 0)
+    if (opt.use_vulkan_compute && CompareMat(b, d, epsilon) != 0)
     {
         fprintf(stderr, "test_layer failed gpu\n");
         return -1;