diff --git a/benchmark/benchncnn.cpp b/benchmark/benchncnn.cpp index 4065532b4..8f7a02651 100644 --- a/benchmark/benchncnn.cpp +++ b/benchmark/benchncnn.cpp @@ -221,9 +221,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("squeezenet_int8", ncnn::Mat(227, 227, 3), opt); - opt.use_packing_layout = true; } benchmark("mobilenet", ncnn::Mat(224, 224, 3), opt); @@ -232,9 +230,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("mobilenet_int8", ncnn::Mat(224, 224, 3), opt); - opt.use_packing_layout = true; } benchmark("mobilenet_v2", ncnn::Mat(224, 224, 3), opt); @@ -260,9 +256,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("googlenet_int8", ncnn::Mat(224, 224, 3), opt); - opt.use_packing_layout = true; } benchmark("resnet18", ncnn::Mat(224, 224, 3), opt); @@ -271,9 +265,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("resnet18_int8", ncnn::Mat(224, 224, 3), opt); - opt.use_packing_layout = true; } benchmark("alexnet", ncnn::Mat(227, 227, 3), opt); @@ -284,9 +276,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("vgg16_int8", ncnn::Mat(224, 224, 3), opt); - opt.use_packing_layout = true; } benchmark("resnet50", ncnn::Mat(224, 224, 3), opt); @@ -295,9 +285,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("resnet50_int8", ncnn::Mat(224, 224, 3), opt); - opt.use_packing_layout = true; } benchmark("squeezenet_ssd", ncnn::Mat(300, 300, 3), opt); @@ -306,9 +294,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("squeezenet_ssd_int8", ncnn::Mat(300, 300, 3), opt); - opt.use_packing_layout = true; } benchmark("mobilenet_ssd", ncnn::Mat(300, 300, 3), opt); @@ -317,9 +303,7 @@ int main(int argc, char** argv) if (!use_vulkan_compute) #endif // NCNN_VULKAN { - opt.use_packing_layout = false; benchmark("mobilenet_ssd_int8", ncnn::Mat(300, 300, 3), opt); - opt.use_packing_layout = true; } benchmark("mobilenet_yolo", ncnn::Mat(416, 416, 3), opt); diff --git a/src/layer/arm/convolution_1x1_int8.h b/src/layer/arm/convolution_1x1_int8.h index 697b1f683..bd60566b9 100644 --- a/src/layer/arm/convolution_1x1_int8.h +++ b/src/layer/arm/convolution_1x1_int8.h @@ -15,7 +15,16 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. +static inline signed char float2int8(float v) +{ + int int32 = round(v); + if (int32 > 127) return 127; + if (int32 < -127) return -127; + return (signed char)int32; +} + #if __aarch64__ +#if 0// FIXME chgemm produce wrong result static void conv1x1s1_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch) { kernel_tm.create(outch, inch, (size_t)1u); @@ -24,80 +33,1445 @@ static void conv1x1s1_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& reorder_a((int8_t*)a, sa, outch, inch, inch); } -static void conv1x1s1_sgemm_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Option& opt) -{ - const size_t n = bottom_blob.w * bottom_blob.h; - const size_t k = bottom_blob.c; - const size_t m = top_blob.c; +static void conv1x1s1_sgemm_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Option& opt) +{ + const size_t n = bottom_blob.w * bottom_blob.h; + const size_t k = bottom_blob.c; + const size_t m = top_blob.c; + + ncnn::Mat bottom_tm(k * n, (size_t)1u, opt.workspace_allocator); + { + const int8_t* pData = bottom_blob; + int8_t *pReorder = bottom_tm; + reorder_b(pData, pReorder, k, n, bottom_blob.cstep); + } + + // GEMM + int32_t *pc = top_blob; + const int8_t *pa = kernel; + const int8_t *pb = bottom_tm; + const size_t ldc = top_blob.cstep; + + int8kernel((void*)pc, pa, pb, m, k, n, ldc, nullptr, nullptr, opt); +} + + +static void conv1x1s1_sgemm_int8_requant_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) +{ + const size_t n = bottom_blob.w * bottom_blob.h; + const size_t k = bottom_blob.c; + const size_t m = top_blob.c; + + ncnn::Mat scales_tm(m); + ncnn::Mat bias_tm(m); + float* scales = scales_tm; + const float* bias = _bias; + + // outptr0[0] = float2int8(((float)sum0 * scale_requant_in + bias0) * scale_requant_out); + // the equation could convert to: + // out = float2int8( (float)sum * (scale_requant_in * scale_requant_out) + (bias * scale_requant_out) ) + // prebuild the list of (scales_requant_in*scale_requant_out) + for (size_t i = 0; i < m; ++i) + { + scales_tm[i] = scales_requant[2*i] * scales_requant[2*i + 1]; + } + if (!_bias.empty()) + { + for (size_t i = 0; i < m; ++i) + { + bias_tm[i] = bias[i] * scales_requant[2*i + 1]; + } + bias = bias_tm; + } + + ncnn::Mat bottom_tm(k * n, (size_t)1u, opt.workspace_allocator); + { + const int8_t *pData = bottom_blob; + int8_t *pReorder = bottom_tm; + reorder_b(pData, pReorder, k, n, bottom_blob.cstep); + } + + // GEMM + int8_t *pc = top_blob; + const int8_t *pa = kernel; + const int8_t *pb = bottom_tm; + const size_t ldc = top_blob.cstep; + int8kernel((void*)pc, pa, pb, m, k, n, ldc, scales, (float*)bias, opt); +} +#else +static void conv1x1s1_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch) +{ + const signed char* kernel = _kernel; + + // kernel memory packed 4 x 4 + kernel_tm.create(4*4, inch/4 + inch%4, outch/4 + outch%4, (size_t)1u); + + int nn_outch = 0; + int remain_outch_start = 0; + + nn_outch = outch >> 2; + remain_outch_start = nn_outch << 2; + + for (int pp=0; pp> 2; + int remain_size_start = nn_size << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii=0; ii> 2; + remain_outch_start = nn_outch << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp=0; pp> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n"// for (; k+3> 1 + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n"// for (; k+1 scales_requant, const Option& opt) +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int inch = bottom_blob.c; + int outch = top_blob.c; + + const int size = w * h; + const float* bias = _bias; + + // bottom_tm memory packed 4 x 4 + ncnn::Mat bottom_tm(4, inch, size/4 + size%4, (size_t)1u, opt.workspace_allocator); + { + int nn_size = size >> 2; + int remain_size_start = nn_size << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii=0; ii> 2; + remain_outch_start = nn_outch << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp=0; pp> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n"// for (; k+3> 1 + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n"// for (; k+1 top_f32 + "scvtf v20.4s, v20.4s \n" + "scvtf v21.4s, v21.4s \n" + "scvtf v22.4s, v22.4s \n" + "scvtf v23.4s, v23.4s \n" + // top_f32 = top_f32 * scale_in + "fmul v20.4s, v20.4s, %17.s[0] \n" + "fmul v21.4s, v21.4s, %17.s[1] \n" + "fmul v22.4s, v22.4s, %17.s[2] \n" + "fmul v23.4s, v23.4s, %17.s[3] \n" + // top_f32 = top_f32 + bias + "fadd v20.4s, v20.4s, %13.4s \n" + "fadd v21.4s, v21.4s, %14.4s \n" + "fadd v22.4s, v22.4s, %15.4s \n" + "fadd v23.4s, v23.4s, %16.4s \n" + // top_f32 = top_f32 * scale_out + "fmul v20.4s, v20.4s, %18.s[0] \n" + "fmul v21.4s, v21.4s, %18.s[1] \n" + "fmul v22.4s, v22.4s, %18.s[2] \n" + "fmul v23.4s, v23.4s, %18.s[3] \n" + // top_f32 -> top_s32 + "fcvtas v20.4s, v20.4s \n" + "fcvtas v21.4s, v21.4s \n" + "fcvtas v22.4s, v22.4s \n" + "fcvtas v23.4s, v23.4s \n" + // top_s32 -> top_s16 + "sqxtn v7.4h, v20.4s \n" + "sqxtn2 v7.8h, v21.4s \n" + "sqxtn v8.4h, v22.4s \n" + "sqxtn2 v8.8h, v23.4s \n" + // top_s16 -> top_s8 + "sqxtn v0.8b, v7.8h \n" + "sqxtn v1.8b, v8.8h \n" + // save top_s8 + "st1 {v0.s}[0], [%0] \n" + "st1 {v0.s}[1], [%1] \n" + "st1 {v1.s}[0], [%2] \n" + "st1 {v1.s}[1], [%3] \n" + + : "=r"(outptr0), // %0 + "=r"(outptr1), // %1 + "=r"(outptr2), // %2 + "=r"(outptr3), // %3 + "=r"(tmpptr), // %4 + "=r"(kptr) // %5 + : "0"(outptr0), + "1"(outptr1), + "2"(outptr2), + "3"(outptr3), + "4"(tmpptr), + "5"(kptr), + "r"(inch), // %12 + "w"(_bias0), // %13 + "w"(_bias1), // %14 + "w"(_bias2), // %15 + "w"(_bias3), // %16 + "w"(_scale_in03), // %17 + "w"(_scale_out03) // %18 + : "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + ); +#else + int sum0_0 = 0; + int sum0_1 = 0; + int sum0_2 = 0; + int sum0_3 = 0; + + int sum1_0 = 0; + int sum1_1 = 0; + int sum1_2 = 0; + int sum1_3 = 0; - ncnn::Mat bottom_tm(k * n, (size_t)1u, opt.workspace_allocator); - { - const int8_t* pData = bottom_blob; - int8_t *pReorder = bottom_tm; - reorder_b(pData, pReorder, k, n, bottom_blob.cstep); - } - - // GEMM - int32_t *pc = top_blob; - const int8_t *pa = kernel; - const int8_t *pb = bottom_tm; - const size_t ldc = top_blob.cstep; + int sum2_0 = 0; + int sum2_1 = 0; + int sum2_2 = 0; + int sum2_3 = 0; - int8kernel((void*)pc, pa, pb, m, k, n, ldc, nullptr, nullptr, opt); -} + int sum3_0 = 0; + int sum3_1 = 0; + int sum3_2 = 0; + int sum3_3 = 0; + int q=0; + for (; q+1 scales_requant, const Option& opt) -{ - const size_t n = bottom_blob.w * bottom_blob.h; - const size_t k = bottom_blob.c; - const size_t m = top_blob.c; + tmpptr += 8; + kptr += 8; + } - ncnn::Mat scales_tm(m); - ncnn::Mat bias_tm(m); - float* scales = scales_tm; - const float* bias = _bias; + for (; q top_f32 + float32x4_t _sum_f32 = vcvtq_f32_s32(_sum); + // top_f32 = top_f32 * scale_in + _sum_f32 = vmulq_f32(_sum_f32, _scale_in03); + // top_f32 = top_f32 + bias + _sum_f32 = vaddq_f32(_sum_f32, _bias03); + // top_f32 = top_f32 * scale_out + _sum_f32 = vmulq_f32(_sum_f32, _scale_out03); + // top_f32 -> top_s32 + _sum = vcvtaq_s32_f32(_sum_f32); + // top_s32 -> top_s16 + int16x4_t _sum_s16 = vqmovn_s32(_sum); + int16x8_t _sum_s16_tp = vcombine_s16(_sum_s16, _sum_s16); + // top_s16 -> top_s8 + int8x8_t _sum_s8 = vqmovn_s16(_sum_s16_tp); + // save top_s8 + + vst1_lane_s8(outptr0, _sum_s8, 0); + vst1_lane_s8(outptr1, _sum_s8, 1); + vst1_lane_s8(outptr2, _sum_s8, 2); + vst1_lane_s8(outptr3, _sum_s8, 3); +#else + int sum0 = 0; + int sum1 = 0; + int sum2 = 0; + int sum3 = 0; + + int q=0; + for (; q+1 top_f32 + float32x4_t _sum_f32 = vcvtq_f32_s32(_sum); + // top_f32 = top_f32 * scale_in + _sum_f32 = vmulq_f32(_sum_f32, _scale_in); + // top_f32 = top_f32 + bias + _sum_f32 = vaddq_f32(_sum_f32, _bias0); + // top_f32 = top_f32 * scale_out + _sum_f32 = vmulq_f32(_sum_f32, _scale_out); + // top_f32 -> top_s32 + _sum = vcvtaq_s32_f32(_sum_f32); + // top_s32 -> top_s16 + int16x4_t _sum_s16 = vqmovn_s32(_sum); + int16x8_t _sum_s16_tp = vcombine_s16(_sum_s16, _sum_s16); + // top_s16 -> top_s8 + int8x8_t _sum_s8 = vqmovn_s16(_sum_s16_tp); + // save top_s8 + + vst1_s8(outptr0, _sum_s8); +#else + int sum0 = 0; + int sum1 = 0; + int sum2 = 0; + int sum3 = 0; + + int q=0; + for (; q+1 127) return 127; - if (int32 < -127) return -127; - return (signed char)int32; -} static void conv1x1s1_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch) { @@ -2204,25 +3578,3 @@ static void conv1x1s1_sgemm_int8_requant_neon(const Mat &bottom_blob, Mat &top_b } } #endif - -static void conv1x1s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv1x1s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} diff --git a/src/layer/arm/convolution_3x3_int8.h b/src/layer/arm/convolution_3x3_int8.h index 64c61ff43..a88c91264 100644 --- a/src/layer/arm/convolution_3x3_int8.h +++ b/src/layer/arm/convolution_3x3_int8.h @@ -4460,25 +4460,3 @@ static void conv3x3s2_packed_int8_neon(const Mat& bottom_blob, Mat& top_blob, co } } } - -static void conv3x3s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv3x3s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} diff --git a/src/layer/arm/convolution_5x5_int8.h b/src/layer/arm/convolution_5x5_int8.h deleted file mode 100644 index acca9f73e..000000000 --- a/src/layer/arm/convolution_5x5_int8.h +++ /dev/null @@ -1,35 +0,0 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. -// -// Copyright (C) 2019 BUG1989. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -static void conv5x5s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv5x5s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} diff --git a/src/layer/arm/convolution_7x7_int8.h b/src/layer/arm/convolution_7x7_int8.h deleted file mode 100644 index dbe937154..000000000 --- a/src/layer/arm/convolution_7x7_int8.h +++ /dev/null @@ -1,35 +0,0 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. -// -// Copyright (C) 2019 BUG1989. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -static void conv7x7s1_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv7x7s2_int8_neon(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_neon(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} diff --git a/src/layer/arm/convolution_arm.cpp b/src/layer/arm/convolution_arm.cpp index b5f84ec9b..3882cea3e 100644 --- a/src/layer/arm/convolution_arm.cpp +++ b/src/layer/arm/convolution_arm.cpp @@ -35,8 +35,6 @@ namespace ncnn { #include "convolution_sgemm_int8.h" #include "convolution_1x1_int8.h" #include "convolution_3x3_int8.h" -#include "convolution_5x5_int8.h" -#include "convolution_7x7_int8.h" #if __ARM_NEON #include "convolution_1x1_pack4.h" @@ -100,38 +98,14 @@ int Convolution_arm::create_pipeline(const Option& opt) activation->create_pipeline(opt); } - const int maxk = kernel_w * kernel_h; - const int num_input = weight_data_size / maxk / num_output; - - if (use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { support_packing = false; - if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) - { - use_winograd3x3 = true; - // conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_int8_data, num_input, num_output); - conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_int8_data, num_input, num_output); - } - - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) - { - conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_int8_data, num_input, num_output); - } - else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) - { - use_sgemm1x1 = true; - conv1x1s1_sgemm_transform_kernel_int8_neon(weight_data, weight_1x1s1_sgemm_int8_data, num_input, num_output); - } - else - { - conv_im2col_sgemm_transform_kernel_int8_neon(weight_data, weight_sgemm_int8_data, num_input, num_output, maxk); - } - - return 0; + return create_pipeline_int8_arm(opt); } - if (opt.use_packing_layout == false && kernel_w == kernel_h && stride_w == 1 && stride_h == 1 && dilation_w != 1 && dilation_h == dilation_w) + if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1) { convolution_dilation1 = ncnn::create_layer(ncnn::LayerType::Convolution); @@ -173,6 +147,9 @@ int Convolution_arm::create_pipeline(const Option& opt) return 0; } + const int maxk = kernel_w * kernel_h; + const int num_input = weight_data_size / maxk / num_output; + int elempack = (opt.use_packing_layout && num_input % 4 == 0) ? 4 : 1; int out_elempack = (opt.use_packing_layout && num_output % 4 == 0) ? 4 : 1; @@ -180,15 +157,15 @@ int Convolution_arm::create_pipeline(const Option& opt) // pack4 if (elempack == 4 && out_elempack == 4) { - if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv1x1s1_sgemm_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_winograd64_transform_kernel_pack4_neon(weight_data, weight_data_pack4, num_input, num_output); } @@ -307,15 +284,15 @@ int Convolution_arm::create_pipeline(const Option& opt) // pack4to1 if (elempack == 4 && out_elempack == 1) { - if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output); } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv1x1s1_sgemm_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output); } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_winograd64_transform_kernel_pack4to1_neon(weight_data, weight_data_pack4to1, num_input, num_output); } @@ -418,12 +395,12 @@ int Convolution_arm::create_pipeline(const Option& opt) conv3x3s2_transform_kernel_neon(weight_data, weight_3x3s2_data, num_input, num_output); } - if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk); } - if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv_im2col_sgemm_transform_kernel_neon(weight_data, weight_sgemm_data, num_input, num_output, maxk); } @@ -451,90 +428,6 @@ int Convolution_arm::destroy_pipeline(const Option& opt) return 0; } -int Convolution_arm::forwardDilation(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - size_t elemsize = bottom_blob.elemsize; - - const int kernel_size = kernel_w; - const int stride = stride_w; - const int dilation = dilation_w; - const int kernel_extent = dilation * (kernel_size - 1) + 1; - - int outw = (w - kernel_extent) / stride + 1; - int outh = (h - kernel_extent) / stride + 1; - - top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // Make (dilation * dilation) batches - Mat inner_bottom_blob; - Mat inner_top_blob; - for (int x = 0; x < dilation; x ++) - { - for (int y = 0; y < dilation; y ++) - { - int inner_w = (w - y + dilation - 1) / dilation; - int inner_h = (h - x + dilation - 1) / dilation; - - int inner_outw = (inner_w - kernel_size) / stride + 1; - int inner_outh = (inner_h - kernel_size) / stride + 1; - - inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator); - if (inner_bottom_blob.empty()) - return -100; - - inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator); - if (inner_top_blob.empty()) - return -100; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int c = 0; c < bottom_blob.c; c ++) - { - float *outptr = inner_bottom_blob.channel(c); - - for (int i = 0; i < inner_h; i ++) - { - const float *ptr = (const float *) bottom_blob.channel(c) + dilation * i * w + x * w + y; - for (int j = 0; j < inner_w; j ++) - { - outptr[j] = ptr[j*dilation]; - } - outptr += inner_w; - } - } - - Option opt_g = opt; - opt_g.blob_allocator = inner_top_blob.allocator; - convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g); - - #pragma omp parallel for num_threads(opt.num_threads) - for (int c = 0; c < num_output; c ++) - { - float *outptr = (float *) top_blob.channel(c) + x * outw + y; - for (int i = 0; i < inner_outh; i ++) - { - const float *ptr = (const float *) inner_top_blob.channel(c) + i * inner_outw; - for (int j = 0; j < inner_outw; j ++) - { - outptr[j*dilation] = ptr[j]; - } - outptr += dilation * outw; - } - } - } - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; -} - int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { if (bottom_blob.dims != 3) @@ -542,9 +435,9 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option return Convolution::forward(bottom_blob, top_blob, opt); } - if (use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { - return forward_int8(bottom_blob, top_blob, opt); + return forward_int8_arm(bottom_blob, top_blob, opt); } int w = bottom_blob.w; @@ -602,9 +495,9 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option if (top_blob.empty()) return -100; - if (opt.use_packing_layout == false && kernel_w == kernel_h && stride_w == 1 && stride_h == 1 && dilation_w != 1 && dilation_h == dilation_w) + if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1) { - return forwardDilation(bottom_blob_bordered, top_blob, opt); + return forwardDilation_arm(bottom_blob_bordered, top_blob, opt); } const int maxk = kernel_w * kernel_h; @@ -631,7 +524,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option #if __ARM_NEON if (elempack == 4 && out_elempack == 4) { - if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -640,7 +533,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv1x1s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -649,7 +542,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_winograd64_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -658,7 +551,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -667,7 +560,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -676,7 +569,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -750,7 +643,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option if (elempack == 1 && out_elempack == 4) { - if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv3x3s1_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt); @@ -759,7 +652,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv3x3s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt); @@ -768,7 +661,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 7 && kernel_h == 7 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv7x7s2_pack1to4_neon(bottom_blob_bordered, top_blob, weight_data_pack1to4, bias_data, opt); @@ -827,7 +720,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option if (elempack == 4 && out_elempack == 1) { - if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt); @@ -836,7 +729,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv1x1s2_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt); @@ -845,7 +738,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { // TODO more proper condition conv3x3s1_winograd64_pack4to1_neon(bottom_blob_bordered, top_blob, weight_data_pack4to1, bias_data, opt); @@ -942,7 +835,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { if (use_sgemm1x1) { @@ -958,7 +851,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 1 && kernel_h == 1 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { // conv1x1s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); conv_im2col_sgemm_neon(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); @@ -968,7 +861,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { if (use_winograd3x3 && w <= 120 && h <= 120) { @@ -985,7 +878,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { // conv3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); if (outw >=8 && outh >=8) @@ -998,7 +891,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 4 && kernel_h == 4 && stride_w == 4 && stride_h == 4 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 4 && kernel_h == 4 && dilation_w == 1 && dilation_h == 1 && stride_w == 4 && stride_h == 4) { conv4x4s4_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -1007,7 +900,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -1016,7 +909,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -1025,7 +918,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 7 && kernel_h == 7 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv7x7s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -1034,7 +927,7 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option activation->forward_inplace(top_blob, opt); } } - else if (kernel_w == 7 && kernel_h == 7 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 7 && kernel_h == 7 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { conv7x7s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -1094,61 +987,41 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option return 0; } -int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +int Convolution_arm::create_pipeline_int8_arm(const Option& opt) { - typedef void (*conv_int8_func)(const Mat&, Mat&, const Mat&, const Option&); - const int kernel_size = kernel_w; - const int stride = stride_w; + const int maxk = kernel_w * kernel_h; + const int num_input = weight_data_size / maxk / num_output; + + use_winograd3x3_int8 = false; + use_sgemm1x1_int8 = false; - // kernel_size x stride - conv_int8_func conv_int8_func_table[7][4] = + if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - { - conv1x1s1_int8_neon, - conv1x1s2_int8_neon, - 0, - 0 - }, // kernel_size = 1 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 2 - { - conv3x3s1_int8_neon, - conv3x3s2_int8_neon, - 0, - 0 - }, // kernel_size = 3 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 4 - { - conv5x5s1_int8_neon, - conv5x5s2_int8_neon, - 0, - 0 - }, // kernel_size = 5 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 6 - { - conv7x7s1_int8_neon, - conv7x7s2_int8_neon, - 0, - 0 - } // kernel_size = 7 - }; - - conv_int8_func conv_int8 = conv_int8_func_table[kernel_size-1][stride-1]; - if (!conv_int8) + use_winograd3x3_int8 = true; +// conv3x3s1_winograd23_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output); + conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output); + } + + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_data_int8, num_input, num_output); + } + else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + use_sgemm1x1_int8 = true; + conv1x1s1_sgemm_transform_kernel_int8_neon(weight_data, weight_1x1s1_sgemm_data_int8, num_input, num_output); + } + else + { + conv_im2col_sgemm_transform_kernel_int8_neon(weight_data, weight_sgemm_data_int8, num_input, num_output, maxk); + } + + return 0; +} + +int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + if (dilation_w > 1 || dilation_h > 1) { return Convolution::forward(bottom_blob, top_blob, opt); } @@ -1158,23 +1031,18 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; +// fprintf(stderr, "Convolution_arm input %d x %d ksize=%d %d stride=%d %d\n", w, h, kernel_w, kernel_h, stride_w, stride_h); + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + Mat bottom_blob_unbordered = bottom_blob; if (elemsize != 1) { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) - return -100; + Option opt_g = opt; + opt_g.blob_allocator = opt.workspace_allocator; - // quantize, scale and round to nearest - { - Option opt_g = opt; - opt_g.blob_allocator = bottom_blob_int8.allocator; - - quantize->forward(bottom_blob, bottom_blob_int8, opt_g); - } - - bottom_blob_unbordered = bottom_blob_int8; + quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g); } Mat bottom_blob_bordered = bottom_blob_unbordered; @@ -1186,8 +1054,9 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O } else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { - int wpad = kernel_size + (w - 1) / stride * stride - w; - int hpad = kernel_size + (h - 1) / stride * stride - h; + // tensorflow padding=SAME or onnx padding=SAME_UPPER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; @@ -1197,8 +1066,9 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O } else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) { - int wpad = kernel_size + (w - 1) / stride * stride - w; - int hpad = kernel_size + (h - 1) / stride * stride - h; + // onnx padding=SAME_LOWER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; @@ -1212,8 +1082,15 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O w = bottom_blob_bordered.w; h = bottom_blob_bordered.h; - int outw = (w - kernel_size) / stride + 1; - int outh = (h - kernel_size) / stride + 1; + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + + // int8 + size_t out_elemsize = use_int8_requantize ? 1u : 4u; + + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; // int8 if (use_int8_requantize == true) @@ -1223,90 +1100,107 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O if (top_blob_tm.empty()) return -100; - top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - if (use_sgemm1x1) - { - conv1x1s1_sgemm_int8_requant_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_int8_data, bias_data, requantize_scales, opt); - + if (use_sgemm1x1_int8) + { + std::vector requantize_scales; + for (int p=0; pforward_inplace(top_blob, opt); - } + } return 0; } - else if (use_winograd3x3) + else if (use_winograd3x3_int8) { - // conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_int8_data, opt); - conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_int8_data, opt); +// conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt); + conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3s2_int8_data, opt); + conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob_tm, weight_3x3s2_data_int8, opt); } else { - conv_int8(bottom_blob_bordered, top_blob_tm, weight_sgemm_int8_data, opt); + conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob_tm, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt); } // requantize, reverse scale inplace #pragma omp parallel for num_threads(opt.num_threads) for (int p=0; pforward(top_blob_tm_g, top_blob_g, opt_g); - } + + // requantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]); + + float scale_out = top_blob_int8_scale;//FIXME load param + + requantize_int8_to_int8(top_blob_tm, top_blob, scale_in, scale_out, &bias_data[p], bias_term ? 1 : 0, 0, opt_g); + } } else { - top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - if (use_sgemm1x1) + if (use_sgemm1x1_int8) { - conv1x1s1_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_int8_data, opt); + conv1x1s1_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_1x1s1_sgemm_data_int8, opt); } - else if (use_winograd3x3) + else if (use_winograd3x3_int8) { - // conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, opt); - // conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, opt); - conv3x3s1_winograd43_dequant_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_int8_data, bias_data, dequantize_scales, opt); - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; +// conv3x3s1_winograd23_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt); + conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob, weight_3x3s2_int8_data, opt); + conv3x3s2_packed_int8_neon(bottom_blob_bordered, top_blob, weight_3x3s2_data_int8, opt); } else { - conv_int8(bottom_blob_bordered, top_blob, weight_sgemm_int8_data, opt); + conv_im2col_sgemm_int8_neon(bottom_blob_bordered, top_blob, weight_sgemm_data_int8, kernel_w, kernel_h, stride_w, stride_h, opt); } // dequantize, reverse scale inplace #pragma omp parallel for num_threads(opt.num_threads) for (int p=0; pforward_inplace(top_blob_g, opt_g); + + // dequantize + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]); + + dequantize_int32_to_float32(top_blob_g, scale_in, &bias_data[p], bias_term ? 1 : 0, opt_g); } } @@ -1318,5 +1212,88 @@ int Convolution_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const O return 0; } +int Convolution_arm::forwardDilation_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + size_t elemsize = bottom_blob.elemsize; + + const int kernel_size = kernel_w; + const int stride = stride_w; + const int dilation = dilation_w; + const int kernel_extent = dilation * (kernel_size - 1) + 1; + + int outw = (w - kernel_extent) / stride + 1; + int outh = (h - kernel_extent) / stride + 1; + + top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // Make (dilation * dilation) batches + Mat inner_bottom_blob; + Mat inner_top_blob; + for (int x = 0; x < dilation; x ++) + { + for (int y = 0; y < dilation; y ++) + { + int inner_w = (w - y + dilation - 1) / dilation; + int inner_h = (h - x + dilation - 1) / dilation; + + int inner_outw = (inner_w - kernel_size) / stride + 1; + int inner_outh = (inner_h - kernel_size) / stride + 1; + + inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator); + if (inner_bottom_blob.empty()) + return -100; + + inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator); + if (inner_top_blob.empty()) + return -100; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int c = 0; c < bottom_blob.c; c ++) + { + float *outptr = inner_bottom_blob.channel(c); + + for (int i = 0; i < inner_h; i ++) + { + const float *ptr = (const float *) bottom_blob.channel(c) + dilation * i * w + x * w + y; + for (int j = 0; j < inner_w; j ++) + { + outptr[j] = ptr[j*dilation]; + } + outptr += inner_w; + } + } + + Option opt_g = opt; + opt_g.blob_allocator = inner_top_blob.allocator; + convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int c = 0; c < num_output; c ++) + { + float *outptr = (float *) top_blob.channel(c) + x * outw + y; + for (int i = 0; i < inner_outh; i ++) + { + const float *ptr = (const float *) inner_top_blob.channel(c) + i * inner_outw; + for (int j = 0; j < inner_outw; j ++) + { + outptr[j*dilation] = ptr[j]; + } + outptr += dilation * outw; + } + } + } + } + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + + return 0; +} } // namespace ncnn diff --git a/src/layer/arm/convolution_arm.h b/src/layer/arm/convolution_arm.h index 87958bf51..cd3f617fd 100644 --- a/src/layer/arm/convolution_arm.h +++ b/src/layer/arm/convolution_arm.h @@ -27,9 +27,12 @@ public: virtual int create_pipeline(const Option& opt); virtual int destroy_pipeline(const Option& opt); - virtual int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; - virtual int forwardDilation(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + +protected: + int create_pipeline_int8_arm(const Option& opt); + int forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + int forwardDilation_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; public: Layer* activation; @@ -38,12 +41,7 @@ public: Mat weight_3x3_winograd64_data; Mat weight_1x1_sgemm_data; Mat weight_3x3s2_data; - Mat weight_3x3s2_int8_data; - Mat weight_1x1s1_sgemm_int8_data; - Mat weight_3x3_winograd23_data; - Mat weight_sgemm_int8_data; Mat weight_sgemm_data; - std::vector weight_3x3_winograd23_int8_data; // forwardDilation Layer* convolution_dilation1; @@ -55,6 +53,14 @@ public: Mat weight_3x3_winograd64_data_pack4; Mat weight_1x1_sgemm_data_pack4; + + // int8 + bool use_winograd3x3_int8; + bool use_sgemm1x1_int8; + Mat weight_3x3s2_data_int8; + Mat weight_1x1s1_sgemm_data_int8; + Mat weight_sgemm_data_int8; + std::vector weight_3x3_winograd23_data_int8; }; } // namespace ncnn diff --git a/src/layer/arm/convolution_sgemm_int8.h b/src/layer/arm/convolution_sgemm_int8.h index c2f577dce..936e38047 100644 --- a/src/layer/arm/convolution_sgemm_int8.h +++ b/src/layer/arm/convolution_sgemm_int8.h @@ -13,6 +13,8 @@ // specific language governing permissions and limitations under the License. #if __aarch64__ + +#if 0// FIXME chgemm produce wrong result #include "gemm_symm_int8.h" static void conv_im2col_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size) @@ -86,6 +88,691 @@ static void conv_im2col_sgemm_int8_neon(const Mat &bottom_blob, Mat &top_blob, c } #else static void conv_im2col_sgemm_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch, int kernel_size) +{ + const signed char* kernel = _kernel; + + // kernel memory packed 4 x 4 + kernel_tm.create(4*kernel_size, inch, outch/4 + outch%4, (size_t)1u); + + int nn_outch = 0; + int remain_outch_start = 0; + + nn_outch = outch >> 2; + remain_outch_start = nn_outch << 2; + + for (int pp=0; pp> 2; + int remain_size_start = nn_size << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int ii=0; ii(i); + const signed char* img1 = bottom_im2row.row(i+1); + const signed char* img2 = bottom_im2row.row(i+2); + const signed char* img3 = bottom_im2row.row(i+3); + + signed char* tmpptr = bottom_tm.channel(i/4); + + int q = 0; + for (; q+1(i); + + signed char* tmpptr = bottom_tm.channel(i/4 + i%4); + + int q=0; + for (; q+1> 2; + remain_outch_start = nn_outch << 2; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp=0; pp> 2 + "cmp w4, #0 \n" + "beq 1f \n" + + "0: \n"// for (; k+3> 1 + "cmp w4, #0 \n" + "beq 3f \n" + + "2: \n"// for (; k+1create_pipeline(opt); } - if (use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { support_packing = false; - - return 0; } // create Convolution op for each group @@ -98,361 +96,147 @@ int ConvolutionDepthWise_arm::create_pipeline(const Option& opt) // depth-wise if (channels == group && group == num_output) { - int elempack = (opt.use_packing_layout && channels % 4 == 0) ? 4 : 1; - -#if __ARM_NEON - // pack4 - if (elempack == 4) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { - Mat weight_data_r2 = weight_data.reshape(maxk, group); - convert_packing(weight_data_r2, weight_data_pack4, 4); + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + return 0; + } + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + return 0; + } } -#endif // __ARM_NEON - } - else - { - // group convolution - for (int i=0; i<(int)group_ops.size(); i++) - delete group_ops[i]; - - group_ops.clear(); + else + { + int elempack = (opt.use_packing_layout && channels % 4 == 0) ? 4 : 1; - const int channels_g = channels / group; - const int num_output_g = num_output / group; +#if __ARM_NEON + // pack4 + if (elempack == 4) + { + Mat weight_data_r2 = weight_data.reshape(maxk, group); + convert_packing(weight_data_r2, weight_data_pack4, 4); - group_ops.resize(group); + return 0; + } +#endif // __ARM_NEON - for (int g=0; gload_param(pd); - - // set weights - if (bias_term) + if (elempack == 1) { - ncnn::Mat weights[4]; - weights[0] = weight_data_g; - weights[1] = bias_data_g; - - if (int8_scale_term) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - weights[2] = weight_data_int8_scales.range(g, 1); - weights[3] = bottom_blob_int8_scales.range(g, 1); + return 0; } - - op->load_model(ModelBinFromMatArray(weights)); - } - else - { - ncnn::Mat weights[3]; - weights[0] = weight_data_g; - - if (int8_scale_term) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { - weights[1] = weight_data_int8_scales.range(g, 1); - weights[2] = bottom_blob_int8_scales.range(g, 1); + return 0; + } + if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + return 0; + } + if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + return 0; } - - op->load_model(ModelBinFromMatArray(weights)); } - - op->create_pipeline(opt); - - group_ops[g] = op; } } - return 0; -} - -int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt) -{ - if (activation) - { - activation->destroy_pipeline(opt); - delete activation; - activation = 0; - } - + // group convolution for (int i=0; i<(int)group_ops.size(); i++) - { - group_ops[i]->destroy_pipeline(opt); delete group_ops[i]; - } - group_ops.clear(); - - return 0; -} - -int ConvolutionDepthWise_arm::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const -{ - int w = bottom_blob.w; - int h = bottom_blob.h; - int channels = bottom_blob.c; - size_t elemsize = bottom_blob.elemsize; -// int elempack = bottom_blob.elempack; - - const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; - const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; - - Mat bottom_blob_unbordered = bottom_blob; - if (elemsize != 1) - { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) - return -100; - - const int channels_g = channels / group; - // quantize, scale and round to nearest - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_g, bottom_blob_int8_g, opt_g); - } - - bottom_blob_unbordered = bottom_blob_int8; - } - - Mat bottom_blob_bordered = bottom_blob_unbordered; - if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) - { - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); - } - else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) - { - int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; - int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; - if (wpad > 0 || hpad > 0) - { - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - } - } - else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) - { - int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; - int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; - if (wpad > 0 || hpad > 0) - { - Option opt_b = opt; - opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - } - } - if (bottom_blob_bordered.empty()) - return -100; + group_ops.clear(); - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; + const int channels_g = channels / group; + const int num_output_g = num_output / group; - int outw = (w - kernel_extent_w) / stride_w + 1; - int outh = (h - kernel_extent_h) / stride_h + 1; -// int out_elempack = num_output % 4 == 0 ? 4 : 1; -// size_t out_elemsize = elemsize / elempack * out_elempack; + group_ops.resize(group); - // int8 - if (use_int8_requantize) + for (int g=0; gload_param(pd); + + // set weights + if (bias_term) { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) - { - if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2)) - { - if (stride_w == 1 && stride_h == 1) - { - convdw3x3s1_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); - } - else if (stride_w == 2 && stride_h == 2) - { - convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } + ncnn::Mat weights[4]; + weights[0] = weight_data_g; + weights[1] = bias_data_g; - return 0; - } - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_tm_g, opt_g); - } - - if (activation) + if (int8_scale_term) { - activation->forward_inplace(top_blob, opt); + weights[2] = weight_data_int8_scales.range(g, 1); + weights[3] = bottom_blob_int8_scales.range(g, 1); } - return 0; - } - - const int channels_g = channels / group; - const int num_output_g = num_output / group; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_tm_g, opt_g); + op->load_model(ModelBinFromMatArray(weights)); } - } - else - { - top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // depth-wise - if (channels == group && group == num_output) + else { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) - { - if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2)) - { - if (stride_w == 1 && stride_h == 1) - { - convdw3x3s1_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt); - } - else if (stride_w == 2 && stride_h == 2) - { - convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt); - } + ncnn::Mat weights[3]; + weights[0] = weight_data_g; - // dequantize, reverse scale inplace - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward_inplace(top_blob_g, opt_g); - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; - } - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); - } - - if (activation) + if (int8_scale_term) { - activation->forward_inplace(top_blob, opt); + weights[1] = weight_data_int8_scales.range(g, 1); + weights[2] = bottom_blob_int8_scales.range(g, 1); } - return 0; + op->load_model(ModelBinFromMatArray(weights)); } - const int channels_g = channels / group; - const int num_output_g = num_output / group; + op->create_pipeline(opt); - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; guse_int8_requantize = use_int8_requantize; FIXME - const ncnn::Layer* op = group_ops[g]; + group_ops[g] = op; + } - Option opt_g = opt; - opt_g.blob_allocator = top_blob.allocator; + return 0; +} - // forward - op->forward(bottom_blob_bordered_g, top_blob_g, opt_g); - } +int ConvolutionDepthWise_arm::destroy_pipeline(const Option& opt) +{ + if (activation) + { + activation->destroy_pipeline(opt); + delete activation; + activation = 0; } - if (activation) + for (int i=0; i<(int)group_ops.size(); i++) { - activation->forward_inplace(top_blob, opt); + group_ops[i]->destroy_pipeline(opt); + delete group_ops[i]; } + group_ops.clear(); return 0; } @@ -462,9 +246,9 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con // convolv with NxN kernel // value = value + bias - if (use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { - return forward_int8(bottom_blob, top_blob, opt); + return forward_int8_arm(bottom_blob, top_blob, opt); } int w = bottom_blob.w; @@ -524,31 +308,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con // depth-wise if (channels * elempack == group && group == num_output) { - const int maxk = kernel_w * kernel_h; - - // kernel offsets - std::vector _space_ofs(maxk); - int* space_ofs = &_space_ofs[0]; - { - int p1 = 0; - int p2 = 0; - int gap = w * dilation_h - kernel_w * dilation_w; - for (int i = 0; i < kernel_h; i++) - { - for (int j = 0; j < kernel_w; j++) - { - space_ofs[p1] = p2; - p1++; - p2 += dilation_w; - } - p2 += gap; - } - } - #if __ARM_NEON if (elempack == 4) { - if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { convdw3x3s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -556,8 +319,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { convdw3x3s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -565,8 +330,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { convdw5x5s1_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -574,8 +341,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { convdw5x5s2_pack4_neon(bottom_blob_bordered, top_blob, weight_data_pack4, bias_data, opt); @@ -583,9 +352,32 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } else { + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + #pragma omp parallel for num_threads(opt.num_threads) for (int g=0; gforward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 3 && kernel_h == 3 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { convdw3x3s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -644,8 +440,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { convdw5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -653,8 +451,10 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else if (kernel_w == 5 && kernel_h == 5 && stride_w == 2 && stride_h == 2 && dilation_w == 1 && dilation_h == 1) + else if (kernel_w == 5 && kernel_h == 5 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { convdw5x5s2_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); @@ -662,98 +462,278 @@ int ConvolutionDepthWise_arm::forward(const Mat& bottom_blob, Mat& top_blob, con { activation->forward_inplace(top_blob, opt); } + + return 0; } - else - { - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); + } + + // packing + if (out_g_elempack == 1 && out_elempack == 4) + { + convert_packing(top_blob_unpacked, top_blob, 4, opt); } else { - // group convolution - const int channels_g = channels * elempack / group; - const int num_output_g = num_output / group; + top_blob = top_blob_unpacked; + } - int g_elempack = (opt.use_packing_layout && channels_g % 4 == 0) ? 4 : 1; - int out_g_elempack = (opt.use_packing_layout && num_output_g % 4 == 0) ? 4 : 1; + return 0; +} - // unpacking - Mat bottom_blob_bordered_unpacked = bottom_blob_bordered; - if (elempack == 4 && g_elempack == 1) +int ConvolutionDepthWise_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + Mat bottom_blob_unbordered = bottom_blob; + if (elemsize != 1) + { + bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator); + if (bottom_blob_unbordered.empty()) + return -100; + + const int channels_g = channels / group; + + // quantize, scale and round to nearest + #pragma omp parallel for num_threads(opt.num_threads) + for (int g=0; g 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); + } + else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) + { + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) { - top_blob_unpacked.create(outw, outh, num_output, out_elemsize / out_elempack, 1, opt.workspace_allocator); - if (top_blob_unpacked.empty()) - return -100; + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); } - - for (int g=0; g 0 || hpad > 0) { - const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack); - Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack); + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + if (bottom_blob_bordered.empty()) + return -100; - const ncnn::Layer* op = group_ops[g]; + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; - Option opt_g = opt; - opt_g.blob_allocator = top_blob_unpacked.allocator; + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; - // forward - op->forward(bottom_blob_bordered_g, top_blob_g, opt_g); - } + // int8 + size_t out_elemsize = use_int8_requantize ? 1u : 4u; - // packing - if (out_g_elempack == 1 && out_elempack == 4) + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // depth-wise + if (channels == group && group == num_output) + { + if (use_int8_requantize) { - convert_packing(top_blob_unpacked, top_blob, 4, opt); + std::vector requantize_scales; + for (int g=0; gforward_inplace(top_blob, opt); + } + + return 0; + } + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + convdw3x3s2_int8_requant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + + return 0; + } } else { - top_blob = top_blob_unpacked; +// std::vector dequantize_scales; +// for (int g=0; gforward_inplace(top_blob, opt); + } + + return 0; + } + else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + convdw3x3s2_int8_neon(bottom_blob_bordered, top_blob, weight_data, opt); +// convdw3x3s2_int8_dequant_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); + + // dequantize, reverse scale inplace + #pragma omp parallel for num_threads(opt.num_threads) + for (int g=0; gforward_inplace(top_blob, opt); + } + + return 0; + } } } + // group convolution + const int channels_g = channels / group; + const int num_output_g = num_output / group; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); + } + return 0; } diff --git a/src/layer/arm/convolutiondepthwise_arm.h b/src/layer/arm/convolutiondepthwise_arm.h index 454f900ce..9985b2a8a 100644 --- a/src/layer/arm/convolutiondepthwise_arm.h +++ b/src/layer/arm/convolutiondepthwise_arm.h @@ -27,9 +27,11 @@ public: virtual int create_pipeline(const Option& opt); virtual int destroy_pipeline(const Option& opt); - virtual int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +protected: + int forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + public: Layer* activation; std::vector group_ops; diff --git a/src/layer/arm/innerproduct_arm.cpp b/src/layer/arm/innerproduct_arm.cpp index 2f325b00d..0d623c2de 100644 --- a/src/layer/arm/innerproduct_arm.cpp +++ b/src/layer/arm/innerproduct_arm.cpp @@ -30,7 +30,6 @@ InnerProduct_arm::InnerProduct_arm() { #if __ARM_NEON support_packing = true; - use_fp32_packing_inference = false; #endif // __ARM_NEON flatten = 0; @@ -39,18 +38,7 @@ InnerProduct_arm::InnerProduct_arm() int InnerProduct_arm::create_pipeline(const Option& opt) { #if __ARM_NEON - bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u); - - use_fp32_packing_inference = opt.use_packing_layout && weight_data_is_float32 && !use_int8_inference; - - if (use_int8_inference) - { - support_packing = false; - } - - if (use_fp32_packing_inference) - { - + if (opt.use_packing_layout) { flatten = ncnn::create_layer(ncnn::LayerType::Flatten); @@ -60,8 +48,6 @@ int InnerProduct_arm::create_pipeline(const Option& opt) flatten->create_pipeline(opt); } - - } // opt.use_packing_layout #endif // __ARM_NEON return 0; @@ -81,7 +67,7 @@ int InnerProduct_arm::destroy_pipeline(const Option& opt) int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { - if (use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { // TODO return InnerProduct::forward(bottom_blob, top_blob, opt); @@ -95,34 +81,27 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio int size = w * h; #if __ARM_NEON - if (use_fp32_packing_inference) - { - if (elempack == 4) { + // flatten + Mat bottom_blob_flattened = bottom_blob; + if (bottom_blob.dims != 1) + { + Option opt_flatten = opt; + opt_flatten.blob_allocator = opt.workspace_allocator; - // flatten - Mat bottom_blob_flattened = bottom_blob; - if (bottom_blob.dims != 1) - { - Option opt_flatten = opt; - opt_flatten.blob_allocator = opt.workspace_allocator; - - flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); - } - - // pack1 - { - bottom_blob_flattened.w *= bottom_blob_flattened.elempack; - bottom_blob_flattened.elemsize = 4u; - bottom_blob_flattened.elempack = 1; - } + flatten->forward(bottom_blob, bottom_blob_flattened, opt_flatten); + } - return forward(bottom_blob_flattened, top_blob, opt); + // pack1 + { + bottom_blob_flattened.w *= bottom_blob_flattened.elempack; + bottom_blob_flattened.elemsize = 4u; + bottom_blob_flattened.elempack = 1; + } + return forward(bottom_blob_flattened, top_blob, opt); } - - } // opt.use_packing_layout #endif // __ARM_NEON top_blob.create(num_output, elemsize, opt.blob_allocator); diff --git a/src/layer/arm/innerproduct_arm.h b/src/layer/arm/innerproduct_arm.h index dbced4604..0233ea506 100644 --- a/src/layer/arm/innerproduct_arm.h +++ b/src/layer/arm/innerproduct_arm.h @@ -30,8 +30,6 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; public: - bool use_fp32_packing_inference; - ncnn::Layer* flatten; }; diff --git a/src/layer/convolution.cpp b/src/layer/convolution.cpp index 746be90cb..85f759190 100644 --- a/src/layer/convolution.cpp +++ b/src/layer/convolution.cpp @@ -24,9 +24,8 @@ Convolution::Convolution() { one_blob_only = true; support_inplace = false; - use_int8_requantize = false; - quantize = 0; + use_int8_requantize = false; } int Convolution::load_param(const ParamDict& pd) @@ -77,170 +76,28 @@ int Convolution::load_model(const ModelBin& mb) int Convolution::create_pipeline(const Option& opt) { - bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u); - bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u); - - if (weight_data_is_int8 && !opt.use_int8_inference) - { - fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n"); - return -1; - } - - use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term)); - // runtime quantize the weight data - if (weight_data_is_float32 && use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term) { - // quantize weight to int8 Mat int8_weight_data(weight_data_size, (size_t)1u); if (int8_weight_data.empty()) return -100; const int weight_data_size_output = weight_data_size / num_output; - for (int n=0; nload_param(pd); - - op->create_pipeline(opt); - Option opt_q = opt; opt_q.blob_allocator = int8_weight_data.allocator; - const Mat weight_data_n = weight_data.range(weight_data_size_output * n, weight_data_size_output); - Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * n, weight_data_size_output); - op->forward(weight_data_n, int8_weight_data_n, opt_q); - - delete op; + const Mat weight_data_n = weight_data.range(weight_data_size_output * p, weight_data_size_output); + Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * p, weight_data_size_output); + quantize_float32_to_int8(weight_data_n, int8_weight_data_n, weight_data_int8_scales[p], opt_q); } weight_data = int8_weight_data; } - // initial the quantize,dequantize op layer - if (use_int8_inference) - { - quantize = ncnn::create_layer(ncnn::LayerType::Quantize); - { - ncnn::ParamDict pd; - pd.set(0, bottom_blob_int8_scale);// scale - - quantize->load_param(pd); - - quantize->create_pipeline(opt); - } - - dequantize_ops.resize(num_output); - for (int n=0; nload_param(pd); - - dequantize_ops[n]->create_pipeline(opt); - - ncnn::Mat weights[1]; - weights[0] = bias_data.range(n, 1); - - dequantize_ops[n]->load_model(ModelBinFromMatArray(weights)); - - dequantize_scales.push_back(top_rescale); - } - } - - return 0; -} - -int Convolution::destroy_pipeline(const Option& opt) -{ - if (quantize) - { - quantize->destroy_pipeline(opt); - delete quantize; - quantize = 0; - } - - for (int i=0; i<(int)dequantize_ops.size(); i++) - { - dequantize_ops[i]->destroy_pipeline(opt); - delete dequantize_ops[i]; - } - dequantize_ops.clear(); - - for (int i=0; i<(int)requantize_ops.size(); i++) - { - requantize_ops[i]->destroy_pipeline(opt); - delete requantize_ops[i]; - } - requantize_ops.clear(); - - dequantize_scales.clear(); - requantize_scales.clear(); - - return 0; -} - -int Convolution::create_requantize_op(void) -{ - if (!use_int8_requantize) - { - fprintf(stderr, "requantized op set but use_int8_requantize disabled\n"); - return -1; - } - - requantize_ops.resize(num_output); - for (int n=0; nload_param(pd); - - ncnn::Mat weights[1]; - weights[0] = bias_data.range(n, 1); - - requantize_ops[n]->load_model(ModelBinFromMatArray(weights)); - - requantize_scales.push_back(scale_in); - requantize_scales.push_back(scale_out); - } - return 0; } @@ -249,6 +106,11 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op // convolv with NxN kernel // value = value + bias + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + { + return forward_int8(bottom_blob, top_blob, opt); + } + // flattened blob, implement as InnerProduct if (bottom_blob.dims == 1 && kernel_w == 1 && kernel_h == 1) { @@ -301,31 +163,12 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; - Mat bottom_blob_unbordered = bottom_blob; - if (use_int8_inference && elemsize != 1) - { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) - return -100; - - // quantize, scale and round to nearest - { - Option opt_g = opt; - opt_g.blob_allocator = bottom_blob_int8.allocator; - - quantize->forward(bottom_blob, bottom_blob_int8, opt_g); - } - - bottom_blob_unbordered = bottom_blob_int8; - } - - Mat bottom_blob_bordered = bottom_blob_unbordered; + Mat bottom_blob_bordered = bottom_blob; if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); + copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); } else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { @@ -336,7 +179,7 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); } } else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) @@ -348,7 +191,7 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; - copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); } } if (bottom_blob_bordered.empty()) @@ -381,148 +224,6 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op } } - // int8 - if (use_int8_inference) - { - if (use_int8_requantize == true) - { - Mat top_blob_tm; - top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator); - if (top_blob_tm.empty()) - return -100; - - top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p=0; p(i*stride_h) + j*stride_w; - - for (int k = 0; k < maxk; k++) - { - int val = sptr[ space_ofs[k] ]; - int w = kptr[k]; - sum += val * w; - } - - kptr += maxk; - } - - outptr[j] = sum; - } - - outptr += outw; - } - - // requantize, reverse scale inplace - { - Option opt_g = opt; - opt_g.num_threads = 1; - opt_g.blob_allocator = top_blob.allocator; - - Mat top_blob_tm_g = top_blob_tm.channel_range(p, 1); - Mat top_blob_g = top_blob.channel_range(p, 1); - requantize_ops[p]->forward(top_blob_tm_g, top_blob_g, opt_g); - } - - // activation relu - if (activation_type == 1) - { - signed char* outptr_s8 = top_blob.channel(p); - - for (int i = 0; i < outh*outw; i++) - { - if (outptr_s8[i] < 0) - outptr_s8[i] = 0; - } - } - } - } - else - { - top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p=0; p(i*stride_h) + j*stride_w; - - for (int k = 0; k < maxk; k++) - { - int val = sptr[ space_ofs[k] ]; - int w = kptr[k]; - sum += val * w; - } - - kptr += maxk; - } - - outptr[j] = sum; - } - - outptr += outw; - } - - // dequantize, reverse scale inplace - { - Option opt_g = opt; - opt_g.num_threads = 1; - opt_g.blob_allocator = top_blob.allocator; - - Mat top_blob_g = top_blob.channel_range(p, 1); - dequantize_ops[p]->forward_inplace(top_blob_g, opt_g); - } - - // activation relu - if (activation_type == 1) - { - float* outptr_fp32 = top_blob.channel(p); - - for (int i = 0; i < outh*outw; i++) - { - outptr_fp32[i] = std::max(outptr_fp32[i], 0.f); - } - } - } - } - - return 0; - } - // float32 top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); if (top_blob.empty()) @@ -594,4 +295,186 @@ int Convolution::forward(const Mat& bottom_blob, Mat& top_blob, const Option& op return 0; } +static inline signed char float2int8(float v) +{ + int int32 = static_cast(round(v)); + if (int32 > 127) return 127; + if (int32 < -127) return -127; + return (signed char)int32; +} + +int Convolution::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + +// fprintf(stderr, "Convolution input %d x %d ksize=%d %d stride=%d %d\n", w, h, kernel_w, kernel_h, stride_w, stride_h); + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + Mat bottom_blob_unbordered = bottom_blob; + if (elemsize != 1) + { + Option opt_g = opt; + opt_g.blob_allocator = opt.workspace_allocator; + + quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g); + } + + Mat bottom_blob_bordered = bottom_blob_unbordered; + if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); + } + else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) + { + // tensorflow padding=SAME or onnx padding=SAME_UPPER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) + { + // onnx padding=SAME_LOWER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + + const int maxk = kernel_w * kernel_h; + + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; + } + } + + // int8 + size_t out_elemsize = use_int8_requantize ? 1u : 4u; + + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p=0; p(i*stride_h) + j*stride_w; + + for (int k = 0; k < maxk; k++) + { + int val = sptr[ space_ofs[k] ]; + int w = kptr[k]; + sum += val * w; + } + + kptr += maxk; + } + + if (use_int8_requantize) + { + // requantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]); + + float sumfp32 = sum * scale_in; + + if (bias_term) + sumfp32 += bias_data[p]; + + float scale_out = top_blob_int8_scale;//FIXME load param + + signed char sums8 = float2int8(sumfp32 * scale_out); + + if (activation_type == 1) + { + sums8 = std::max(sums8, (signed char)0); + } + + outptr[0] = sums8; + outptr += 1; + } + else + { + // dequantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]); + + float sumfp32 = sum * scale_in; + + if (bias_term) + sumfp32 += bias_data[p]; + + if (activation_type == 1) + { + sumfp32 = std::max(sumfp32, 0.f); + } + + ((float*)outptr)[0] = sumfp32; + outptr += 4; + } + } + } + } + + return 0; +} + } // namespace ncnn diff --git a/src/layer/convolution.h b/src/layer/convolution.h index 0c55a4282..6ab7de754 100644 --- a/src/layer/convolution.h +++ b/src/layer/convolution.h @@ -29,12 +29,12 @@ public: virtual int load_model(const ModelBin& mb); virtual int create_pipeline(const Option& opt); - virtual int destroy_pipeline(const Option& opt); - - virtual int create_requantize_op(void); virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +protected: + int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + public: // param int num_output; @@ -65,19 +65,10 @@ public: Mat weight_data_int8_scales; float bottom_blob_int8_scale; - float top_blob_int8_scale; + float top_blob_int8_scale;// TODO load param - bool use_int8_inference; bool use_int8_requantize; - ncnn::Layer* quantize; - std::vector dequantize_ops; - std::vector requantize_ops; - - // merge de/requantize op into convolution op - std::vector dequantize_scales; - std::vector requantize_scales; - // implementation type, 0 means do not use auto pack model int impl_type; }; diff --git a/src/layer/convolutiondepthwise.cpp b/src/layer/convolutiondepthwise.cpp index 46339702b..ff0786f3b 100644 --- a/src/layer/convolutiondepthwise.cpp +++ b/src/layer/convolutiondepthwise.cpp @@ -100,20 +100,9 @@ int ConvolutionDepthWise::load_model(const ModelBin& mb) int ConvolutionDepthWise::create_pipeline(const Option& opt) { - bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u); - bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u); - - if (weight_data_is_int8 && !opt.use_int8_inference) + // runtime quantize the weight data + if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term) { - fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n"); - return -1; - } - - use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term)); - - if (weight_data_is_float32 && use_int8_inference) - { - // quantize weight to int8 Mat int8_weight_data(weight_data_size, (size_t)1u); if (int8_weight_data.empty()) return -100; @@ -122,153 +111,259 @@ int ConvolutionDepthWise::create_pipeline(const Option& opt) for (int g=0; gload_param(pd); - - op->create_pipeline(opt); - Option opt_q = opt; opt_q.blob_allocator = int8_weight_data.allocator; const Mat weight_data_g = weight_data.range(weight_data_size_g * g, weight_data_size_g); Mat int8_weight_data_g = int8_weight_data.range(weight_data_size_g * g, weight_data_size_g); - op->forward(weight_data_g, int8_weight_data_g, opt_q); - - delete op; + quantize_float32_to_int8(weight_data_g, int8_weight_data_g, weight_data_int8_scales[g], opt_q); } weight_data = int8_weight_data; } - if (use_int8_inference) - { - quantize_ops.resize(group); - dequantize_ops.resize(group); - - for (int g=0; gload_param(pd); + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + { + return forward_int8(bottom_blob, top_blob, opt); + } - quantize_ops[g]->create_pipeline(opt); - } + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; - for (int g=0; gload_param(pd); + Mat bottom_blob_bordered = bottom_blob; + if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); + } + else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) + { + // tensorflow padding=SAME or onnx padding=SAME_UPPER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) + { + // onnx padding=SAME_LOWER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + if (bottom_blob_bordered.empty()) + return -100; - ncnn::Mat weights[1]; - weights[0] = bias_data.range(g, 1); + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; - dequantize_ops[g]->load_model(ModelBinFromMatArray(weights)); + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; - dequantize_ops[g]->create_pipeline(opt); + const int maxk = kernel_w * kernel_h; - dequantize_scales.push_back(top_rescale); + // kernel offsets + std::vector _space_ofs(maxk); + int* space_ofs = &_space_ofs[0]; + { + int p1 = 0; + int p2 = 0; + int gap = w * dilation_h - kernel_w * dilation_w; + for (int i = 0; i < kernel_h; i++) + { + for (int j = 0; j < kernel_w; j++) + { + space_ofs[p1] = p2; + p1++; + p2 += dilation_w; + } + p2 += gap; } } - return 0; -} - -int ConvolutionDepthWise::destroy_pipeline(const Option& opt) -{ - for (int i=0; i<(int)quantize_ops.size(); i++) - { - quantize_ops[i]->destroy_pipeline(opt); - delete quantize_ops[i]; - } - quantize_ops.clear(); + // float32 + top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; - for (int i=0; i<(int)dequantize_ops.size(); i++) + // depth-wise + if (channels == group && group == num_output) { - dequantize_ops[i]->destroy_pipeline(opt); - delete dequantize_ops[i]; - } - dequantize_ops.clear(); + #pragma omp parallel for num_threads(opt.num_threads) + for (int g=0; gdestroy_pipeline(opt); - delete requantize_ops[i]; - } - requantize_ops.clear(); + for (int i = 0; i < outh; i++) + { + for (int j = 0; j < outw; j++) + { + float sum = 0.f; - dequantize_scales.clear(); - requantize_scales.clear(); + if (bias_term) + sum = bias_data[g]; - return 0; -} + const float* sptr = m.row(i*stride_h) + j*stride_w; -int ConvolutionDepthWise::create_requantize_op(void) -{ - if (!use_int8_requantize) - { - fprintf(stderr, "requantized op set but use_int8_requantize disabled\n"); - return -1; - } + for (int k = 0; k < maxk; k++) + { + float val = sptr[ space_ofs[k] ]; + float w = kptr[k]; + sum += val * w; + } - requantize_ops.resize(group); - for (int g=0; g 0.f ? sum : sum * slope; + } + else if (activation_type == 3) + { + float min = activation_params[0]; + float max = activation_params[1]; + if (sum < min) + sum = min; + if (sum > max) + sum = max; + } + else if (activation_type == 4) + { + sum = static_cast(1.f / (1.f + exp(-sum))); + } - float scale_in = 1.f; - float scale_out = 1.f; + outptr[j] = sum; + } - if (weight_data_int8_scales[g] == 0) - { - scale_in = 0; + outptr += outw; + } } - else + } + else + { + // group convolution + const int channels_g = channels / group; + const int num_output_g = num_output / group; + +#ifdef _WIN32 + #pragma omp parallel for num_threads(opt.num_threads) +#else // _WIN32 + #pragma omp parallel for collapse(2) num_threads(opt.num_threads) +#endif // _WIN32 + for (int g=0; gload_param(pd); + for (int k = 0; k < maxk; k++) + { + float val = sptr[ space_ofs[k] ]; + float w = kptr[k]; + sum += val * w; + } + + kptr += maxk; + } - ncnn::Mat weights[1]; - weights[0] = bias_data.range(g, 1); + if (activation_type == 1) + { + sum = std::max(sum, 0.f); + } + else if (activation_type == 2) + { + float slope = activation_params[0]; + sum = sum > 0.f ? sum : sum * slope; + } + else if (activation_type == 3) + { + float min = activation_params[0]; + float max = activation_params[1]; + if (sum < min) + sum = min; + if (sum > max) + sum = max; + } + else if (activation_type == 4) + { + sum = static_cast(1.f / (1.f + exp(-sum))); + } - requantize_ops[g]->load_model(ModelBinFromMatArray(weights)); + outptr[j] = sum; + } - requantize_scales.push_back(scale_in); - requantize_scales.push_back(scale_out); + outptr += outw; + } + } + } } return 0; } -int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +static inline signed char float2int8(float v) +{ + int int32 = static_cast(round(v)); + if (int32 > 127) return 127; + if (int32 < -127) return -127; + return (signed char)int32; +} + +int ConvolutionDepthWise::forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { // convolv with NxN kernel // value = value + bias @@ -290,11 +385,10 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; Mat bottom_blob_unbordered = bottom_blob; - if (use_int8_inference && elemsize != 1) + if (elemsize != 1) { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) + bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator); + if (bottom_blob_unbordered.empty()) return -100; const int channels_g = channels / group; @@ -305,14 +399,13 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O { Option opt_g = opt; opt_g.num_threads = 1; - opt_g.blob_allocator = bottom_blob_int8.allocator; + opt_g.blob_allocator = bottom_blob_unbordered.allocator; const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g); - Mat bottom_blob_int8_g = bottom_blob_int8.channel_range(channels_g * g, channels_g); - quantize_ops[g]->forward(bottom_blob_g, bottom_blob_int8_g, opt_g); - } + Mat bottom_blob_int8_g = bottom_blob_unbordered.channel_range(channels_g * g, channels_g); - bottom_blob_unbordered = bottom_blob_int8; + quantize_float32_to_int8(bottom_blob_g, bottom_blob_int8_g, bottom_blob_int8_scales[g], opt_g); + } } Mat bottom_blob_bordered = bottom_blob_unbordered; @@ -377,173 +470,119 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O } // int8 - if (use_int8_inference) + size_t out_elemsize = use_int8_requantize ? 1u : 4u; + + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // depth-wise + if (channels == group && group == num_output) { - if (use_int8_requantize == true) + #pragma omp parallel for num_threads(opt.num_threads) + for (int g=0; g(i*stride_h) + j*stride_w; - const signed char* sptr = m.row(i*stride_h) + j*stride_w; - - for (int k = 0; k < maxk; k++) - { - signed char val = sptr[ space_ofs[k] ]; - signed char w = kptr[k]; - sum += val * w; - } - - outptr[j] = sum; - } - - outptr += outw; - } - - // requantize, reverse scale inplace + for (int k = 0; k < maxk; k++) { - Option opt_g = opt; - opt_g.num_threads = 1; - opt_g.blob_allocator = top_blob.allocator; - - Mat top_blob_tm_g = top_blob_tm.channel_range(g, 1); - Mat top_blob_g = top_blob.channel_range(g, 1); - requantize_ops[g]->forward(top_blob_tm_g, top_blob_g, opt_g); + signed char val = sptr[ space_ofs[k] ]; + signed char w = kptr[k]; + sum += val * w; } - // activation relu - if (activation_type == 1) - { - signed char* outptr_s8 = top_blob.channel(g); - - for (int i = 0; i < outh*outw; i++) - { - if (outptr_s8[i] < 0) - outptr_s8[i] = 0; - } - } - } - } - else - { - const int channels_g = channels / group; - const int num_output_g = num_output / group; - -#ifdef _WIN32 - #pragma omp parallel for num_threads(opt.num_threads) -#else // _WIN32 - #pragma omp parallel for collapse(2) num_threads(opt.num_threads) -#endif // _WIN32 - for (int g=0; g(i*stride_h) + j*stride_w; + if (bias_term) + sumfp32 += bias_data[g]; - for (int k = 0; k < maxk; k++) - { - signed char val = sptr[ space_ofs[k] ]; - signed char w = kptr[k]; - sum += val * w; - } + float scale_out = top_blob_int8_scale;//FIXME load param - kptr += maxk; - } + signed char sums8 = float2int8(sumfp32 * scale_out); - outptr[j] = sum; - } - - outptr += outw; + if (activation_type == 1) + { + sums8 = std::max(sums8, (signed char)0); } - } - } - // requantize, reverse scale inplace - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(top_blob_tm_g, top_blob_g, opt_g); + float sumfp32 = sum * scale_in; - if (activation_type == 1) - { - for (int p=0; p(i*stride_h) + j*stride_w; for (int k = 0; k < maxk; k++) @@ -553,248 +592,59 @@ int ConvolutionDepthWise::forward(const Mat& bottom_blob, Mat& top_blob, const O sum += val * w; } - outptr[j] = sum; - } - - outptr += outw; - } - - // dequantize, reverse scale inplace - { - Option opt_g = opt; - opt_g.num_threads = 1; - opt_g.blob_allocator = top_blob.allocator; - - Mat top_blob_g = top_blob.channel_range(g, 1); - dequantize_ops[g]->forward_inplace(top_blob_g, opt_g); - } - - // activation relu - if (activation_type == 1) - { - float* outptr_fp32 = top_blob.channel(g); - - for (int i = 0; i < outh*outw; i++) - { - outptr_fp32[i] = std::max(outptr_fp32[i], 0.f); + kptr += maxk; } - } - } - } - else - { - const int channels_g = channels / group; - const int num_output_g = num_output / group; - -#ifdef _WIN32 - #pragma omp parallel for num_threads(opt.num_threads) -#else // _WIN32 - #pragma omp parallel for collapse(2) num_threads(opt.num_threads) -#endif // _WIN32 - for (int g=0; g(i*stride_h) + j*stride_w; + if (bias_term) + sumfp32 += bias_data[g * num_output_g + p]; - for (int k = 0; k < maxk; k++) - { - signed char val = sptr[ space_ofs[k] ]; - signed char w = kptr[k]; - sum += val * w; - } + float scale_out = top_blob_int8_scale;//FIXME load param - kptr += maxk; - } + signed char sums8 = float2int8(sumfp32 * scale_out); - outptr[j] = sum; + if (activation_type == 1) + { + sums8 = std::max(sums8, (signed char)0); } - outptr += outw; + outptr[0] = sums8; + outptr += 1; } - } - } - - // dequantize, reverse scale inplace - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward_inplace(top_blob_g, opt_g); + float sumfp32 = sum * scale_in; - if (activation_type == 1) - { - for (int p=0; p 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; - } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } - - outptr[j] = sum; - } - - outptr += outw; - } - } - - return 0; - } - - const int channels_g = channels / group; - const int num_output_g = num_output / group; - -#ifdef _WIN32 - #pragma omp parallel for num_threads(opt.num_threads) -#else // _WIN32 - #pragma omp parallel for collapse(2) num_threads(opt.num_threads) -#endif // _WIN32 - for (int g=0; g 0.f ? sum : sum * slope; - } - else if (activation_type == 3) - { - float min = activation_params[0]; - float max = activation_params[1]; - if (sum < min) - sum = min; - if (sum > max) - sum = max; } - else if (activation_type == 4) - { - sum = static_cast(1.f / (1.f + exp(-sum))); - } - - outptr[j] = sum; } - - outptr += outw; } } } diff --git a/src/layer/convolutiondepthwise.h b/src/layer/convolutiondepthwise.h index 9705188e2..48f84635e 100644 --- a/src/layer/convolutiondepthwise.h +++ b/src/layer/convolutiondepthwise.h @@ -29,12 +29,12 @@ public: virtual int load_model(const ModelBin& mb); virtual int create_pipeline(const Option& opt); - virtual int destroy_pipeline(const Option& opt); - - virtual int create_requantize_op(void); virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +protected: + int forward_int8(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + public: // param int num_output; @@ -68,16 +68,7 @@ public: Mat bottom_blob_int8_scales; float top_blob_int8_scale; - bool use_int8_inference; bool use_int8_requantize; - - std::vector quantize_ops; - std::vector dequantize_ops; - std::vector requantize_ops; - - // merge de/requantize op into convolutiondepthwise op - std::vector dequantize_scales; - std::vector requantize_scales; }; } // namespace ncnn diff --git a/src/layer/innerproduct.cpp b/src/layer/innerproduct.cpp index 9c803d92c..4d2cdc313 100644 --- a/src/layer/innerproduct.cpp +++ b/src/layer/innerproduct.cpp @@ -24,8 +24,6 @@ InnerProduct::InnerProduct() { one_blob_only = true; support_inplace = false; - - quantize = 0; } int InnerProduct::load_param(const ParamDict& pd) @@ -64,87 +62,23 @@ int InnerProduct::load_model(const ModelBin& mb) int InnerProduct::create_pipeline(const Option& opt) { - bool weight_data_is_int8 = (weight_data.elemsize == (size_t)1u); - bool weight_data_is_float32 = (weight_data.elemsize == (size_t)4u); - - if (weight_data_is_int8 && !opt.use_int8_inference) - { - fprintf(stderr, "quantized int8 weight loaded but use_int8_inference disabled\n"); - return -1; - } - - use_int8_inference = opt.use_int8_inference && (weight_data_is_int8 || (weight_data_is_float32 && int8_scale_term)); - - // initial the quantize,dequantize op layer - if (use_int8_inference) - { - quantize = ncnn::create_layer(ncnn::LayerType::Quantize); - { - ncnn::ParamDict pd; - pd.set(0, bottom_blob_int8_scale);// scale - - quantize->load_param(pd); - - quantize->create_pipeline(opt); - } - - dequantize_ops.resize(num_output); - for (int n=0; nload_param(pd); - - ncnn::Mat weights[1]; - weights[0] = bias_data.range(n, 1); - - dequantize_ops[n]->load_model(ModelBinFromMatArray(weights)); - - dequantize_ops[n]->create_pipeline(opt); - } - } - // runtime quantize the weight data - if (weight_data_is_float32 && use_int8_inference) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)4u && int8_scale_term) { - // quantize weight to int8 Mat int8_weight_data(weight_data_size, (size_t)1u); if (int8_weight_data.empty()) return -100; const int weight_data_size_output = weight_data_size / num_output; - for (int n=0; nload_param(pd); - - op->create_pipeline(opt); - Option opt_q = opt; opt_q.blob_allocator = int8_weight_data.allocator; - const Mat weight_data_n = weight_data.range(weight_data_size_output * n, weight_data_size_output); - Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * n, weight_data_size_output); - op->forward(weight_data_n, int8_weight_data_n, opt_q); - - delete op; + const Mat weight_data_n = weight_data.range(weight_data_size_output * p, weight_data_size_output); + Mat int8_weight_data_n = int8_weight_data.range(weight_data_size_output * p, weight_data_size_output); + quantize_float32_to_int8(weight_data_n, int8_weight_data_n, weight_data_int8_scales[p], opt_q); } weight_data = int8_weight_data; @@ -153,27 +87,13 @@ int InnerProduct::create_pipeline(const Option& opt) return 0; } -int InnerProduct::destroy_pipeline(const Option& opt) +int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { - if (quantize) + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) { - quantize->destroy_pipeline(opt); - delete quantize; - quantize = 0; + return forward_int8(bottom_blob, top_blob, opt); } - for (int i=0; i<(int)dequantize_ops.size(); i++) - { - dequantize_ops[i]->destroy_pipeline(opt); - delete dequantize_ops[i]; - } - dequantize_ops.clear(); - - return 0; -} - -int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const -{ int w = bottom_blob.w; int h = bottom_blob.h; int channels = bottom_blob.c; @@ -184,74 +104,6 @@ int InnerProduct::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o if (top_blob.empty()) return -100; - if (use_int8_inference) - { - Mat bottom_blob_tm = bottom_blob; - if (elemsize != 1) - { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) - return -100; - - // quantize, scale and round to nearest - { - Option opt_g = opt; - opt_g.blob_allocator = bottom_blob_int8.allocator; - - quantize->forward(bottom_blob, bottom_blob_int8, opt_g); - } - - bottom_blob_tm = bottom_blob_int8; - } - - // num_output - #pragma omp parallel for num_threads(opt.num_threads) - for (int p=0; p dequantize_ops; }; } // namespace ncnn diff --git a/src/layer/x86/convolution_1x1_int8.h b/src/layer/x86/convolution_1x1_int8.h index 66208649f..6e0e63d41 100644 --- a/src/layer/x86/convolution_1x1_int8.h +++ b/src/layer/x86/convolution_1x1_int8.h @@ -196,47 +196,3 @@ static void conv1x1s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat } } } - -static void conv1x1s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv1x1s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv1x1s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} - -static void conv1x1s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 1; - int kernel_h = 1; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} \ No newline at end of file diff --git a/src/layer/x86/convolution_3x3_int8.h b/src/layer/x86/convolution_3x3_int8.h index cff8d13db..2d42cb23f 100644 --- a/src/layer/x86/convolution_3x3_int8.h +++ b/src/layer/x86/convolution_3x3_int8.h @@ -909,47 +909,3 @@ static void conv3x3s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat } } } - -static void conv3x3s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv3x3s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv3x3s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} - -static void conv3x3s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 3; - int kernel_h = 3; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} diff --git a/src/layer/x86/convolution_5x5_int8.h b/src/layer/x86/convolution_5x5_int8.h deleted file mode 100644 index 6aaedf1e0..000000000 --- a/src/layer/x86/convolution_5x5_int8.h +++ /dev/null @@ -1,79 +0,0 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. -// -// Copyright (C) 2019 BUG1989. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -static void conv5x5s1_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv5x5s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv5x5s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv5x5s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv5x5s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} - -static void conv5x5s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 5; - int kernel_h = 5; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} \ No newline at end of file diff --git a/src/layer/x86/convolution_7x7_int8.h b/src/layer/x86/convolution_7x7_int8.h deleted file mode 100644 index f1adcdc37..000000000 --- a/src/layer/x86/convolution_7x7_int8.h +++ /dev/null @@ -1,79 +0,0 @@ -// BUG1989 is pleased to support the open source community by supporting ncnn available. -// -// Copyright (C) 2019 BUG1989. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -static void conv7x7s1_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv7x7s2_int8_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, opt); -} - -static void conv7x7s1_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv7x7s2_int8_dequant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_dequant, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_dequant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_dequant, opt); -} - -static void conv7x7s1_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 1; - int stride_h = 1; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} - -static void conv7x7s2_int8_requant_sse(const Mat &bottom_blob, Mat &top_blob, const Mat &_kernel, const Mat &_bias, std::vector scales_requant, const Option& opt) -{ - int kernel_w = 7; - int kernel_h = 7; - - int stride_w = 2; - int stride_h = 2; - - conv_im2col_sgemm_int8_requant_sse(bottom_blob, top_blob, _kernel, kernel_w, kernel_h, stride_w, stride_h, _bias, scales_requant, opt); -} \ No newline at end of file diff --git a/src/layer/x86/convolution_x86.cpp b/src/layer/x86/convolution_x86.cpp index 6b4297472..1cafcae4b 100644 --- a/src/layer/x86/convolution_x86.cpp +++ b/src/layer/x86/convolution_x86.cpp @@ -35,14 +35,13 @@ namespace ncnn { #include "convolution_sgemm_int8.h" #include "convolution_1x1_int8.h" #include "convolution_3x3_int8.h" -#include "convolution_5x5_int8.h" -#include "convolution_7x7_int8.h" DEFINE_LAYER_CREATOR(Convolution_x86) Convolution_x86::Convolution_x86() { activation = 0; + convolution_dilation1 = 0; } int Convolution_x86::create_pipeline(const Option& opt) @@ -84,35 +83,71 @@ int Convolution_x86::create_pipeline(const Option& opt) activation->create_pipeline(opt); } + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + { + return create_pipeline_int8_x86(opt); + } + + int kernel_size = kernel_w * kernel_h; + int num_input = weight_data_size / kernel_size / num_output; + use_winograd3x3 = false; - if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + if (kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1) { - int num_input = weight_data_size / 9 / num_output; - // winograd is slow on small channel count - if(num_input >= 16 && num_output >= 16) - use_winograd3x3 = true; - } + convolution_dilation1 = ncnn::create_layer(ncnn::LayerType::Convolution); - if (use_winograd3x3) - { - int num_input = weight_data_size / 9 / num_output; + // set param + ncnn::ParamDict pd; + pd.set(0, num_output);// num_output + pd.set(1, kernel_w); + pd.set(11, kernel_h); + pd.set(2, 1); + pd.set(12, 1); + pd.set(3, 1);// stride_w + pd.set(13, 1);// stride_h + pd.set(4, 0);// pad_w + pd.set(14, 0);// pad_h + pd.set(5, bias_term); + pd.set(6, weight_data_size); + + convolution_dilation1->load_param(pd); + + // set weights + if (bias_term) + { + ncnn::Mat weights[2]; + weights[0] = weight_data; + weights[1] = bias_data; - if (use_int8_inference) - // conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output); - conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output); + convolution_dilation1->load_model(ModelBinFromMatArray(weights)); + } else - conv3x3s1_winograd23_transform_kernel_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output); -// conv3x3s1_winograd43_transform_kernel_sse(weight_data, weight_3x3_winograd43_data, num_input, num_output); - } + { + ncnn::Mat weights[1]; + weights[0] = weight_data; + + convolution_dilation1->load_model(ModelBinFromMatArray(weights)); + } - if (use_int8_inference == false) + convolution_dilation1->create_pipeline(opt); + } + else if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 + && num_input >= 16 && num_output >= 16) { - int kernel_size = kernel_w * kernel_h; - int num_input = weight_data_size / kernel_size / num_output; + // winograd is slow on small channel count + use_winograd3x3 = true; + conv3x3s1_winograd23_transform_kernel_sse(weight_data, weight_3x3_winograd23_data, num_input, num_output); +// conv3x3s1_winograd43_transform_kernel_sse(weight_data, weight_3x3_winograd43_data, num_input, num_output); + + // for small size + conv_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_size); + } + else + { conv_im2col_sgemm_transform_kernel_sse(weight_data, weight_sgemm_data, num_input, num_output, kernel_size); - } + } return 0; } @@ -126,19 +161,43 @@ int Convolution_x86::destroy_pipeline(const Option& opt) activation = 0; } + if (convolution_dilation1) + { + convolution_dilation1->destroy_pipeline(opt); + delete convolution_dilation1; + convolution_dilation1 = 0; + } + return 0; } -int Convolution_x86::forwardDilation(const Mat& bottom_blob, Mat& top_blob, conv_func conv, const Option& opt) const +int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { + // convolv with NxN kernel + // value = value + bias + + if (bottom_blob.dims != 3) + { + return Convolution::forward(bottom_blob, top_blob, opt); + } + + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + { + return forward_int8_x86(bottom_blob, top_blob, opt); + } + + if ((dilation_w > 1 || dilation_h > 1) && (stride_w > 1 || stride_h > 1)) + { + return Convolution::forward(bottom_blob, top_blob, opt); + } + int w = bottom_blob.w; int h = bottom_blob.h; + int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; - const int kernel_size = kernel_w; - const int stride = stride_w; - const int dilation = dilation_w; - const int kernel_extent = dilation * (kernel_size - 1) + 1; + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; Mat bottom_blob_bordered = bottom_blob; if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) @@ -146,319 +205,113 @@ int Convolution_x86::forwardDilation(const Mat& bottom_blob, Mat& top_blob, conv Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { - int wpad = kernel_extent + (w - 1) / stride * stride - w; - int hpad = kernel_extent + (h - 1) / stride * stride - h; + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) { - int wpad = kernel_extent + (w - 1) / stride * stride - w; - int hpad = kernel_extent + (h - 1) / stride * stride - h; + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } + if (bottom_blob_bordered.empty()) + return -100; - int outw = (w - kernel_extent) / stride + 1; - int outh = (h - kernel_extent) / stride + 1; + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + // float32 top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); if (top_blob.empty()) return -100; - // Make (dilation * dilation) batches - Mat inner_bottom_blob; - Mat inner_top_blob; - for (int x = 0; x < dilation; x ++) + if (kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1) { - for (int y = 0; y < dilation; y ++) - { - int inner_w = (w - y + dilation - 1) / dilation; - int inner_h = (h - x + dilation - 1) / dilation; - - int inner_outw = (inner_w - kernel_size) / stride + 1; - int inner_outh = (inner_h - kernel_size) / stride + 1; - - inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator); - if (inner_bottom_blob.empty()) - return -100; - - inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator); - if (inner_top_blob.empty()) - return -100; + return forwardDilation_x86(bottom_blob_bordered, top_blob, opt); + } - #pragma omp parallel for num_threads(opt.num_threads) - for (int c = 0; c < bottom_blob.c; c ++) - { - float *outptr = inner_bottom_blob.channel(c); + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + if (use_winograd3x3 && outw >= 8 && outh >=8) + { + conv3x3s1_winograd23_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, bias_data, opt); +// conv3x3s1_winograd43_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd43_data, bias_data, opt); + } + else + { + conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); + } - for (int i = 0; i < inner_h; i ++) - { - const float* ptr = (const float *)bottom_blob_bordered.channel(c) + dilation * i * w + x * w + y; - for (int j = 0; j < inner_w; j ++) - { - outptr[j] = ptr[j*dilation]; - } - outptr += inner_w; - } - } + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + } + else + { +// conv1x1s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); +// conv1x1s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); +// conv3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); +// conv3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); +// conv5x5s1_neon(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); - Option opt_g = opt; - opt_g.blob_allocator = inner_top_blob.allocator; - if (kernel_size == 7) - { - // FIXME conv7x7s1_sse use sgemm - conv(inner_bottom_blob, inner_top_blob, weight_sgemm_data, bias_data, opt_g); - } - else - { - conv(inner_bottom_blob, inner_top_blob, weight_data, bias_data, opt_g); - } + conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); - #pragma omp parallel for num_threads(opt.num_threads) - for (int c = 0; c < num_output; c ++) - { - float *outptr = (float *)top_blob.channel(c) + x * outw + y; - for (int i = 0; i < inner_outh; i ++) - { - const float* ptr = (const float *)inner_top_blob.channel(c) + i * inner_outw; - for (int j = 0; j < inner_outw; j ++) - { - outptr[j*dilation] = ptr[j]; - } - outptr += dilation * outw; - } - } + if (activation) + { + activation->forward_inplace(top_blob, opt); } } return 0; } -int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +int Convolution_x86::create_pipeline_int8_x86(const Option& opt) { - // convolv with NxN kernel - // value = value + bias - - if (bottom_blob.dims != 3) - { - return Convolution::forward(bottom_blob, top_blob, opt); - } - - if (kernel_w != kernel_h || stride_w != stride_h) - { - return Convolution::forward(bottom_blob, top_blob, opt); - } + int kernel_size = kernel_w * kernel_h; + int num_input = weight_data_size / kernel_size / num_output; - const int kernel_size = kernel_w; - const int stride = stride_w; - - if (kernel_size > 7 || stride > 7 || dilation_w != dilation_h) - { - return Convolution::forward(bottom_blob, top_blob, opt); - } - - typedef void (*conv_func)(const Mat&, Mat&, const Mat&, const Mat&, const Option&); - - // kernel_size x stride - conv_func conv_func_table[7][4] = - { - { - conv1x1s1_sse, - conv1x1s2_sse, - 0, - 0 - }, // kernel_size = 1 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 2 - { - conv3x3s1_sse, - conv3x3s2_sse, - 0, - 0 - }, // kernel_size = 3 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 4 - { - conv5x5s1_sse, - conv5x5s2_sse, - 0, - 0 - }, // kernel_size = 5 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 6 - { - conv7x7s1_sse, - conv7x7s2_sse, - 0, - 0 - } // kernel_size = 7 - }; - - typedef void (*conv_int8_dequant_func)(const Mat&, Mat&, const Mat&, const Mat&, std::vector, const Option&); - typedef void (*conv_int8_requant_func)(const Mat&, Mat&, const Mat&, const Mat&, std::vector, const Option&); - - // kernel_size x stride - conv_int8_dequant_func conv_int8_dequant_func_table[7][4] = - { - { - conv1x1s1_int8_dequant_sse, - conv1x1s2_int8_dequant_sse, - 0, - 0 - }, // kernel_size = 1 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 2 - { - conv3x3s1_int8_dequant_sse, - conv3x3s2_int8_dequant_sse, - 0, - 0, - }, // kernel_size = 3 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 4 - { - conv5x5s1_int8_dequant_sse, - conv5x5s2_int8_dequant_sse, - 0, - 0 - }, // kernel_size = 5 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 6 - { - conv7x7s1_int8_dequant_sse, - conv7x7s2_int8_dequant_sse, - 0, - 0 - } // kernel_size = 7 - }; + use_winograd3x3_int8 = false; - conv_int8_requant_func conv_int8_requant_func_table[7][4] = + if (opt.use_winograd_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 + && num_input >= 16 && num_output >= 16) { - { - conv1x1s1_int8_requant_sse, - conv1x1s2_int8_requant_sse, - 0, - 0 - }, // kernel_size = 1 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 2 - { - conv3x3s1_int8_requant_sse, - conv3x3s2_int8_requant_sse, - 0, - 0, - }, // kernel_size = 3 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 4 - { - conv5x5s1_int8_requant_sse, - conv5x5s2_int8_requant_sse, - 0, - 0 - }, // kernel_size = 5 - { - 0, - 0, - 0, - 0 - }, // kernel_size = 6 - { - conv7x7s1_int8_requant_sse, - conv7x7s2_int8_requant_sse, - 0, - 0 - } // kernel_size = 7 - }; - - conv_func conv = 0; - conv_int8_dequant_func conv_int8_dequant = 0; - conv_int8_requant_func conv_int8_requant = 0; + // winograd is slow on small channel count + use_winograd3x3_int8 = true; - if (use_int8_inference) - { - if (use_int8_requantize) - conv_int8_requant = conv_int8_requant_func_table[kernel_size-1][stride-1]; - else - conv_int8_dequant = conv_int8_dequant_func_table[kernel_size-1][stride-1]; - if ((!conv_int8_requant) && (!conv_int8_dequant)) - { - return Convolution::forward(bottom_blob, top_blob, opt); - } + conv3x3s1_winograd23_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output); +// conv3x3s1_winograd43_transform_kernel_int8_sse(weight_data, weight_3x3_winograd23_data_int8, num_input, num_output); } else { - conv = conv_func_table[kernel_size-1][stride-1]; - if (!conv) - { - return Convolution::forward(bottom_blob, top_blob, opt); - } + // TODO offline transform weight + } - if (dilation_w != 1) - { - if (stride != 1) - return Convolution::forward(bottom_blob, top_blob, opt); + return 0; +} - return forwardDilation(bottom_blob, top_blob, conv, opt); - } +int Convolution_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + if (dilation_w > 1 || dilation_h > 1) + { + return Convolution::forward(bottom_blob, top_blob, opt); } int w = bottom_blob.w; @@ -466,23 +319,16 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + Mat bottom_blob_unbordered = bottom_blob; - if (use_int8_inference && elemsize != 1) + if (elemsize != 1) { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) - return -100; - - // quantize, scale and round to nearest - { - Option opt_g = opt; - opt_g.blob_allocator = bottom_blob_int8.allocator; + Option opt_g = opt; + opt_g.blob_allocator = opt.workspace_allocator; - quantize->forward(bottom_blob, bottom_blob_int8, opt_g); - } - - bottom_blob_unbordered = bottom_blob_int8; + quantize_float32_to_int8(bottom_blob, bottom_blob_unbordered, bottom_blob_int8_scale, opt_g); } Mat bottom_blob_bordered = bottom_blob_unbordered; @@ -491,131 +337,231 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { - int wpad = kernel_size + (w - 1) / stride * stride - w; - int hpad = kernel_size + (h - 1) / stride * stride - h; + // tensorflow padding=SAME or onnx padding=SAME_UPPER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) { - int wpad = kernel_size + (w - 1) / stride * stride - w; - int hpad = kernel_size + (h - 1) / stride * stride - h; + // onnx padding=SAME_LOWER + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; if (wpad > 0 || hpad > 0) { Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + + // int8 + size_t out_elemsize = use_int8_requantize ? 1u : 4u; - int outw = (w - kernel_size) / stride + 1; - int outh = (h - kernel_size) / stride + 1; + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; // int8 - if (use_int8_inference) - { - if (use_int8_requantize == true) + if (use_int8_requantize) + { + Mat top_blob_tm; + top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator); + if (top_blob_tm.empty()) + return -100; + + if (use_winograd3x3_int8) { - Mat top_blob_tm; - top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator); - if (top_blob_tm.empty()) - return -100; - - top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator); - if (top_blob.empty()) - return -100; + conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt); +// conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob_tm, weight_3x3_winograd23_data_int8, opt); - if (use_winograd3x3) + // requantize, reverse scale inplace + #pragma omp parallel for num_threads(opt.num_threads) + for (int p=0; pforward(top_blob_tm_g, top_blob_g, opt_g); - } + // requantize and relu + float scale_in; + if (weight_data_int8_scales[p] == 0) + scale_in = 0; + else + scale_in = 1.f / (bottom_blob_int8_scale * weight_data_int8_scales[p]); + + float scale_out = top_blob_int8_scale;//FIXME load param + + requantize_int8_to_int8(top_blob_tm, top_blob, scale_in, scale_out, &bias_data[p], bias_term ? 1 : 0, 0, opt_g); } - else - conv_int8_requant(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); } else { - top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - if (use_winograd3x3) + std::vector requantize_scales; + for (int p=0; pforward_inplace(top_blob_g, opt_g); - } + requantize_scales.push_back(scale_in); + requantize_scales.push_back(scale_out); } - else - conv_int8_dequant(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); + + conv_im2col_sgemm_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, kernel_w, kernel_h, stride_w, stride_h, bias_data, requantize_scales, opt); } + } + else + { + if (use_winograd3x3_int8) + { + conv3x3s1_winograd23_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt); +// conv3x3s1_winograd43_int8_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data_int8, opt); - if (activation) + // dequantize, reverse scale inplace + #pragma omp parallel for num_threads(opt.num_threads) + for (int p=0; pforward_inplace(top_blob, opt); - } + std::vector dequantize_scales; + for (int p=0; pforward_inplace(top_blob, opt); + } + + return 0; +} + +int Convolution_x86::forwardDilation_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + size_t elemsize = bottom_blob.elemsize; + + const int kernel_size = kernel_w; + const int stride = stride_w; + const int dilation = dilation_w; + const int kernel_extent = dilation * (kernel_size - 1) + 1; + + int outw = (w - kernel_extent) / stride + 1; + int outh = (h - kernel_extent) / stride + 1; + top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); if (top_blob.empty()) - return -100; + return -100; - if (use_winograd3x3 && outw >= 8 && outh >=8) + // Make (dilation * dilation) batches + Mat inner_bottom_blob; + Mat inner_top_blob; + for (int x = 0; x < dilation; x ++) { - conv3x3s1_winograd23_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd23_data, bias_data, opt); -// conv3x3s1_winograd43_sse(bottom_blob_bordered, top_blob, weight_3x3_winograd43_data, bias_data, opt); + for (int y = 0; y < dilation; y ++) + { + int inner_w = (w - y + dilation - 1) / dilation; + int inner_h = (h - x + dilation - 1) / dilation; + + int inner_outw = (inner_w - kernel_size) / stride + 1; + int inner_outh = (inner_h - kernel_size) / stride + 1; + + inner_bottom_blob.create(inner_w, inner_h, bottom_blob.c, elemsize, opt.workspace_allocator); + if (inner_bottom_blob.empty()) + return -100; + + inner_top_blob.create(inner_outw, inner_outh, num_output, elemsize, opt.workspace_allocator); + if (inner_top_blob.empty()) + return -100; + + #pragma omp parallel for num_threads(opt.num_threads) + for (int c = 0; c < bottom_blob.c; c ++) + { + float *outptr = inner_bottom_blob.channel(c); + + for (int i = 0; i < inner_h; i ++) + { + const float* ptr = (const float *)bottom_blob.channel(c) + dilation * i * w + x * w + y; + for (int j = 0; j < inner_w; j ++) + { + outptr[j] = ptr[j*dilation]; + } + outptr += inner_w; + } + } + + Option opt_g = opt; + opt_g.blob_allocator = inner_top_blob.allocator; + convolution_dilation1->forward(inner_bottom_blob, inner_top_blob, opt_g); + + #pragma omp parallel for num_threads(opt.num_threads) + for (int c = 0; c < num_output; c ++) + { + float *outptr = (float *)top_blob.channel(c) + x * outw + y; + for (int i = 0; i < inner_outh; i ++) + { + const float* ptr = (const float *)inner_top_blob.channel(c) + i * inner_outw; + for (int j = 0; j < inner_outw; j ++) + { + outptr[j*dilation] = ptr[j]; + } + outptr += dilation * outw; + } + } + } } - else - //conv(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); - conv_im2col_sgemm_sse(bottom_blob_bordered, top_blob, weight_sgemm_data, bias_data, kernel_w, kernel_h, stride_w, stride_h, opt); if (activation) { diff --git a/src/layer/x86/convolution_x86.h b/src/layer/x86/convolution_x86.h index 4714e3001..333e8706f 100644 --- a/src/layer/x86/convolution_x86.h +++ b/src/layer/x86/convolution_x86.h @@ -19,8 +19,6 @@ namespace ncnn { -typedef void (*conv_func)(const Mat&, Mat&, const Mat&, const Mat&, const Option&); - class Convolution_x86 : virtual public Convolution { public: @@ -30,7 +28,11 @@ public: virtual int destroy_pipeline(const Option& opt); virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; - virtual int forwardDilation(const Mat& bottom_blob, Mat &top_blob, conv_func conv, const Option& opt) const; + +protected: + int create_pipeline_int8_x86(const Option& opt); + int forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + int forwardDilation_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; public: Layer* activation; @@ -38,6 +40,13 @@ public: Mat weight_3x3_winograd23_data; Mat weight_sgemm_data; std::vector weight_3x3_winograd43_data; + + // forwardDilation + Layer* convolution_dilation1; + + // int8 + bool use_winograd3x3_int8; + Mat weight_3x3_winograd23_data_int8; }; } // namespace ncnn diff --git a/src/layer/x86/convolutiondepthwise_x86.cpp b/src/layer/x86/convolutiondepthwise_x86.cpp index 57068e622..d384fe00f 100644 --- a/src/layer/x86/convolutiondepthwise_x86.cpp +++ b/src/layer/x86/convolutiondepthwise_x86.cpp @@ -14,10 +14,6 @@ #include "convolutiondepthwise_x86.h" -#ifdef _OPENMP -#include -#endif - #include "layer_type.h" namespace ncnn { @@ -84,14 +80,16 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) if (channels == group && group == num_output) { // depth-wise specific - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) + // special path for both int8 and fp32 + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { - if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2)) - { - return 0; - } + return 0; } - } + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + return 0; + } + } const int channels_g = channels / group; const int num_output_g = num_output / group; @@ -121,6 +119,8 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) pd.set(5, bias_term); pd.set(6, maxk * channels_g * num_output_g);// weight_data_size pd.set(8, int8_scale_term); + pd.set(9, activation_type); + pd.set(10, activation_params); op->load_param(pd); @@ -155,6 +155,8 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) op->create_pipeline(opt); +// op->use_int8_requantize = use_int8_requantize; FIXME + group_ops[g] = op; } @@ -185,26 +187,125 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con // convolv with NxN kernel // value = value + bias + if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u) + { + return forward_int8_x86(bottom_blob, top_blob, opt); + } + int w = bottom_blob.w; int h = bottom_blob.h; int channels = bottom_blob.c; size_t elemsize = bottom_blob.elemsize; - if (channels % group != 0 || num_output % group != 0) + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + Mat bottom_blob_bordered = bottom_blob; + if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); + } + else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { - // reject invalid group + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) + { + int wpad = kernel_extent_w + (w - 1) / stride_w * stride_w - w; + int hpad = kernel_extent_h + (h - 1) / stride_h * stride_h - h; + if (wpad > 0 || hpad > 0) + { + Option opt_b = opt; + opt_b.blob_allocator = opt.workspace_allocator; + copy_make_border(bottom_blob, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); + } + } + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; + + int outw = (w - kernel_extent_w) / stride_w + 1; + int outh = (h - kernel_extent_h) / stride_h + 1; + + // float32 + top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); + if (top_blob.empty()) return -100; + + // depth-wise + if (channels == group && group == num_output) + { + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + + return 0; + } + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) + { + convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); + + if (activation) + { + activation->forward_inplace(top_blob, opt); + } + + return 0; + } } + // group convolution + const int channels_g = channels / group; + const int num_output_g = num_output / group; + + for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); + } + + return 0; +} + +int ConvolutionDepthWise_x86::forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; Mat bottom_blob_unbordered = bottom_blob; - if (use_int8_inference && elemsize != 1) + if (elemsize != 1) { - Mat bottom_blob_int8; - bottom_blob_int8.create(w, h, channels, (size_t)1u, opt.workspace_allocator); - if (bottom_blob_int8.empty()) + bottom_blob_unbordered.create(w, h, channels, (size_t)1u, opt.workspace_allocator); + if (bottom_blob_unbordered.empty()) return -100; const int channels_g = channels / group; @@ -215,15 +316,14 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { Option opt_g = opt; opt_g.num_threads = 1; - opt_g.blob_allocator = bottom_blob_int8.allocator; + opt_g.blob_allocator = bottom_blob_unbordered.allocator; const Mat bottom_blob_g = bottom_blob.channel_range(channels_g * g, channels_g); - Mat bottom_blob_int8_g = bottom_blob_int8.channel_range(channels_g * g, channels_g); - quantize_ops[g]->forward(bottom_blob_g, bottom_blob_int8_g, opt_g); - } + Mat bottom_blob_int8_g = bottom_blob_unbordered.channel_range(channels_g * g, channels_g); - bottom_blob_unbordered = bottom_blob_int8; - } + quantize_float32_to_int8(bottom_blob_g, bottom_blob_int8_g, bottom_blob_int8_scales[g], opt_g); + } + } Mat bottom_blob_bordered = bottom_blob_unbordered; if (pad_left > 0 || pad_right > 0 || pad_top > 0 || pad_bottom > 0) @@ -231,11 +331,6 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, pad_top, pad_bottom, pad_left, pad_right, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -233 && pad_right == -233 && pad_top == -233 && pad_bottom == -233) { @@ -246,12 +341,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } else if (pad_left == -234 && pad_right == -234 && pad_top == -234 && pad_bottom == -234) { @@ -262,234 +352,107 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con Option opt_b = opt; opt_b.blob_allocator = opt.workspace_allocator; copy_make_border(bottom_blob_unbordered, bottom_blob_bordered, hpad - hpad / 2, hpad / 2, wpad - wpad / 2, wpad / 2, BORDER_CONSTANT, pad_value, opt_b); - if (bottom_blob_bordered.empty()) - return -100; } - - w = bottom_blob_bordered.w; - h = bottom_blob_bordered.h; } + if (bottom_blob_bordered.empty()) + return -100; + + w = bottom_blob_bordered.w; + h = bottom_blob_bordered.h; int outw = (w - kernel_extent_w) / stride_w + 1; int outh = (h - kernel_extent_h) / stride_h + 1; // int8 - if (use_int8_inference) + size_t out_elemsize = use_int8_requantize ? 1u : 4u; + + top_blob.create(outw, outh, num_output, out_elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // depth-wise + if (channels == group && group == num_output) { if (use_int8_requantize) { - Mat top_blob_tm; - top_blob_tm.create(outw, outh, num_output, (size_t)4u, opt.workspace_allocator); - if (top_blob_tm.empty()) - return -100; - - top_blob.create(outw, outh, num_output, (size_t)1u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // depth-wise - if (channels == group && group == num_output) - { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) - { - if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2)) - { - if (stride_w == 1 && stride_h == 1) - { - convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); - } - else if (stride_w == 2 && stride_h == 2) - { - convdw3x3s2_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; - } - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; g requantize_scales; + for (int g=0; gforward(bottom_blob_bordered_g, top_blob_tm_g, opt_g); - } + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) + { + convdw3x3s1_int8_requant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, requantize_scales, opt); if (activation) { activation->forward_inplace(top_blob, opt); - } + } return 0; } - - const int channels_g = channels / group; - const int num_output_g = num_output / group; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward_inplace(top_blob, opt); + } - // forward - op->forward(bottom_blob_bordered_g, top_blob_tm_g, opt_g); - } + return 0; + } } else { - top_blob.create(outw, outh, num_output, (size_t)4u, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // depth-wise - if (channels == group && group == num_output) - { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) - { - if ((stride_w == 1 && stride_h == 1) || (stride_w == 2 && stride_h == 2)) - { - if (stride_w == 1 && stride_h == 1) - { - convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); - } - else if (stride_w == 2 && stride_h == 2) - { - convdw3x3s2_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; - } - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; g dequantize_scales; + for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); - } + if (kernel_w == 3 && kernel_h == 3 && stride_w == 1 && stride_h == 1 && dilation_w == 1 && dilation_h == 1) + { + convdw3x3s1_int8_dequant_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, dequantize_scales, opt); if (activation) { activation->forward_inplace(top_blob, opt); - } - + } + return 0; } - - const int channels_g = channels / group; - const int num_output_g = num_output / group; - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); - } - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - - return 0; - } - - // float32 - top_blob.create(outw, outh, num_output, elemsize, opt.blob_allocator); - if (top_blob.empty()) - return -100; - - // depth-wise - if (channels == group && group == num_output) - { - if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1) - { - if (stride_w == 1 && stride_h == 1) - { - convdw3x3s1_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); - } - else if (stride_w == 2 && stride_h == 2) - { - convdw3x3s2_sse(bottom_blob_bordered, top_blob, weight_data, bias_data, opt); - } + if (activation) + { + activation->forward_inplace(top_blob, opt); + } - if (activation) - { - activation->forward_inplace(top_blob, opt); + return 0; } - - return 0; - } - - #pragma omp parallel for num_threads(opt.num_threads) - for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); - } - - if (activation) - { - activation->forward_inplace(top_blob, opt); } - - return 0; } + // group convolution const int channels_g = channels / group; const int num_output_g = num_output / group; + #pragma omp parallel for num_threads(opt.num_threads) for (int g=0; gforward(bottom_blob_bordered_g, top_blob_g, opt_g); } - if (activation) - { - activation->forward_inplace(top_blob, opt); - } - return 0; } diff --git a/src/layer/x86/convolutiondepthwise_x86.h b/src/layer/x86/convolutiondepthwise_x86.h index cc7e57f2e..282e411b8 100644 --- a/src/layer/x86/convolutiondepthwise_x86.h +++ b/src/layer/x86/convolutiondepthwise_x86.h @@ -29,6 +29,9 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; +protected: + int forward_int8_x86(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + public: Layer* activation; std::vector group_ops; diff --git a/src/mat.cpp b/src/mat.cpp index c10802002..e402706db 100644 --- a/src/mat.cpp +++ b/src/mat.cpp @@ -24,66 +24,68 @@ #include "layer_type.h" #include "layer.h" +#if NCNN_VULKAN #if __ANDROID_API__ >= 26 #include #endif // __ANDROID_API__ >= 26 +#endif // NCNN_VULKAN namespace ncnn { void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_vals) { - ncnn::Layer* op; + Layer* op; if (mean_vals && !norm_vals) { // substract mean only - op = ncnn::create_layer(ncnn::LayerType::Bias); + op = create_layer(LayerType::Bias); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, c); op->load_param(pd); - ncnn::Mat weights[1]; + Mat weights[1]; weights[0] = Mat(c); for (int q=0; qload_model(ncnn::ModelBinFromMatArray(weights)); + op->load_model(ModelBinFromMatArray(weights)); } else if (!mean_vals && norm_vals) { // normalize only - op = ncnn::create_layer(ncnn::LayerType::Scale); + op = create_layer(LayerType::Scale); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, c); op->load_param(pd); - ncnn::Mat weights[1]; + Mat weights[1]; weights[0] = Mat(c); for (int q=0; qload_model(ncnn::ModelBinFromMatArray(weights)); + op->load_model(ModelBinFromMatArray(weights)); } else if (mean_vals && norm_vals) { // substract mean and normalize - op = ncnn::create_layer(ncnn::LayerType::Scale); + op = create_layer(LayerType::Scale); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, c); pd.set(1, 1); op->load_param(pd); - ncnn::Mat weights[2]; + Mat weights[2]; weights[0] = Mat(c); weights[1] = Mat(c); for (int q=0; qload_model(ncnn::ModelBinFromMatArray(weights)); + op->load_model(ModelBinFromMatArray(weights)); } else // if (!mean_vals && !norm_vals) { @@ -101,8 +103,13 @@ void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_val Option opt; opt.num_threads = 1;// TODO + + op->create_pipeline(opt); + op->forward_inplace(*this, opt); + op->destroy_pipeline(opt); + delete op; } @@ -226,6 +233,7 @@ Mat Mat::from_float16(const unsigned short* data, int size) return m; } +#if NCNN_VULKAN #if __ANDROID_API__ >= 26 VkImageMat VkImageMat::from_android_hardware_buffer(AHardwareBuffer* hb, VkAndroidHardwareBufferImageAllocator* allocator) { @@ -248,12 +256,13 @@ VkImageMat VkImageMat::from_android_hardware_buffer(AHardwareBuffer* hb, VkAndro return m; } #endif // __ANDROID_API__ >= 26 +#endif // NCNN_VULKAN void copy_make_border(const Mat& src, Mat& dst, int top, int bottom, int left, int right, int type, float v, const Option& opt) { - ncnn::Layer* padding = ncnn::create_layer(ncnn::LayerType::Padding); + Layer* padding = create_layer(LayerType::Padding); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, top); pd.set(1, bottom); pd.set(2, left); @@ -263,16 +272,20 @@ void copy_make_border(const Mat& src, Mat& dst, int top, int bottom, int left, i padding->load_param(pd); + padding->create_pipeline(opt); + padding->forward(src, dst, opt); + padding->destroy_pipeline(opt); + delete padding; } void copy_cut_border(const Mat& src, Mat& dst, int top, int bottom, int left, int right, const Option& opt) { - ncnn::Layer* crop = ncnn::create_layer(ncnn::LayerType::Crop); + Layer* crop = create_layer(LayerType::Crop); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, left); pd.set(1, top); pd.set(2, 0); @@ -282,85 +295,179 @@ void copy_cut_border(const Mat& src, Mat& dst, int top, int bottom, int left, in crop->load_param(pd); + crop->create_pipeline(opt); + crop->forward(src, dst, opt); + crop->destroy_pipeline(opt); + delete crop; } void resize_bilinear(const Mat& src, Mat& dst, int w, int h, const Option& opt) { - ncnn::Layer* interp = ncnn::create_layer(ncnn::LayerType::Interp); + Layer* interp = create_layer(LayerType::Interp); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, 2); pd.set(3, h); pd.set(4, w); interp->load_param(pd); + interp->create_pipeline(opt); + interp->forward(src, dst, opt); + interp->destroy_pipeline(opt); + delete interp; } void resize_bicubic(const Mat& src, Mat& dst, int w, int h, const Option& opt) { - ncnn::Layer* interp = ncnn::create_layer(ncnn::LayerType::Interp); + Layer* interp = create_layer(LayerType::Interp); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, 3); pd.set(3, h); pd.set(4, w); interp->load_param(pd); + interp->create_pipeline(opt); + interp->forward(src, dst, opt); + interp->destroy_pipeline(opt); + delete interp; } void convert_packing(const Mat& src, Mat& dst, int _elempack, const Option& opt) { - ncnn::Layer* packing = ncnn::create_layer(ncnn::LayerType::Packing); + Layer* packing = create_layer(LayerType::Packing); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, _elempack); packing->load_param(pd); + packing->create_pipeline(opt); + packing->forward(src, dst, opt); + packing->destroy_pipeline(opt); + delete packing; } void cast_float32_to_float16(const Mat& src, Mat& dst, const Option& opt) { - ncnn::Layer* cast = ncnn::create_layer(ncnn::LayerType::Cast); + Layer* cast = create_layer(LayerType::Cast); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, 1); pd.set(1, 2); cast->load_param(pd); + cast->create_pipeline(opt); + cast->forward(src, dst, opt); + cast->destroy_pipeline(opt); + delete cast; } void cast_float16_to_float32(const Mat& src, Mat& dst, const Option& opt) { - ncnn::Layer* cast = ncnn::create_layer(ncnn::LayerType::Cast); + Layer* cast = create_layer(LayerType::Cast); - ncnn::ParamDict pd; + ParamDict pd; pd.set(0, 2); pd.set(1, 1); cast->load_param(pd); + cast->create_pipeline(opt); + cast->forward(src, dst, opt); + cast->destroy_pipeline(opt); + delete cast; } +void quantize_float32_to_int8(const Mat& src, Mat& dst, float scale, const Option& opt) +{ + Layer* quantize = create_layer(LayerType::Quantize); + + ParamDict pd; + pd.set(0, scale); + + quantize->load_param(pd); + + quantize->create_pipeline(opt); + + quantize->forward(src, dst, opt); + + quantize->destroy_pipeline(opt); + + delete quantize; +} + +void dequantize_int32_to_float32(Mat& m, float scale, const float* bias, int bias_data_size, const Option& opt) +{ + Layer* dequantize = create_layer(LayerType::Dequantize); + + ParamDict pd; + pd.set(0, scale); + pd.set(1, bias ? 1 : 0); + pd.set(2, bias_data_size); + + dequantize->load_param(pd); + + Mat weights[1]; + weights[0] = Mat(bias_data_size, (void*)bias); + + dequantize->load_model(ModelBinFromMatArray(weights)); + + dequantize->create_pipeline(opt); + + dequantize->forward_inplace(m, opt); + + dequantize->destroy_pipeline(opt); + + delete dequantize; +} + +void requantize_int8_to_int8(const Mat& src, Mat& dst, float scale_in, float scale_out, const float* bias, int bias_data_size, int fusion_relu, const Option& opt) +{ + Layer* requantize = create_layer(LayerType::Requantize); + + ParamDict pd; + pd.set(0, scale_in); + pd.set(1, scale_out); + pd.set(2, bias ? 1 : 0); + pd.set(3, bias_data_size); + pd.set(4, fusion_relu); + + requantize->load_param(pd); + + Mat weights[1]; + weights[0] = Mat(bias_data_size, (void*)bias); + + requantize->load_model(ModelBinFromMatArray(weights)); + + requantize->create_pipeline(opt); + + requantize->forward(src, dst, opt); + + requantize->destroy_pipeline(opt); + + delete requantize; +} + } // namespace ncnn diff --git a/src/mat.h b/src/mat.h index ed3dcb6ac..df21f05ce 100644 --- a/src/mat.h +++ b/src/mat.h @@ -483,6 +483,9 @@ void resize_bicubic(const Mat& src, Mat& dst, int w, int h, const Option& opt = void convert_packing(const Mat& src, Mat& dst, int elempack, const Option& opt = Option()); void cast_float32_to_float16(const Mat& src, Mat& dst, const Option& opt = Option()); void cast_float16_to_float32(const Mat& src, Mat& dst, const Option& opt = Option()); +void quantize_float32_to_int8(const Mat& src, Mat& dst, float scale, const Option& opt = Option()); +void dequantize_int32_to_float32(Mat& m, float scale, const float* bias, int bias_data_size, const Option& opt = Option()); +void requantize_int8_to_int8(const Mat& src, Mat& dst, float scale_in, float scale_out, const float* bias, int bias_data_size, int fusion_relu, const Option& opt = Option()); inline Mat::Mat() : data(0), refcount(0), elemsize(0), elempack(0), allocator(0), dims(0), w(0), h(0), c(0), cstep(0) diff --git a/src/net.cpp b/src/net.cpp index 4e87d63e0..e031274e5 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -661,25 +661,21 @@ int Net::fuse_network() { ((Convolution*)layer)->use_int8_requantize = true; ((Convolution*)layer)->top_blob_int8_scale = ((Convolution*)layer_next)->bottom_blob_int8_scale; - ((Convolution*)layer)->create_requantize_op(); } else if (layer->type == "ConvolutionDepthWise" && layer_next->type == "Convolution") { ((ConvolutionDepthWise*)layer)->use_int8_requantize = true; ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((Convolution*)layer_next)->bottom_blob_int8_scale; - ((ConvolutionDepthWise*)layer)->create_requantize_op(); } else if (layer->type == "Convolution" && layer_next->type == "ConvolutionDepthWise") { ((Convolution*)layer)->use_int8_requantize = true; ((Convolution*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next)->bottom_blob_int8_scales[0]; - ((Convolution*)layer)->create_requantize_op(); } else { ((ConvolutionDepthWise*)layer)->use_int8_requantize = true; ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next)->bottom_blob_int8_scales[0]; - ((ConvolutionDepthWise*)layer)->create_requantize_op(); } } else if (layer_next->type == "ReLU") @@ -699,25 +695,21 @@ int Net::fuse_network() { ((Convolution*)layer)->use_int8_requantize = true; ((Convolution*)layer)->top_blob_int8_scale = ((Convolution*)layer_next_2)->bottom_blob_int8_scale; - ((Convolution*)layer)->create_requantize_op(); } else if (layer->type == "ConvolutionDepthWise" && layer_next_2->type == "Convolution") { ((ConvolutionDepthWise*)layer)->use_int8_requantize = true; ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((Convolution*)layer_next_2)->bottom_blob_int8_scale; - ((ConvolutionDepthWise*)layer)->create_requantize_op(); } else if (layer->type == "Convolution" && layer_next_2->type == "ConvolutionDepthWise") { ((Convolution*)layer)->use_int8_requantize = true; ((Convolution*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next_2)->bottom_blob_int8_scales[0]; - ((Convolution*)layer)->create_requantize_op(); } else { ((ConvolutionDepthWise*)layer)->use_int8_requantize = true; ((ConvolutionDepthWise*)layer)->top_blob_int8_scale = ((ConvolutionDepthWise*)layer_next_2)->bottom_blob_int8_scales[0]; - ((ConvolutionDepthWise*)layer)->create_requantize_op(); } } else if (layer_next_2->type == "Split") @@ -749,7 +741,6 @@ int Net::fuse_network() } ((Convolution*)layer)->use_int8_requantize = true; - ((Convolution*)layer)->create_requantize_op(); // fprintf(stderr, "\n"); } } diff --git a/tests/test_convolution.cpp b/tests/test_convolution.cpp index 3988f3d02..51571374c 100644 --- a/tests/test_convolution.cpp +++ b/tests/test_convolution.cpp @@ -29,13 +29,16 @@ static int test_convolution(int w, int h, int c, int outch, int kernel, int dila pd.set(5, bias);// bias_term pd.set(6, outch*c*kernel*kernel); - std::vector weights(2); + std::vector weights(bias ? 2 : 1); weights[0] = RandomMat(outch*c*kernel*kernel); - weights[1] = RandomMat(outch); + if (bias) + weights[1] = RandomMat(outch); ncnn::ModelBinFromMatArray mb(weights.data()); ncnn::Option opt; opt.num_threads = 1; + opt.use_vulkan_compute = true; + opt.use_int8_inference = false; opt.use_fp16_packed = false; opt.use_fp16_storage = false; opt.use_fp16_arithmetic = false; @@ -111,9 +114,117 @@ static int test_convolution_0() return 0; } +static int test_convolution_int8(int w, int h, int c, int outch, int kernel, int dilation, int stride, int pad, int bias) +{ + ncnn::Mat a = RandomMat(w, h, c); + + ncnn::ParamDict pd; + pd.set(0, outch);// num_output + pd.set(1, kernel);// kernel_w + pd.set(2, dilation);// dilation_w + pd.set(3, stride);// stride_w + pd.set(4, pad);// pad_w + pd.set(5, bias);// bias_term + pd.set(6, outch*c*kernel*kernel); + pd.set(8, 1);// int8_scale_term + + std::vector weights(bias ? 4 : 3); + weights[0] = RandomMat(outch*c*kernel*kernel); + if (bias) + { + weights[1] = RandomMat(outch); + weights[2] = RandomMat(outch); + weights[3] = RandomMat(1); + } + else + { + weights[1] = RandomMat(outch); + weights[2] = RandomMat(1); + } + ncnn::ModelBinFromMatArray mb(weights.data()); + + ncnn::Option opt; + opt.num_threads = 1; + opt.use_vulkan_compute = false; + opt.use_int8_inference = true; + opt.use_fp16_packed = false; + opt.use_fp16_storage = false; + opt.use_fp16_arithmetic = false; + opt.use_int8_storage = false; + opt.use_int8_arithmetic = false; + opt.use_packing_layout = false; + + int ret = test_layer("Convolution", pd, mb, opt, a); + if (ret != 0) + { + fprintf(stderr, "test_convolution_int8 failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias); + } + + return 0; +} + +static int test_convolution_1() +{ + static const int kdsp[24][4] = { + {1, 1, 1, 0}, + {1, 1, 2, 0}, + {2, 1, 1, 1}, + {2, 1, 2, 1}, + {2, 2, 1, 1}, + {2, 2, 2, 1}, + {3, 1, 1, 1}, + {3, 1, 2, 1}, + {3, 2, 1, 1}, + {3, 2, 2, 1}, + {4, 1, 1, 2}, + {4, 1, 2, 2}, + {4, 2, 1, 2}, + {4, 2, 2, 2}, + {5, 1, 1, 2}, + {5, 1, 2, 2}, + {5, 2, 1, 2}, + {5, 2, 2, 2}, + {7, 1, 1, 3}, + {7, 1, 2, 3}, + {7, 1, 3, 3}, + {7, 2, 1, 3}, + {7, 2, 2, 3}, + {7, 2, 3, 3}, + }; + + for (int i=0; i<24; i++) + { + int ret = 0 + || test_convolution_int8(13, 11, 1, 1, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 2, 2, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 3, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 4, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 7, 7, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 8, 8, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 15, 15, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 16, 16, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + + || test_convolution_int8(13, 11, 1, 1, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 2, 2, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 3, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 3, 12, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 4, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 8, 3, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 8, 8, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 16, 4, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + || test_convolution_int8(13, 11, 16, 16, kdsp[i][0], kdsp[i][1], kdsp[i][2], kdsp[i][3], 1) + ; + + if (ret != 0) + return -1; + } + + return 0; +} + int main() { SRAND(7767517); - return test_convolution_0(); + return test_convolution_0() || test_convolution_1(); } diff --git a/tests/test_convolutiondepthwise.cpp b/tests/test_convolutiondepthwise.cpp index 50433a4fa..fd8049507 100644 --- a/tests/test_convolutiondepthwise.cpp +++ b/tests/test_convolutiondepthwise.cpp @@ -37,6 +37,8 @@ static int test_convolutiondepthwise(int w, int h, int c, int outch, int kernel, ncnn::Option opt; opt.num_threads = 1; + opt.use_vulkan_compute = true; + opt.use_int8_inference = false; opt.use_fp16_packed = false; opt.use_fp16_storage = false; opt.use_fp16_arithmetic = false; diff --git a/tests/test_deconvolution.cpp b/tests/test_deconvolution.cpp index 970894790..59245e364 100644 --- a/tests/test_deconvolution.cpp +++ b/tests/test_deconvolution.cpp @@ -36,6 +36,7 @@ static int test_deconvolution(int w, int h, int c, int outch, int kernel, int di ncnn::Option opt; opt.num_threads = 1; + opt.use_vulkan_compute = true; opt.use_fp16_packed = false; opt.use_fp16_storage = false; opt.use_fp16_arithmetic = false; diff --git a/tests/test_deconvolutiondepthwise.cpp b/tests/test_deconvolutiondepthwise.cpp index 8e2c45976..0af48ca42 100644 --- a/tests/test_deconvolutiondepthwise.cpp +++ b/tests/test_deconvolutiondepthwise.cpp @@ -37,6 +37,7 @@ static int test_deconvolutiondepthwise(int w, int h, int c, int outch, int kerne ncnn::Option opt; opt.num_threads = 1; + opt.use_vulkan_compute = true; opt.use_fp16_packed = false; opt.use_fp16_storage = false; opt.use_fp16_arithmetic = false; diff --git a/tests/test_relu.cpp b/tests/test_relu.cpp index 895415b38..d84dedf63 100644 --- a/tests/test_relu.cpp +++ b/tests/test_relu.cpp @@ -28,6 +28,7 @@ static int test_relu(float slope, bool use_packing_layout) ncnn::Option opt; opt.num_threads = 1; + opt.use_vulkan_compute = true; opt.use_fp16_packed = false; opt.use_fp16_storage = false; opt.use_fp16_arithmetic = false; diff --git a/tests/testutil.h b/tests/testutil.h index 2d8024ce6..2144fb651 100644 --- a/tests/testutil.h +++ b/tests/testutil.h @@ -168,7 +168,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m ncnn::VkBlobBufferAllocator g_blob_vkallocator(vkdev); ncnn::VkStagingBufferAllocator g_staging_vkallocator(vkdev); - opt.use_vulkan_compute = true; opt.blob_vkallocator = &g_blob_vkallocator; opt.workspace_vkallocator = &g_blob_vkallocator; opt.staging_vkallocator = &g_staging_vkallocator; @@ -193,6 +192,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m op->create_pipeline(opt); #if NCNN_VULKAN + if (opt.use_vulkan_compute) { ncnn::VkTransfer cmd(vkdev); cmd.weight_vkallocator = &g_weight_vkallocator; @@ -240,6 +240,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m #if NCNN_VULKAN std::vector d(top_blob_count); + if (opt.use_vulkan_compute) { // pack std::vector a4(a.size()); @@ -335,7 +336,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m } #if NCNN_VULKAN - if (CompareMat(b, d, epsilon) != 0) + if (opt.use_vulkan_compute && CompareMat(b, d, epsilon) != 0) { fprintf(stderr, "test_layer failed gpu\n"); return -1; @@ -362,7 +363,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m ncnn::VkBlobBufferAllocator g_blob_vkallocator(vkdev); ncnn::VkStagingBufferAllocator g_staging_vkallocator(vkdev); - opt.use_vulkan_compute = true; opt.blob_vkallocator = &g_blob_vkallocator; opt.workspace_vkallocator = &g_blob_vkallocator; opt.staging_vkallocator = &g_staging_vkallocator; @@ -380,6 +380,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m op->create_pipeline(opt); #if NCNN_VULKAN + if (opt.use_vulkan_compute) { ncnn::VkTransfer cmd(vkdev); cmd.weight_vkallocator = &g_weight_vkallocator; @@ -423,6 +424,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m #if NCNN_VULKAN ncnn::Mat d; + if (opt.use_vulkan_compute) { // pack ncnn::Mat a4; @@ -497,7 +499,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const ncnn::ModelBin& m } #if NCNN_VULKAN - if (CompareMat(b, d, epsilon) != 0) + if (opt.use_vulkan_compute && CompareMat(b, d, epsilon) != 0) { fprintf(stderr, "test_layer failed gpu\n"); return -1;