test activation fusion

src/layer/arm/convolution_arm.cpp

@@ -491,7 +491,10 @@ int Convolution_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 
     if (opt.use_packing_layout == false && kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
+        if (outw >= dilation_w && outh >= dilation_h)
+        {
         return forwardDilation_arm(bottom_blob_bordered, top_blob, opt);
+        }
     }
 
     const int maxk = kernel_w * kernel_h;

src/layer/vulkan/convolution_vulkan.cpp

@@ -180,7 +180,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
     specializations[5].i = stride_h;
     specializations[6].i = bias_term;
     specializations[7].i = activation_type;
-    specializations[8].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+    specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
     specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f;
     specializations[10 + 0].i = shape_bordered_packed.dims;
     specializations[10 + 1].i = shape_bordered_packed.w;
@@ -215,7 +215,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
             std::vector<vk_specialization_type> specializations(4 + 8);
             specializations[0].i = bias_term;
             specializations[1].i = activation_type;
-            specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+            specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
             specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
             specializations[4 + 0].i = shape_bordered_packed.dims;
             specializations[4 + 1].i = shape_bordered_packed.cstep / 4;
@@ -247,7 +247,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
             std::vector<vk_specialization_type> specializations(4 + 8);
             specializations[0].i = bias_term;
             specializations[1].i = activation_type;
-            specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+            specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
             specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
             specializations[4 + 0].i = shape_bordered_packed.dims;
             specializations[4 + 1].i = shape_bordered_packed.w * shape_bordered_packed.h;
@@ -374,7 +374,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
                 std::vector<vk_specialization_type> specializations(4 + 7);
                 specializations[0].i = bias_term;
                 specializations[1].i = activation_type;
-                specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+                specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
                 specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
                 specializations[4 + 0].i = shape_winograd_gemm_packed.c;
                 specializations[4 + 1].i = shape_winograd_gemm_packed.cstep;
@@ -424,7 +424,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
             std::vector<vk_specialization_type> specializations(4 + 8);
             specializations[0].i = bias_term;
             specializations[1].i = activation_type;
-            specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+            specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
             specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
             specializations[4 + 0].i = shape_bordered_packed.dims;
             specializations[4 + 1].i = shape_bordered_packed.w * shape_bordered_packed.h;
@@ -551,7 +551,7 @@ int Convolution_vulkan::create_pipeline(const Option& opt)
                 std::vector<vk_specialization_type> specializations(4 + 7);
                 specializations[0].i = bias_term;
                 specializations[1].i = activation_type;
-                specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+                specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
                 specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
                 specializations[4 + 0].i = shape_winograd_gemm_packed.c;
                 specializations[4 + 1].i = shape_winograd_gemm_packed.cstep;

src/layer/vulkan/convolutiondepthwise_vulkan.cpp

@@ -138,7 +138,7 @@ int ConvolutionDepthWise_vulkan::create_pipeline(const Option& opt)
     specializations[6].i = bias_term;
     specializations[7].i = group;
     specializations[8].i = activation_type;
-    specializations[9].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+    specializations[9].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
     specializations[10].f = activation_params.w == 2 ? activation_params[1] : 0.f;
 
     // depth-wise

src/layer/vulkan/deconvolution_vulkan.cpp

@@ -165,7 +165,7 @@ int Deconvolution_vulkan::create_pipeline(const Option& opt)
     specializations[5].i = stride_h;
     specializations[6].i = bias_term;
     specializations[7].i = activation_type;
-    specializations[8].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+    specializations[8].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
     specializations[9].f = activation_params.w == 2 ? activation_params[1] : 0.f;
     specializations[10 + 0].i = shape_packed.dims;
     specializations[10 + 1].i = shape_packed.w;

src/layer/vulkan/deconvolutiondepthwise_vulkan.cpp

@@ -172,7 +172,7 @@ int DeconvolutionDepthWise_vulkan::create_pipeline(const Option& opt)
     specializations[6].i = bias_term;
     specializations[7].i = group;
     specializations[8].i = activation_type;
-    specializations[9].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+    specializations[9].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
     specializations[10].f = activation_params.w == 2 ? activation_params[1] : 0.f;
 
     // depth-wise

src/layer/vulkan/innerproduct_vulkan.cpp

@@ -97,7 +97,7 @@ int InnerProduct_vulkan::create_pipeline(const Option& opt)
     std::vector<vk_specialization_type> specializations(4 + 10);
     specializations[0].i = bias_term;
     specializations[1].i = activation_type;
-    specializations[2].f = activation_params.w == 1 ? activation_params[0] : 0.f;
+    specializations[2].f = activation_params.w >= 1 ? activation_params[0] : 0.f;
     specializations[3].f = activation_params.w == 2 ? activation_params[1] : 0.f;
     specializations[4 + 0].i = shape_flatten_packed.dims;
     specializations[4 + 1].i = shape_flatten_packed.w;

src/layer/x86/convolution_x86.cpp

@@ -228,6 +228,11 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option
 
     if (kernel_w == kernel_h && dilation_w != 1 && dilation_h == dilation_w && stride_w == 1 && stride_h == 1)
     {
+        if (outw < dilation_w || outh < dilation_h)
+        {
+            return Convolution::forward(bottom_blob, top_blob, opt);
+        }
+
         return forwardDilation_x86(bottom_blob_bordered, top_blob, opt);
     }
 

tests/test_convolution.cpp

@@ -29,6 +29,13 @@ static int test_convolution(int w, int h, int c, int outch, int kernel, int dila
     pd.set(5, bias);// bias_term
     pd.set(6, outch*c*kernel*kernel);
 
+    int activation_type = RAND() % 5;// 0 1 2 3 4
+    ncnn::Mat activation_params(2);
+    activation_params[0] = RandomFloat(-1, 0);// alpha
+    activation_params[1] = RandomFloat(0, 1);// beta
+    pd.set(9, activation_type);
+    pd.set(10, activation_params);
+
     std::vector<ncnn::Mat> weights(bias ? 2 : 1);
     weights[0] = RandomMat(outch*c*kernel*kernel);
     if (bias)
@@ -42,7 +49,7 @@ static int test_convolution(int w, int h, int c, int outch, int kernel, int dila
     int ret = test_layer<ncnn::Convolution>("Convolution", pd, weights, opt, a);
     if (ret != 0)
     {
-        fprintf(stderr, "test_convolution failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias);
+        fprintf(stderr, "test_convolution failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d act=%d actparams=[%f,%f]\n", w, h, c, outch, kernel, dilation, stride, pad, bias, activation_type, activation_params[0], activation_params[1]);
     }
 
     return ret;

tests/test_convolutiondepthwise.cpp

@@ -30,6 +30,13 @@ static int test_convolutiondepthwise(int w, int h, int c, int outch, int kernel,
     pd.set(6, outch/group*c/group*kernel*kernel*group);
     pd.set(7, group);
 
+    int activation_type = RAND() % 5;// 0 1 2 3 4
+    ncnn::Mat activation_params(2);
+    activation_params[0] = RandomFloat(-1, 0);// alpha
+    activation_params[1] = RandomFloat(0, 1);// beta
+    pd.set(9, activation_type);
+    pd.set(10, activation_params);
+
     std::vector<ncnn::Mat> weights(2);
     weights[0] = RandomMat(outch/group*c/group*kernel*kernel*group);
     weights[1] = RandomMat(outch);
@@ -42,7 +49,7 @@ static int test_convolutiondepthwise(int w, int h, int c, int outch, int kernel,
     int ret = test_layer<ncnn::ConvolutionDepthWise>("ConvolutionDepthWise", pd, weights, opt, a);
     if (ret != 0)
     {
-        fprintf(stderr, "test_convolutiondepthwise failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d group=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias, group);
+        fprintf(stderr, "test_convolutiondepthwise failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d group=%d act=%d actparams=[%f,%f]\n", w, h, c, outch, kernel, dilation, stride, pad, bias, group, activation_type, activation_params[0], activation_params[1]);
     }
 
     return ret;

tests/test_deconvolution.cpp

@@ -29,6 +29,13 @@ static int test_deconvolution(int w, int h, int c, int outch, int kernel, int di
     pd.set(5, bias);// bias_term
     pd.set(6, outch*c*kernel*kernel);
 
+    int activation_type = RAND() % 5;// 0 1 2 3 4
+    ncnn::Mat activation_params(2);
+    activation_params[0] = RandomFloat(-1, 0);// alpha
+    activation_params[1] = RandomFloat(0, 1);// beta
+    pd.set(9, activation_type);
+    pd.set(10, activation_params);
+
     std::vector<ncnn::Mat> weights(2);
     weights[0] = RandomMat(outch*c*kernel*kernel);
     weights[1] = RandomMat(outch);
@@ -41,7 +48,7 @@ static int test_deconvolution(int w, int h, int c, int outch, int kernel, int di
     int ret = test_layer<ncnn::Deconvolution>("Deconvolution", pd, weights, opt, a);
     if (ret != 0)
     {
-        fprintf(stderr, "test_deconvolution failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias);
+        fprintf(stderr, "test_deconvolution failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d act=%d actparams=[%f,%f]\n", w, h, c, outch, kernel, dilation, stride, pad, bias, activation_type, activation_params[0], activation_params[1]);
     }
 
     return ret;

tests/test_deconvolutiondepthwise.cpp

@@ -30,6 +30,13 @@ static int test_deconvolutiondepthwise(int w, int h, int c, int outch, int kerne
     pd.set(6, outch/group*c/group*kernel*kernel*group);
     pd.set(7, group);
 
+    int activation_type = RAND() % 5;// 0 1 2 3 4
+    ncnn::Mat activation_params(2);
+    activation_params[0] = RandomFloat(-1, 0);// alpha
+    activation_params[1] = RandomFloat(0, 1);// beta
+    pd.set(9, activation_type);
+    pd.set(10, activation_params);
+
     std::vector<ncnn::Mat> weights(2);
     weights[0] = RandomMat(outch/group*c/group*kernel*kernel*group);
     weights[1] = RandomMat(outch);
@@ -42,7 +49,7 @@ static int test_deconvolutiondepthwise(int w, int h, int c, int outch, int kerne
     int ret = test_layer<ncnn::DeconvolutionDepthWise>("DeconvolutionDepthWise", pd, weights, opt, a);
     if (ret != 0)
     {
-        fprintf(stderr, "test_deconvolutiondepthwise failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d group=%d\n", w, h, c, outch, kernel, dilation, stride, pad, bias, group);
+        fprintf(stderr, "test_deconvolutiondepthwise failed w=%d h=%d c=%d outch=%d kernel=%d dilation=%d stride=%d pad=%d bias=%d group=%d act=%d actparams=[%f,%f]\n", w, h, c, outch, kernel, dilation, stride, pad, bias, group, activation_type, activation_params[0], activation_params[1]);
     }
 
     return ret;

tests/test_innerproduct.cpp

@@ -23,6 +23,13 @@ static int test_innerproduct(const ncnn::Mat& a, int outch, int bias)
     pd.set(1, bias);// bias_term
     pd.set(2, outch*a.w*a.h*a.c);
 
+    int activation_type = RAND() % 5;// 0 1 2 3 4
+    ncnn::Mat activation_params(2);
+    activation_params[0] = RandomFloat(-1, 0);// alpha
+    activation_params[1] = RandomFloat(0, 1);// beta
+    pd.set(9, activation_type);
+    pd.set(10, activation_params);
+
     std::vector<ncnn::Mat> weights(bias ? 2 : 1);
     weights[0] = RandomMat(outch*a.w*a.h*a.c);
     if (bias)
@@ -36,7 +43,7 @@ static int test_innerproduct(const ncnn::Mat& a, int outch, int bias)
     int ret = test_layer<ncnn::InnerProduct>("InnerProduct", pd, weights, opt, a);
     if (ret != 0)
     {
-        fprintf(stderr, "test_innerproduct failed a.dims=%d a=(%d %d %d) outch=%d bias=%d\n", a.dims, a.w, a.h, a.c, outch, bias);
+        fprintf(stderr, "test_innerproduct failed a.dims=%d a=(%d %d %d) outch=%d bias=%d act=%d actparams=[%f,%f]\n", a.dims, a.w, a.h, a.c, outch, bias, activation_type, activation_params[0], activation_params[1]);
     }
 
     return ret;