From eea3fc9b41e895097010aff9301935a9d003c882 Mon Sep 17 00:00:00 2001 From: nihui Date: Fri, 1 Dec 2023 19:23:41 +0800 Subject: [PATCH] optimize vulkan global pooling (#5191) Co-authored-by: nihui Co-authored-by: michaelcai --- src/layer/vulkan/pooling_vulkan.cpp | 432 ++++++++++++++---- src/layer/vulkan/pooling_vulkan.h | 14 +- src/layer/vulkan/shader/pooling_global.comp | 126 ----- .../vulkan/shader/pooling_global_pack4.comp | 126 ----- .../vulkan/shader/pooling_global_pack8.comp | 134 ------ .../shader/pooling_global_reduce_max.comp | 84 ++++ .../pooling_global_reduce_max_first.comp | 88 ++++ ...pooling_global_reduce_max_first_pack4.comp | 88 ++++ ...pooling_global_reduce_max_first_pack8.comp | 90 ++++ .../pooling_global_reduce_max_last.comp | 75 +++ .../pooling_global_reduce_max_last_pack4.comp | 75 +++ .../pooling_global_reduce_max_last_pack8.comp | 77 ++++ .../pooling_global_reduce_max_pack4.comp | 84 ++++ .../pooling_global_reduce_max_pack8.comp | 86 ++++ .../shader/pooling_global_reduce_sum.comp | 82 ++++ .../pooling_global_reduce_sum_first.comp | 86 ++++ ...pooling_global_reduce_sum_first_pack4.comp | 86 ++++ ...pooling_global_reduce_sum_first_pack8.comp | 89 ++++ .../pooling_global_reduce_sum_last.comp | 75 +++ .../pooling_global_reduce_sum_last_pack4.comp | 75 +++ .../pooling_global_reduce_sum_last_pack8.comp | 81 ++++ .../pooling_global_reduce_sum_pack4.comp | 82 ++++ .../pooling_global_reduce_sum_pack8.comp | 86 ++++ tests/test_pooling.cpp | 4 + 24 files changed, 1843 insertions(+), 482 deletions(-) delete mode 100644 src/layer/vulkan/shader/pooling_global.comp delete mode 100644 src/layer/vulkan/shader/pooling_global_pack4.comp delete mode 100644 src/layer/vulkan/shader/pooling_global_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_first.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_first_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_first_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_last.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_last_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_last_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_max_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_first.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_last.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack8.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_pack4.comp create mode 100644 src/layer/vulkan/shader/pooling_global_reduce_sum_pack8.comp diff --git a/src/layer/vulkan/pooling_vulkan.cpp b/src/layer/vulkan/pooling_vulkan.cpp index 3d20a0d5b..eeba214cc 100644 --- a/src/layer/vulkan/pooling_vulkan.cpp +++ b/src/layer/vulkan/pooling_vulkan.cpp @@ -30,12 +30,20 @@ Pooling_vulkan::Pooling_vulkan() pipeline_pooling = 0; pipeline_pooling_pack4 = 0; pipeline_pooling_pack8 = 0; - pipeline_pooling_global = 0; - pipeline_pooling_global_pack4 = 0; - pipeline_pooling_global_pack8 = 0; + pipeline_pooling_adaptive = 0; pipeline_pooling_adaptive_pack4 = 0; pipeline_pooling_adaptive_pack8 = 0; + + pipeline_pooling_global_reduce_first = 0; + pipeline_pooling_global_reduce_first_pack4 = 0; + pipeline_pooling_global_reduce_first_pack8 = 0; + pipeline_pooling_global_reduce = 0; + pipeline_pooling_global_reduce_pack4 = 0; + pipeline_pooling_global_reduce_pack8 = 0; + pipeline_pooling_global_reduce_last = 0; + pipeline_pooling_global_reduce_last_pack4 = 0; + pipeline_pooling_global_reduce_last_pack8 = 0; } int Pooling_vulkan::create_pipeline(const Option& _opt) @@ -156,49 +164,129 @@ int Pooling_vulkan::create_pipeline(const Option& _opt) if (global_pooling) { - std::vector specializations(1 + 10); - specializations[0].i = pooling_type; - specializations[1 + 0].i = shape_bordered_packed.dims; - specializations[1 + 1].i = shape_bordered_packed.w; - specializations[1 + 2].i = shape_bordered_packed.h; - specializations[1 + 3].i = shape_bordered_packed.c; - specializations[1 + 4].i = shape_bordered_packed.cstep; - specializations[1 + 5].i = out_shape_packed.dims; - specializations[1 + 6].i = out_shape_packed.w; - specializations[1 + 7].i = out_shape_packed.h; - specializations[1 + 8].i = out_shape_packed.c; - specializations[1 + 9].i = out_shape_packed.cstep; - - Mat local_size_xyz(64, 1, 1, (void*)0); - if (out_shape_packed.dims != 0) + // reduce first { - local_size_xyz.w = std::min(64, out_shape_packed.w); - local_size_xyz.h = 1; - local_size_xyz.c = 1; - } + std::vector specializations(6); + specializations[0].i = shape_bordered_packed.w; + specializations[1].i = shape_bordered_packed.h; + specializations[2].i = shape_bordered_packed.c; + specializations[3].i = shape_bordered_packed.cstep; + specializations[4].i = 0; + specializations[5].i = 0; + + Mat local_size_xyz(64, 1, 1, (void*)0); + + // pack1 + if (shape.dims == 0 || elempack == 1) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_first : LayerShaderType::pooling_global_reduce_sum_first; - // pack1 - if (shape.dims == 0 || elempack == 1) - { - pipeline_pooling_global = new Pipeline(vkdev); - pipeline_pooling_global->set_optimal_local_size_xyz(local_size_xyz); - pipeline_pooling_global->create(LayerShaderType::pooling_global, opt, specializations); + pipeline_pooling_global_reduce_first = new Pipeline(vkdev); + pipeline_pooling_global_reduce_first->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_first->create(layer_shader_type, opt, specializations); + } + + // pack4 + if (shape.dims == 0 || elempack == 4) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_first_pack4 : LayerShaderType::pooling_global_reduce_sum_first_pack4; + + pipeline_pooling_global_reduce_first_pack4 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_first_pack4->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_first_pack4->create(layer_shader_type, opt, specializations); + } + + // pack8 + if ((opt.use_shader_pack8 && shape.dims == 0) || elempack == 8) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_first_pack8 : LayerShaderType::pooling_global_reduce_sum_first_pack8; + + pipeline_pooling_global_reduce_first_pack8 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_first_pack8->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_first_pack8->create(layer_shader_type, opt, specializations); + } } - // pack4 - if (shape.dims == 0 || elempack == 4) + // reduce more { - pipeline_pooling_global_pack4 = new Pipeline(vkdev); - pipeline_pooling_global_pack4->set_optimal_local_size_xyz(local_size_xyz); - pipeline_pooling_global_pack4->create(LayerShaderType::pooling_global_pack4, opt, specializations); + std::vector specializations(5); + specializations[0].i = 0; + specializations[1].i = shape_bordered_packed.c; + specializations[2].i = 0; + specializations[3].i = 0; + specializations[4].i = 0; + + Mat local_size_xyz(64, 1, 1, (void*)0); + + // pack1 + if (shape.dims == 0 || elempack == 1) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max : LayerShaderType::pooling_global_reduce_sum; + + pipeline_pooling_global_reduce = new Pipeline(vkdev); + pipeline_pooling_global_reduce->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce->create(layer_shader_type, opt, specializations); + } + + // pack4 + if (shape.dims == 0 || elempack == 4) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_pack4 : LayerShaderType::pooling_global_reduce_sum_pack4; + + pipeline_pooling_global_reduce_pack4 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_pack4->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_pack4->create(layer_shader_type, opt, specializations); + } + + // pack8 + if ((opt.use_shader_pack8 && shape.dims == 0) || elempack == 8) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_pack8 : LayerShaderType::pooling_global_reduce_sum_pack8; + + pipeline_pooling_global_reduce_pack8 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_pack8->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_pack8->create(layer_shader_type, opt, specializations); + } } - // pack8 - if ((opt.use_shader_pack8 && shape.dims == 0) || elempack == 8) + // reduce last { - pipeline_pooling_global_pack8 = new Pipeline(vkdev); - pipeline_pooling_global_pack8->set_optimal_local_size_xyz(local_size_xyz); - pipeline_pooling_global_pack8->create(LayerShaderType::pooling_global_pack8, opt, specializations); + std::vector specializations(3); + specializations[0].i = 0; + specializations[1].i = shape_bordered_packed.c; + specializations[2].i = 0; + + Mat local_size_xyz(1, 1, 64, (void*)0); + + // pack1 + if (shape.dims == 0 || elempack == 1) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_last : LayerShaderType::pooling_global_reduce_sum_last; + + pipeline_pooling_global_reduce_last = new Pipeline(vkdev); + pipeline_pooling_global_reduce_last->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_last->create(layer_shader_type, opt, specializations); + } + + // pack4 + if (shape.dims == 0 || elempack == 4) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_last_pack4 : LayerShaderType::pooling_global_reduce_sum_last_pack4; + + pipeline_pooling_global_reduce_last_pack4 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_last_pack4->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_last_pack4->create(layer_shader_type, opt, specializations); + } + + // pack8 + if ((opt.use_shader_pack8 && shape.dims == 0) || elempack == 8) + { + int layer_shader_type = pooling_type == 0 ? LayerShaderType::pooling_global_reduce_max_last_pack8 : LayerShaderType::pooling_global_reduce_sum_last_pack8; + + pipeline_pooling_global_reduce_last_pack8 = new Pipeline(vkdev); + pipeline_pooling_global_reduce_last_pack8->set_optimal_local_size_xyz(local_size_xyz); + pipeline_pooling_global_reduce_last_pack8->create(layer_shader_type, opt, specializations); + } } } else if (adaptive_pooling) @@ -331,15 +419,6 @@ int Pooling_vulkan::destroy_pipeline(const Option& _opt) delete pipeline_pooling_pack8; pipeline_pooling_pack8 = 0; - delete pipeline_pooling_global; - pipeline_pooling_global = 0; - - delete pipeline_pooling_global_pack4; - pipeline_pooling_global_pack4 = 0; - - delete pipeline_pooling_global_pack8; - pipeline_pooling_global_pack8 = 0; - delete pipeline_pooling_adaptive; pipeline_pooling_adaptive = 0; @@ -349,6 +428,33 @@ int Pooling_vulkan::destroy_pipeline(const Option& _opt) delete pipeline_pooling_adaptive_pack8; pipeline_pooling_adaptive_pack8 = 0; + delete pipeline_pooling_global_reduce_first; + pipeline_pooling_global_reduce_first = 0; + + delete pipeline_pooling_global_reduce_first_pack4; + pipeline_pooling_global_reduce_first_pack4 = 0; + + delete pipeline_pooling_global_reduce_first_pack8; + pipeline_pooling_global_reduce_first_pack8 = 0; + + delete pipeline_pooling_global_reduce; + pipeline_pooling_global_reduce = 0; + + delete pipeline_pooling_global_reduce_pack4; + pipeline_pooling_global_reduce_pack4 = 0; + + delete pipeline_pooling_global_reduce_pack8; + pipeline_pooling_global_reduce_pack8 = 0; + + delete pipeline_pooling_global_reduce_last; + pipeline_pooling_global_reduce_last = 0; + + delete pipeline_pooling_global_reduce_last_pack4; + pipeline_pooling_global_reduce_last_pack4 = 0; + + delete pipeline_pooling_global_reduce_last_pack8; + pipeline_pooling_global_reduce_last_pack8 = 0; + return 0; } @@ -372,31 +478,101 @@ int Pooling_vulkan::forward(const VkMat& bottom_blob, VkMat& top_blob, VkCompute if (global_pooling) { - top_blob.create(channels, elemsize, elempack, opt.blob_vkallocator); - if (top_blob.empty()) - return -100; - - std::vector bindings(2); - bindings[0] = bottom_blob; - bindings[1] = top_blob; - - std::vector constants(10); - constants[0].i = bottom_blob.dims; - constants[1].i = bottom_blob.w; - constants[2].i = bottom_blob.h; - constants[3].i = bottom_blob.c; - constants[4].i = bottom_blob.cstep; - constants[5].i = top_blob.dims; - constants[6].i = top_blob.w; - constants[7].i = top_blob.h; - constants[8].i = top_blob.c; - constants[9].i = top_blob.cstep; + // reduce first + VkMat reduced_blob; + { + int reduced_size = (w * h + 7) / 8; + size_t reduced_elemsize = pooling_type == 0 ? elemsize : 4u * elempack; + reduced_blob.create(reduced_size, 1, channels, reduced_elemsize, elempack, opt.workspace_vkallocator); + if (reduced_blob.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = bottom_blob; + bindings[1] = reduced_blob; + + std::vector constants(6); + constants[0].i = bottom_blob.w; + constants[1].i = bottom_blob.h; + constants[2].i = bottom_blob.c; + constants[3].i = bottom_blob.cstep; + constants[4].i = reduced_blob.w; + constants[5].i = reduced_blob.cstep; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_first_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_first_pack4 + : pipeline_pooling_global_reduce_first; + + VkMat dispatcher; + dispatcher.w = reduced_blob.w; + dispatcher.h = 1; + dispatcher.c = bottom_blob.c; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + } - const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_pack8 - : elempack == 4 ? pipeline_pooling_global_pack4 - : pipeline_pooling_global; + // reduce more + while (reduced_blob.w > 32) + { + int reduced_size = (reduced_blob.w + 7) / 8; + size_t reduced_elemsize = pooling_type == 0 ? elemsize : 4u * elempack; + VkMat reduced_blob2; + reduced_blob2.create(reduced_size, 1, channels, reduced_elemsize, elempack, opt.workspace_vkallocator); + if (reduced_blob2.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = reduced_blob; + bindings[1] = reduced_blob2; + + std::vector constants(5); + constants[0].i = reduced_blob.w; + constants[1].i = reduced_blob.c; + constants[2].i = reduced_blob.cstep; + constants[3].i = reduced_blob2.w; + constants[4].i = reduced_blob2.cstep; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_pack4 + : pipeline_pooling_global_reduce; + + VkMat dispatcher; + dispatcher.w = reduced_blob2.w; + dispatcher.h = 1; + dispatcher.c = reduced_blob2.c; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + + reduced_blob = reduced_blob2; + } - cmd.record_pipeline(pipeline, bindings, constants, top_blob); + // reduce last + { + top_blob.create(channels, elemsize, elempack, opt.blob_vkallocator); + if (top_blob.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = reduced_blob; + bindings[1] = top_blob; + + std::vector constants(4); + constants[0].i = reduced_blob.w; + constants[1].i = reduced_blob.c; + constants[2].i = reduced_blob.cstep; + constants[3].i = w * h; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_last_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_last_pack4 + : pipeline_pooling_global_reduce_last; + + VkMat dispatcher; + dispatcher.w = 1; + dispatcher.h = 1; + dispatcher.c = top_blob.w; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + } return 0; } @@ -588,31 +764,101 @@ int Pooling_vulkan::forward(const VkImageMat& bottom_blob, VkImageMat& top_blob, if (global_pooling) { - top_blob.create(channels, elemsize, elempack, opt.blob_vkallocator); - if (top_blob.empty()) - return -100; - - std::vector bindings(2); - bindings[0] = bottom_blob; - bindings[1] = top_blob; - - std::vector constants(10); - constants[0].i = bottom_blob.dims; - constants[1].i = bottom_blob.w; - constants[2].i = bottom_blob.h; - constants[3].i = bottom_blob.c; - constants[4].i = 0; //bottom_blob.cstep; - constants[5].i = top_blob.dims; - constants[6].i = top_blob.w; - constants[7].i = top_blob.h; - constants[8].i = top_blob.c; - constants[9].i = 0; //top_blob.cstep; + // reduce first + VkImageMat reduced_blob; + { + int reduced_size = (w * h + 7) / 8; + size_t reduced_elemsize = pooling_type == 0 ? elemsize : 4u * elempack; + reduced_blob.create(reduced_size, 1, channels, reduced_elemsize, elempack, opt.workspace_vkallocator); + if (reduced_blob.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = bottom_blob; + bindings[1] = reduced_blob; + + std::vector constants(6); + constants[0].i = bottom_blob.w; + constants[1].i = bottom_blob.h; + constants[2].i = bottom_blob.c; + constants[3].i = 0; //bottom_blob.cstep; + constants[4].i = reduced_blob.w; + constants[5].i = 0; //reduced_blob.cstep; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_first_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_first_pack4 + : pipeline_pooling_global_reduce_first; + + VkImageMat dispatcher; + dispatcher.w = reduced_blob.w; + dispatcher.h = 1; + dispatcher.c = bottom_blob.c; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + } - const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_pack8 - : elempack == 4 ? pipeline_pooling_global_pack4 - : pipeline_pooling_global; + // reduce more + while (reduced_blob.w > 32) + { + int reduced_size = (reduced_blob.w + 7) / 8; + size_t reduced_elemsize = pooling_type == 0 ? elemsize : 4u * elempack; + VkImageMat reduced_blob2; + reduced_blob2.create(reduced_size, 1, channels, reduced_elemsize, elempack, opt.workspace_vkallocator); + if (reduced_blob2.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = reduced_blob; + bindings[1] = reduced_blob2; + + std::vector constants(5); + constants[0].i = reduced_blob.w; + constants[1].i = reduced_blob.c; + constants[2].i = 0; //reduced_blob.cstep; + constants[3].i = reduced_blob2.w; + constants[4].i = 0; //reduced_blob2.cstep; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_pack4 + : pipeline_pooling_global_reduce; + + VkImageMat dispatcher; + dispatcher.w = reduced_blob2.w; + dispatcher.h = 1; + dispatcher.c = reduced_blob2.c; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + + reduced_blob = reduced_blob2; + } - cmd.record_pipeline(pipeline, bindings, constants, top_blob); + // reduce last + { + top_blob.create(channels, elemsize, elempack, opt.blob_vkallocator); + if (top_blob.empty()) + return -100; + + std::vector bindings(2); + bindings[0] = reduced_blob; + bindings[1] = top_blob; + + std::vector constants(4); + constants[0].i = reduced_blob.w; + constants[1].i = reduced_blob.c; + constants[2].i = 0; //reduced_blob.cstep; + constants[3].i = w * h; + + const Pipeline* pipeline = elempack == 8 ? pipeline_pooling_global_reduce_last_pack8 + : elempack == 4 ? pipeline_pooling_global_reduce_last_pack4 + : pipeline_pooling_global_reduce_last; + + VkImageMat dispatcher; + dispatcher.w = 1; + dispatcher.h = 1; + dispatcher.c = top_blob.w; + + cmd.record_pipeline(pipeline, bindings, constants, dispatcher); + } return 0; } diff --git a/src/layer/vulkan/pooling_vulkan.h b/src/layer/vulkan/pooling_vulkan.h index 96a9b6f8e..a3529b270 100644 --- a/src/layer/vulkan/pooling_vulkan.h +++ b/src/layer/vulkan/pooling_vulkan.h @@ -39,12 +39,20 @@ public: Pipeline* pipeline_pooling; Pipeline* pipeline_pooling_pack4; Pipeline* pipeline_pooling_pack8; - Pipeline* pipeline_pooling_global; - Pipeline* pipeline_pooling_global_pack4; - Pipeline* pipeline_pooling_global_pack8; + Pipeline* pipeline_pooling_adaptive; Pipeline* pipeline_pooling_adaptive_pack4; Pipeline* pipeline_pooling_adaptive_pack8; + + Pipeline* pipeline_pooling_global_reduce_first; + Pipeline* pipeline_pooling_global_reduce_first_pack4; + Pipeline* pipeline_pooling_global_reduce_first_pack8; + Pipeline* pipeline_pooling_global_reduce; + Pipeline* pipeline_pooling_global_reduce_pack4; + Pipeline* pipeline_pooling_global_reduce_pack8; + Pipeline* pipeline_pooling_global_reduce_last; + Pipeline* pipeline_pooling_global_reduce_last_pack4; + Pipeline* pipeline_pooling_global_reduce_last_pack8; }; } // namespace ncnn diff --git a/src/layer/vulkan/shader/pooling_global.comp b/src/layer/vulkan/shader/pooling_global.comp deleted file mode 100644 index dc27d3195..000000000 --- a/src/layer/vulkan/shader/pooling_global.comp +++ /dev/null @@ -1,126 +0,0 @@ -// Tencent is pleased to support the open source community by making ncnn available. -// -// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#version 450 - -#if NCNN_fp16_storage -#extension GL_EXT_shader_16bit_storage: require -#endif -#if NCNN_fp16_arithmetic -#extension GL_EXT_shader_explicit_arithmetic_types_float16: require -#endif - -#define FLT_MAX 3.402823466e+38 - -layout (constant_id = 0) const int pooling_type = 0; - -#define shape_constant_id_offset 1 -layout (constant_id = shape_constant_id_offset + 0) const int dims = 0; -layout (constant_id = shape_constant_id_offset + 1) const int w = 0; -layout (constant_id = shape_constant_id_offset + 2) const int h = 0; -layout (constant_id = shape_constant_id_offset + 3) const int c = 0; -layout (constant_id = shape_constant_id_offset + 4) const int cstep = 0; - -layout (constant_id = shape_constant_id_offset + 5) const int outdims = 0; -layout (constant_id = shape_constant_id_offset + 6) const int outw = 0; -layout (constant_id = shape_constant_id_offset + 7) const int outh = 0; -layout (constant_id = shape_constant_id_offset + 8) const int outc = 0; -layout (constant_id = shape_constant_id_offset + 9) const int outcstep = 0; - -#if NCNN_image_shader -layout (binding = 0) uniform unfp sampler3D bottom_blob; -layout (binding = 1, imfmtc1) writeonly uniform unfp image3D top_blob; -#else -layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; -layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; -#endif - -layout (push_constant) uniform parameter -{ - int dims; - int w; - int h; - int c; - int cstep; - - int outdims; - int outw; - int outh; - int outc; - int outcstep; -} p; - -void main() -{ - int gx = int(gl_GlobalInvocationID.x); - int gy = int(gl_GlobalInvocationID.y); - int gz = int(gl_GlobalInvocationID.z); - - if (gx >= psc(outw) || gy >= 1 || gz >= 1) - return; - - int size = psc(w) * psc(h); - int v_offset = gx * psc(cstep); - - afp res; - - if (pooling_type == 0) - { - res = afp(-FLT_MAX); - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - afp v = image3d_ld1(bottom_blob, ivec3(x, y, gx)); - res = max(res, v); - } - } -#else - for (int i = 0; i < size; i++) - { - afp v = buffer_ld1(bottom_blob_data, v_offset + i); - res = max(res, v); - } -#endif - } - if (pooling_type == 1) - { - float sum = 0.f; - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - sum += float(image3d_ld1(bottom_blob, ivec3(x, y, gx))); - } - } -#else - for (int i = 0; i < size; i++) - { - sum += float(buffer_ld1(bottom_blob_data, v_offset + i)); - } -#endif - - res = afp(sum / size); - } - -#if NCNN_image_shader - image3d_st1(top_blob, ivec3(gx, 0, 0), res); -#else - buffer_st1(top_blob_data, gx, res); -#endif -} diff --git a/src/layer/vulkan/shader/pooling_global_pack4.comp b/src/layer/vulkan/shader/pooling_global_pack4.comp deleted file mode 100644 index 4f35639be..000000000 --- a/src/layer/vulkan/shader/pooling_global_pack4.comp +++ /dev/null @@ -1,126 +0,0 @@ -// Tencent is pleased to support the open source community by making ncnn available. -// -// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#version 450 - -#if NCNN_fp16_storage -#extension GL_EXT_shader_16bit_storage: require -#endif -#if NCNN_fp16_arithmetic -#extension GL_EXT_shader_explicit_arithmetic_types_float16: require -#endif - -#define FLT_MAX 3.402823466e+38 - -layout (constant_id = 0) const int pooling_type = 0; - -#define shape_constant_id_offset 1 -layout (constant_id = shape_constant_id_offset + 0) const int dims = 0; -layout (constant_id = shape_constant_id_offset + 1) const int w = 0; -layout (constant_id = shape_constant_id_offset + 2) const int h = 0; -layout (constant_id = shape_constant_id_offset + 3) const int c = 0; -layout (constant_id = shape_constant_id_offset + 4) const int cstep = 0; - -layout (constant_id = shape_constant_id_offset + 5) const int outdims = 0; -layout (constant_id = shape_constant_id_offset + 6) const int outw = 0; -layout (constant_id = shape_constant_id_offset + 7) const int outh = 0; -layout (constant_id = shape_constant_id_offset + 8) const int outc = 0; -layout (constant_id = shape_constant_id_offset + 9) const int outcstep = 0; - -#if NCNN_image_shader -layout (binding = 0) uniform unfp sampler3D bottom_blob; -layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; -#else -layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; -layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; -#endif - -layout (push_constant) uniform parameter -{ - int dims; - int w; - int h; - int c; - int cstep; - - int outdims; - int outw; - int outh; - int outc; - int outcstep; -} p; - -void main() -{ - int gx = int(gl_GlobalInvocationID.x); - int gy = int(gl_GlobalInvocationID.y); - int gz = int(gl_GlobalInvocationID.z); - - if (gx >= psc(outw) || gy >= 1 || gz >= 1) - return; - - int size = psc(w) * psc(h); - int v_offset = gx * psc(cstep); - - afpvec4 res; - - if (pooling_type == 0) - { - res = afpvec4(-FLT_MAX); - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - afpvec4 v = image3d_ld4(bottom_blob, ivec3(x, y, gx)); - res = max(res, v); - } - } -#else - for (int i = 0; i < size; i++) - { - afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + i); - res = max(res, v); - } -#endif - } - if (pooling_type == 1) - { - vec4 sum = vec4(0.f); - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - sum += vec4(image3d_ld4(bottom_blob, ivec3(x, y, gx))); - } - } -#else - for (int i = 0; i < size; i++) - { - sum += vec4(buffer_ld4(bottom_blob_data, v_offset + i)); - } -#endif - - res = afpvec4(sum / size); - } - -#if NCNN_image_shader - image3d_st4(top_blob, ivec3(gx, 0, 0), res); -#else - buffer_st4(top_blob_data, gx, res); -#endif -} diff --git a/src/layer/vulkan/shader/pooling_global_pack8.comp b/src/layer/vulkan/shader/pooling_global_pack8.comp deleted file mode 100644 index 872d2b9ce..000000000 --- a/src/layer/vulkan/shader/pooling_global_pack8.comp +++ /dev/null @@ -1,134 +0,0 @@ -// Tencent is pleased to support the open source community by making ncnn available. -// -// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. -// -// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// https://opensource.org/licenses/BSD-3-Clause -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#version 450 - -#if NCNN_fp16_storage -#extension GL_EXT_shader_16bit_storage: require -struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; -#endif -#if NCNN_fp16_arithmetic -#extension GL_EXT_shader_explicit_arithmetic_types_float16: require -#endif - -#define FLT_MAX 3.402823466e+38 - -layout (constant_id = 0) const int pooling_type = 0; - -#define shape_constant_id_offset 1 -layout (constant_id = shape_constant_id_offset + 0) const int dims = 0; -layout (constant_id = shape_constant_id_offset + 1) const int w = 0; -layout (constant_id = shape_constant_id_offset + 2) const int h = 0; -layout (constant_id = shape_constant_id_offset + 3) const int c = 0; -layout (constant_id = shape_constant_id_offset + 4) const int cstep = 0; - -layout (constant_id = shape_constant_id_offset + 5) const int outdims = 0; -layout (constant_id = shape_constant_id_offset + 6) const int outw = 0; -layout (constant_id = shape_constant_id_offset + 7) const int outh = 0; -layout (constant_id = shape_constant_id_offset + 8) const int outc = 0; -layout (constant_id = shape_constant_id_offset + 9) const int outcstep = 0; - -#if NCNN_image_shader -layout (binding = 0) uniform unfp sampler3D bottom_blob; -layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; -#else -layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; -layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; -#endif - -layout (push_constant) uniform parameter -{ - int dims; - int w; - int h; - int c; - int cstep; - - int outdims; - int outw; - int outh; - int outc; - int outcstep; -} p; - -void main() -{ - int gx = int(gl_GlobalInvocationID.x); - int gy = int(gl_GlobalInvocationID.y); - int gz = int(gl_GlobalInvocationID.z); - - if (gx >= psc(outw) || gy >= 1 || gz >= 1) - return; - - int size = psc(w) * psc(h); - int v_offset = gx * psc(cstep); - - afpvec8 res; - - if (pooling_type == 0) - { - res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - afpvec8 v = image3d_ld8(bottom_blob, ivec3(x, y, gx)); - res[0] = max(res[0], v[0]); - res[1] = max(res[1], v[1]); - } - } -#else - for (int i = 0; i < size; i++) - { - afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); - res[0] = max(res[0], v[0]); - res[1] = max(res[1], v[1]); - } -#endif - } - if (pooling_type == 1) - { - mat2x4 sum = mat2x4(0.f); - -#if NCNN_image_shader - for (int y = 0; y < psc(h); y++) - { - for (int x = 0; x < psc(w); x++) - { - afpvec8 v = image3d_ld8(bottom_blob, ivec3(x, y, gx)); - sum[0] += vec4(v[0]); - sum[1] += vec4(v[1]); - } - } -#else - for (int i = 0; i < size; i++) - { - afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); - sum[0] += vec4(v[0]); - sum[1] += vec4(v[1]); - } -#endif - - res[0] = afpvec4(sum[0] / size); - res[1] = afpvec4(sum[1] / size); - } - -#if NCNN_image_shader - image3d_st8(top_blob, ivec3(gx, 0, 0), res); -#else - buffer_st8(top_blob_data, gx, res); -#endif -} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max.comp b/src/layer/vulkan/shader/pooling_global_reduce_max.comp new file mode 100644 index 000000000..ccd079c3b --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max.comp @@ -0,0 +1,84 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc1) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) - 1; + + const int v_offset = gz * psc(cstep); + + afp res = afp(-FLT_MAX); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + afp v = image3d_ld1(bottom_blob, ivec3(i, 0, gz)); +#else + afp v = buffer_ld1(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st1(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st1(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_first.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_first.comp new file mode 100644 index 000000000..45ef3c14a --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_first.comp @@ -0,0 +1,88 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc1) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) * psc(h) - 1; + + const int v_offset = gz * psc(cstep); + + afp res = afp(-FLT_MAX); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afp v = image3d_ld1(bottom_blob, ivec3(x, y, gz)); +#else + afp v = buffer_ld1(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st1(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st1(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack4.comp new file mode 100644 index 000000000..38d4aece0 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack4.comp @@ -0,0 +1,88 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) * psc(h) - 1; + + const int v_offset = gz * psc(cstep); + + afpvec4 res = afpvec4(-FLT_MAX); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afpvec4 v = image3d_ld4(bottom_blob, ivec3(x, y, gz)); +#else + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st4(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st4(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack8.comp new file mode 100644 index 000000000..49948494e --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_first_pack8.comp @@ -0,0 +1,90 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) * psc(h) - 1; + + const int v_offset = gz * psc(cstep); + + afpvec8 res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afpvec8 v = image3d_ld8(bottom_blob, ivec3(x, y, gz)); +#else + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); +#endif + res[0] = max(res[0], v[0]); + res[1] = max(res[1], v[1]); + } + +#if NCNN_image_shader + image3d_st8(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st8(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_last.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_last.comp new file mode 100644 index 000000000..6bfd1c04f --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_last.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc1) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + afp res = afp(-FLT_MAX); + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + afp v = image3d_ld1(bottom_blob, ivec3(i, 0, gz)); +#else + afp v = buffer_ld1(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st1(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st1(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack4.comp new file mode 100644 index 000000000..0be42690d --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack4.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + afpvec4 res = afpvec4(-FLT_MAX); + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + afpvec4 v = image3d_ld4(bottom_blob, ivec3(i, 0, gz)); +#else + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st4(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st4(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack8.comp new file mode 100644 index 000000000..8e25fc3d2 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_last_pack8.comp @@ -0,0 +1,77 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + afpvec8 res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + afpvec8 v = image3d_ld8(bottom_blob, ivec3(i, 0, gz)); +#else + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); +#endif + res[0] = max(res[0], v[0]); + res[1] = max(res[1], v[1]); + } + +#if NCNN_image_shader + image3d_st8(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st8(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_pack4.comp new file mode 100644 index 000000000..c16f78718 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_pack4.comp @@ -0,0 +1,84 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) - 1; + + const int v_offset = gz * psc(cstep); + + afpvec4 res = afpvec4(-FLT_MAX); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + afpvec4 v = image3d_ld4(bottom_blob, ivec3(i, 0, gz)); +#else + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + i); +#endif + res = max(res, v); + } + +#if NCNN_image_shader + image3d_st4(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st4(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_max_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_max_pack8.comp new file mode 100644 index 000000000..7cff99f00 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_max_pack8.comp @@ -0,0 +1,86 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int size_1 = psc(w) - 1; + + const int v_offset = gz * psc(cstep); + + afpvec8 res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); + + for (int ii = 0; ii < 8; ii++) + { + int i = min(gx + ii * psc(outw), size_1); + +#if NCNN_image_shader + afpvec8 v = image3d_ld8(bottom_blob, ivec3(i, 0, gz)); +#else + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); +#endif + res[0] = max(res[0], v[0]); + res[1] = max(res[1], v[1]); + } + +#if NCNN_image_shader + image3d_st8(top_blob, ivec3(gx, 0, gz), res); +#else + buffer_st8(top_blob_data, gz * psc(outcstep) + gx, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum.comp new file mode 100644 index 000000000..1aa353c75 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum.comp @@ -0,0 +1,82 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, r32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { float bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { float top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + float sum = 0.f; + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + float v = texelFetch(bottom_blob, ivec3(i, 0, gz), 0).r; +#else + float v = bottom_blob_data[v_offset + i]; +#endif + sum += v; + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx, 0, gz), vec4(sum)); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_first.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_first.comp new file mode 100644 index 000000000..82ffb803a --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_first.comp @@ -0,0 +1,86 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, r32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { float top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) * psc(h) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + float sum = 0.f; + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afp v = image3d_ld1(bottom_blob, ivec3(x, y, gz)); +#else + afp v = buffer_ld1(bottom_blob_data, v_offset + i); +#endif + sum += float(v); + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx, 0, gz), vec4(sum)); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack4.comp new file mode 100644 index 000000000..9af7ee3bb --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack4.comp @@ -0,0 +1,86 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, rgba32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { vec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) * psc(h) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + vec4 sum = vec4(0.f); + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afpvec4 v = image3d_ld4(bottom_blob, ivec3(x, y, gz)); +#else + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + i); +#endif + sum += vec4(v); + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx, 0, gz), sum); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack8.comp new file mode 100644 index 000000000..11fd13a07 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_first_pack8.comp @@ -0,0 +1,89 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int h = 0; +layout (constant_id = shape_constant_id_offset + 2) const int c = 0; +layout (constant_id = shape_constant_id_offset + 3) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 4) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 5) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform unfp sampler3D bottom_blob; +layout (binding = 1, rgba32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { mat2x4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int h; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) * psc(h) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + mat2x4 sum = mat2x4(0.f); + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + int y = i / psc(w); + int x = i % psc(w); + afpvec8 v = image3d_ld8(bottom_blob, ivec3(x, y, gz)); +#else + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); +#endif + sum[0] += vec4(v[0]); + sum[1] += vec4(v[1]); + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx * 2, 0, gz), sum[0]); + imageStore(top_blob, ivec3(gx * 2 + 1, 0, gz), sum[1]); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_last.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_last.comp new file mode 100644 index 000000000..4acb2599e --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_last.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, imfmtc1) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { float bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + float sum = 0.f; + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + float v = texelFetch(bottom_blob, ivec3(i, 0, gz), 0).r; +#else + float v = bottom_blob_data[v_offset + i]; +#endif + sum += v; + } + + afp res = afp(sum / p.size); + +#if NCNN_image_shader + image3d_st1(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st1(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack4.comp new file mode 100644 index 000000000..b4dfaae68 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack4.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { vec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + vec4 sum = vec4(0.f); + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + vec4 v = texelFetch(bottom_blob, ivec3(i, 0, gz), 0); +#else + vec4 v = bottom_blob_data[v_offset + i]; +#endif + sum += v; + } + + afpvec4 res = afpvec4(sum / p.size); + +#if NCNN_image_shader + image3d_st4(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st4(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack8.comp new file mode 100644 index 000000000..e5dc03391 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_last_pack8.comp @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, imfmtc4) writeonly uniform unfp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { mat2x4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + int size; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= 1 || gy >= 1 || gz >= psc(c)) + return; + + const int v_offset = gz * psc(cstep); + + mat2x4 sum = mat2x4(0.f); + + for (int i = 0; i < psc(w); i++) + { +#if NCNN_image_shader + mat2x4 v; + v[0] = texelFetch(bottom_blob, ivec3(i * 2, 0, gz), 0); + v[1] = texelFetch(bottom_blob, ivec3(i * 2 + 1, 0, gz), 0); +#else + mat2x4 v = bottom_blob_data[v_offset + i]; +#endif + sum[0] += v[0]; + sum[1] += v[1]; + } + + afpvec8 res; + res[0] = afpvec4(sum[0] / p.size); + res[1] = afpvec4(sum[1] / p.size); + +#if NCNN_image_shader + image3d_st8(top_blob, ivec3(gz, 0, 0), res); +#else + buffer_st8(top_blob_data, gz, res); +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_pack4.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_pack4.comp new file mode 100644 index 000000000..f2a7d88d8 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_pack4.comp @@ -0,0 +1,82 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, rgba32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { vec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { vec4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + vec4 sum = vec4(0.f); + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + vec4 v = texelFetch(bottom_blob, ivec3(i, 0, gz), 0); +#else + vec4 v = bottom_blob_data[v_offset + i]; +#endif + sum += v; + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx, 0, gz), sum); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/src/layer/vulkan/shader/pooling_global_reduce_sum_pack8.comp b/src/layer/vulkan/shader/pooling_global_reduce_sum_pack8.comp new file mode 100644 index 000000000..b5b4ab533 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_reduce_sum_pack8.comp @@ -0,0 +1,86 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define shape_constant_id_offset 0 +layout (constant_id = shape_constant_id_offset + 0) const int w = 0; +layout (constant_id = shape_constant_id_offset + 1) const int c = 0; +layout (constant_id = shape_constant_id_offset + 2) const int cstep = 0; + +layout (constant_id = shape_constant_id_offset + 3) const int outw = 0; +layout (constant_id = shape_constant_id_offset + 4) const int outcstep = 0; + +#if NCNN_image_shader +layout (binding = 0) uniform highp sampler3D bottom_blob; +layout (binding = 1, rgba32f) writeonly uniform highp image3D top_blob; +#else +layout (binding = 0) readonly buffer bottom_blob { mat2x4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { mat2x4 top_blob_data[]; }; +#endif + +layout (push_constant) uniform parameter +{ + int w; + int c; + int cstep; + + int outw; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= psc(outw) || gy >= 1 || gz >= psc(c)) + return; + + const int end = min(8, (psc(w) - gx - 1) / psc(outw) + 1); + + const int v_offset = gz * psc(cstep); + + mat2x4 sum = mat2x4(0.f); + + for (int ii = 0; ii < end; ii++) + { + int i = gx + ii * psc(outw); + +#if NCNN_image_shader + mat2x4 v; + v[0] = texelFetch(bottom_blob, ivec3(i * 2, 0, gz), 0); + v[1] = texelFetch(bottom_blob, ivec3(i * 2 + 1, 0, gz), 0); +#else + mat2x4 v = bottom_blob_data[v_offset + i]; +#endif + sum[0] += v[0]; + sum[1] += v[1]; + } + +#if NCNN_image_shader + imageStore(top_blob, ivec3(gx * 2, 0, gz), sum[0]); + imageStore(top_blob, ivec3(gx * 2 + 1, 0, gz), sum[1]); +#else + top_blob_data[gz * psc(outcstep) + gx] = sum; +#endif +} diff --git a/tests/test_pooling.cpp b/tests/test_pooling.cpp index bfefd6310..df0e69e92 100644 --- a/tests/test_pooling.cpp +++ b/tests/test_pooling.cpp @@ -126,6 +126,10 @@ static int test_pooling_2() || test_pooling(7, 8, 8, 1, 1, 1, 0, 1, 0, 0, 0, 0) || test_pooling(11, 13, 16, 0, 1, 1, 0, 1, 0, 0, 0, 0) || test_pooling(13, 11, 16, 1, 1, 1, 0, 1, 0, 0, 0, 0) + || test_pooling(110, 103, 106, 0, 1, 1, 0, 1, 0, 0, 0, 0) + || test_pooling(130, 101, 106, 1, 1, 1, 0, 1, 0, 0, 0, 0) + || test_pooling(80, 93, 128, 0, 1, 1, 0, 1, 0, 0, 0, 0) + || test_pooling(80, 91, 128, 1, 1, 1, 0, 1, 0, 0, 0, 0) || test_pooling(48, 48, 4, 0, 2, 2, 0, 0, 0, 0, 0, 0) || test_pooling(48, 48, 15, 0, 2, 2, 1, 0, 0, 0, 0, 0); }