diff --git a/src/layer/vulkan/shader/absval_pack8.comp b/src/layer/vulkan/shader/absval_pack8.comp new file mode 100644 index 000000000..5b3021fbd --- /dev/null +++ b/src/layer/vulkan/shader/absval_pack8.comp @@ -0,0 +1,57 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = abs(v[0]); + v[1] = abs(v[1]); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/batchnorm_pack8.comp b/src/layer/vulkan/shader/batchnorm_pack8.comp new file mode 100644 index 000000000..f1448fafb --- /dev/null +++ b/src/layer/vulkan/shader/batchnorm_pack8.comp @@ -0,0 +1,64 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) readonly buffer a { sfpvec8 a_data[]; }; +layout (binding = 2) readonly buffer b { sfpvec8 b_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + const ivec3 gxyz = ivec3(gx, gy, gz); + const int abi = gxyz[p.dims - 1]; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + afpvec8 b = buffer_ld8(b_data, abi); + afpvec8 a = buffer_ld8(a_data, abi); + + v[0] = b[0] * v[0] + a[0]; + v[1] = b[1] * v[1] + a[1]; + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/binaryop_broadcast_a1_pack8.comp b/src/layer/vulkan/shader/binaryop_broadcast_a1_pack8.comp new file mode 100644 index 000000000..e3627c50b --- /dev/null +++ b/src/layer/vulkan/shader/binaryop_broadcast_a1_pack8.comp @@ -0,0 +1,120 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer a_blob { sfp a_blob_data[]; }; +layout (binding = 1) readonly buffer b_blob { sfpvec8 b_blob_data[]; }; +layout (binding = 2) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int adims; + int aw; + int ah; + int ac; + int acstep; + + int bdims; + int bw; + int bh; + int bc; + int bcstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + // type 2 3 4 + afpvec4 v1 = afpvec4(buffer_ld1(a_blob_data, 0)); + afpvec8 v2 = buffer_ld8(b_blob_data, gi); + + afpvec8 res; + + if (op_type == 0) + { + res[0] = v1 + v2[0]; + res[1] = v1 + v2[1]; + } + if (op_type == 1) + { + res[0] = v1 - v2[0]; + res[1] = v1 - v2[1]; + } + if (op_type == 2) + { + res[0] = v1 * v2[0]; + res[1] = v1 * v2[1]; + } + if (op_type == 3) + { + res[0] = v1 / v2[0]; + res[1] = v1 / v2[1]; + } + if (op_type == 4) + { + res[0] = max(v1, v2[0]); + res[1] = max(v1, v2[1]); + } + if (op_type == 5) + { + res[0] = min(v1, v2[0]); + res[1] = min(v1, v2[1]); + } + if (op_type == 6) + { + res[0] = pow(v1, v2[0]); + res[1] = pow(v1, v2[1]); + } + if (op_type == 7) + { + res[0] = v2[0] - v1; + res[1] = v2[1] - v1; + } + if (op_type == 8) + { + res[0] = v2[0] / v1; + res[1] = v2[1] / v1; + } + + buffer_st8(top_blob_data, gi, res); +} diff --git a/src/layer/vulkan/shader/binaryop_broadcast_b1_pack8.comp b/src/layer/vulkan/shader/binaryop_broadcast_b1_pack8.comp new file mode 100644 index 000000000..aa6cf171a --- /dev/null +++ b/src/layer/vulkan/shader/binaryop_broadcast_b1_pack8.comp @@ -0,0 +1,128 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer a_blob { sfpvec8 a_blob_data[]; }; +layout (binding = 1) readonly buffer b_blob { sfp b_blob_data[]; }; +layout (binding = 2) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int adims; + int aw; + int ah; + int ac; + int acstep; + + int bdims; + int bw; + int bh; + int bc; + int bcstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + int bi = 0; + + if (p.bw == p.aw && p.bh == p.ah && p.bc == 1) + { + // special type 2 + bi = gy * p.bw + gx; + } + + // type 6 11 16 + afpvec8 v1 = buffer_ld8(a_blob_data, gi); + afpvec4 v2 = afpvec4(buffer_ld1(b_blob_data, bi)); + + afpvec8 res; + + if (op_type == 0) + { + res[0] = v1[0] + v2; + res[1] = v1[1] + v2; + } + if (op_type == 1) + { + res[0] = v1[0] - v2; + res[1] = v1[1] - v2; + } + if (op_type == 2) + { + res[0] = v1[0] * v2; + res[1] = v1[1] * v2; + } + if (op_type == 3) + { + res[0] = v1[0] / v2; + res[1] = v1[1] / v2; + } + if (op_type == 4) + { + res[0] = max(v1[0], v2); + res[1] = max(v1[1], v2); + } + if (op_type == 5) + { + res[0] = min(v1[0], v2); + res[1] = min(v1[1], v2); + } + if (op_type == 6) + { + res[0] = pow(v1[0], v2); + res[1] = pow(v1[1], v2); + } + if (op_type == 7) + { + res[0] = v2 - v1[0]; + res[1] = v2 - v1[1]; + } + if (op_type == 8) + { + res[0] = v2 / v1[0]; + res[1] = v2 / v1[1]; + } + + buffer_st8(top_blob_data, gi, res); +} diff --git a/src/layer/vulkan/shader/binaryop_broadcast_pack8.comp b/src/layer/vulkan/shader/binaryop_broadcast_pack8.comp new file mode 100644 index 000000000..9c6f6b864 --- /dev/null +++ b/src/layer/vulkan/shader/binaryop_broadcast_pack8.comp @@ -0,0 +1,181 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer a_blob { sfpvec8 a_blob_data[]; }; +layout (binding = 1) readonly buffer b_blob { sfpvec8 b_blob_data[]; }; +layout (binding = 2) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int adims; + int aw; + int ah; + int ac; + int acstep; + + int bdims; + int bw; + int bh; + int bc; + int bcstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + int ai; + int bi; + + if (p.adims == 3) + { + if (p.bdims == 3) + { + if (p.bw == 1 && p.bh == 1) + { + // special type 1 + ai = gi; + bi = gz; + } + } + + if (p.bdims == 2) + { + // type 18 + ai = gi; + bi = gz * p.bw + gy; + } + + if (p.bdims == 1) + { + // type 17 + ai = gi; + bi = gz; + } + } + else if (p.adims == 2) + { + if (p.bdims == 3) + { + // type 14 + ai = gz * p.aw + gy; + bi = gi; + } + + if (p.bdims == 1) + { + // type 12 + ai = gi; + bi = gy; + } + } + else if (p.adims == 1) + { + if (p.bdims == 3) + { + // type 9 + ai = gz; + bi = gi; + } + + if (p.bdims == 2) + { + // type 8 + ai = gy; + bi = gi; + } + } + + afpvec8 v1 = buffer_ld8(a_blob_data, ai); + afpvec8 v2 = buffer_ld8(b_blob_data, bi); + + afpvec8 res; + + if (op_type == 0) + { + res[0] = v1[0] + v2[0]; + res[1] = v1[1] + v2[1]; + } + if (op_type == 1) + { + res[0] = v1[0] - v2[0]; + res[1] = v1[1] - v2[1]; + } + if (op_type == 2) + { + res[0] = v1[0] * v2[0]; + res[1] = v1[1] * v2[1]; + } + if (op_type == 3) + { + res[0] = v1[0] / v2[0]; + res[1] = v1[1] / v2[1]; + } + if (op_type == 4) + { + res[0] = max(v1[0], v2[0]); + res[1] = max(v1[1], v2[1]); + } + if (op_type == 5) + { + res[0] = min(v1[0], v2[0]); + res[1] = min(v1[1], v2[1]); + } + if (op_type == 6) + { + res[0] = pow(v1[0], v2[0]); + res[1] = pow(v1[1], v2[1]); + } + if (op_type == 7) + { + res[0] = v2[0] - v1[0]; + res[1] = v2[1] - v1[1]; + } + if (op_type == 8) + { + res[0] = v2[0] / v1[0]; + res[1] = v2[1] / v1[1]; + } + + buffer_st8(top_blob_data, gi, res); +} diff --git a/src/layer/vulkan/shader/binaryop_pack8.comp b/src/layer/vulkan/shader/binaryop_pack8.comp new file mode 100644 index 000000000..9b2291c6f --- /dev/null +++ b/src/layer/vulkan/shader/binaryop_pack8.comp @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; +layout (constant_id = 1) const int with_scalar = 0; +layout (constant_id = 2) const float const_b = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer a_blob { sfpvec8 a_blob_data[]; }; +layout (binding = 1) readonly buffer b_blob { sfpvec8 b_blob_data[]; }; +layout (binding = 2) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int adims; + int aw; + int ah; + int ac; + int acstep; + + int bdims; + int bw; + int bh; + int bc; + int bcstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + afpvec8 v1 = buffer_ld8(a_blob_data, gi); + + afpvec8 res; + + if (with_scalar == 1) + { + // type 5 10 15 + afp b = afp(const_b); + + if (op_type == 0) + { + res[0] = v1[0] + b; + res[1] = v1[1] + b; + } + if (op_type == 1) + { + res[0] = v1[0] - b; + res[1] = v1[1] - b; + } + if (op_type == 2) + { + res[0] = v1[0] * b; + res[1] = v1[1] * b; + } + if (op_type == 3) + { + res[0] = v1[0] / b; + res[1] = v1[1] / b; + } + if (op_type == 4) + { + res[0] = max(v1[0], b); + res[1] = max(v1[1], b); + } + if (op_type == 5) + { + res[0] = min(v1[0], b); + res[1] = min(v1[1], b); + } + if (op_type == 6) + { + res[0] = pow(v1[0], afpvec4(b)); + res[1] = pow(v1[1], afpvec4(b)); + } + if (op_type == 7) + { + res[0] = b - v1[0]; + res[1] = b - v1[1]; + } + if (op_type == 8) + { + res[0] = b / v1[0]; + res[1] = b / v1[1]; + } + + buffer_st8(a_blob_data, gi, res); + } + else + { + // type 7 13 19 + afpvec8 v2 = buffer_ld8(b_blob_data, gi); + + if (op_type == 0) + { + res[0] = v1[0] + v2[0]; + res[1] = v1[1] + v2[1]; + } + if (op_type == 1) + { + res[0] = v1[0] - v2[0]; + res[1] = v1[1] - v2[1]; + } + if (op_type == 2) + { + res[0] = v1[0] * v2[0]; + res[1] = v1[1] * v2[1]; + } + if (op_type == 3) + { + res[0] = v1[0] / v2[0]; + res[1] = v1[1] / v2[1]; + } + if (op_type == 4) + { + res[0] = max(v1[0], v2[0]); + res[1] = max(v1[1], v2[1]); + } + if (op_type == 5) + { + res[0] = min(v1[0], v2[0]); + res[1] = min(v1[1], v2[1]); + } + if (op_type == 6) + { + res[0] = pow(v1[0], v2[0]); + res[1] = pow(v1[1], v2[1]); + } + if (op_type == 7) + { + res[0] = v2[0] - v1[0]; + res[1] = v2[1] - v1[1]; + } + if (op_type == 8) + { + res[0] = v2[0] / v1[0]; + res[1] = v2[1] / v1[1]; + } + + buffer_st8(top_blob_data, gi, res); + } +} diff --git a/src/layer/vulkan/shader/cast_fp16_to_fp32_pack8.comp b/src/layer/vulkan/shader/cast_fp16_to_fp32_pack8.comp new file mode 100644 index 000000000..58a358720 --- /dev/null +++ b/src/layer/vulkan/shader/cast_fp16_to_fp32_pack8.comp @@ -0,0 +1,61 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { mat2x4 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + const int v_offset = gz * p.cstep + gy * p.w + gx; + + top_blob_data[gi] = mat2x4(buffer_ld8(bottom_blob_data, v_offset)); +} diff --git a/src/layer/vulkan/shader/cast_fp32_to_fp16_pack8.comp b/src/layer/vulkan/shader/cast_fp32_to_fp16_pack8.comp new file mode 100644 index 000000000..035b03d1e --- /dev/null +++ b/src/layer/vulkan/shader/cast_fp32_to_fp16_pack8.comp @@ -0,0 +1,61 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { mat2x4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + const int v_offset = gz * p.cstep + gy * p.w + gx; + + buffer_st8(top_blob_data, gi, afpvec8(bottom_blob_data[v_offset])); +} diff --git a/src/layer/vulkan/shader/clip_pack8.comp b/src/layer/vulkan/shader/clip_pack8.comp new file mode 100644 index 000000000..8a3ebb197 --- /dev/null +++ b/src/layer/vulkan/shader/clip_pack8.comp @@ -0,0 +1,60 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const float const_min = 0; +layout (constant_id = 1) const float const_max = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = clamp(v[0], afp(const_min), afp(const_max)); + v[1] = clamp(v[1], afp(const_min), afp(const_max)); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/concat_pack8.comp b/src/layer/vulkan/shader/concat_pack8.comp new file mode 100644 index 000000000..b308d6dc6 --- /dev/null +++ b/src/layer/vulkan/shader/concat_pack8.comp @@ -0,0 +1,69 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int offset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + ivec3 gxyz = ivec3(gx, gy, gz); + + gxyz[p.dims - 1 - axis] += p.offset; + + int v_offset = gxyz.z * p.outcstep + gxyz.y * p.outw + gxyz.x; + + buffer_cp8(top_blob_data, v_offset, bottom_blob_data, gi); +} diff --git a/src/layer/vulkan/shader/concat_pack8to1.comp b/src/layer/vulkan/shader/concat_pack8to1.comp new file mode 100644 index 000000000..30e93543c --- /dev/null +++ b/src/layer/vulkan/shader/concat_pack8to1.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int offset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + ivec3 gxyz = ivec3(gx, gy, gz); + + gxyz[p.dims - 1] *= 8; + gxyz[p.dims - 1 - axis] += p.offset; + + int v_offset_0 = gxyz.z * p.outcstep + gxyz.y * p.outw + gxyz.x; + + ivec3 gxyz4 = ivec3(1, p.outw, p.outcstep); + + ivec4 v_offset = v_offset_0 + ivec4(0, 1, 2, 3) * gxyz4[p.dims - 1 - axis]; + ivec4 vv_offset = v_offset + 4 * gxyz4[p.dims - 1 - axis]; + + buffer_cp8to1(top_blob_data, v_offset, vv_offset, bottom_blob_data, gi); +} diff --git a/src/layer/vulkan/shader/concat_pack8to4.comp b/src/layer/vulkan/shader/concat_pack8to4.comp new file mode 100644 index 000000000..58d7b0484 --- /dev/null +++ b/src/layer/vulkan/shader/concat_pack8to4.comp @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int offset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + ivec3 gxyz = ivec3(gx, gy, gz); + + gxyz[p.dims - 1] *= 2; + gxyz[p.dims - 1 - axis] += p.offset; + + int v_offset_0 = gxyz.z * p.outcstep + gxyz.y * p.outw + gxyz.x; + + ivec3 gxyz4 = ivec3(1, p.outw, p.outcstep); + + ivec2 v_offset = v_offset_0 + ivec2(0, 1) * gxyz4[p.dims - 1 - axis]; + + buffer_cp8to4(top_blob_data, v_offset, bottom_blob_data, gi); +} diff --git a/src/layer/vulkan/shader/convolution_pack1to8.comp b/src/layer/vulkan/shader/convolution_pack1to8.comp new file mode 100644 index 000000000..e627600f9 --- /dev/null +++ b/src/layer/vulkan/shader/convolution_pack1to8.comp @@ -0,0 +1,131 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gz * p.c * kernel_w * kernel_h; + + for (int z = 0; z < p.c; z++) + { + int v_offset = z * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afp v = buffer_ld1(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k = buffer_ld8(weight_data, w_offset + x); + + // sum += v * k; + sum[0] += v * k[0]; + sum[1] += v * k[1]; + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolution_pack4to8.comp b/src/layer/vulkan/shader/convolution_pack4to8.comp new file mode 100644 index 000000000..b9ffb9072 --- /dev/null +++ b/src/layer/vulkan/shader/convolution_pack4to8.comp @@ -0,0 +1,144 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec4 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gz * p.c * kernel_w * kernel_h; + + for (int z = 0; z < p.c; z++) + { + int v_offset = z * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + x * dilation_w); + + afpvec4 k0 = buffer_ld4(weight_data, (w_offset + x) * 8 + 0); + afpvec4 k1 = buffer_ld4(weight_data, (w_offset + x) * 8 + 1); + afpvec4 k2 = buffer_ld4(weight_data, (w_offset + x) * 8 + 2); + afpvec4 k3 = buffer_ld4(weight_data, (w_offset + x) * 8 + 3); + afpvec4 k4 = buffer_ld4(weight_data, (w_offset + x) * 8 + 4); + afpvec4 k5 = buffer_ld4(weight_data, (w_offset + x) * 8 + 5); + afpvec4 k6 = buffer_ld4(weight_data, (w_offset + x) * 8 + 6); + afpvec4 k7 = buffer_ld4(weight_data, (w_offset + x) * 8 + 7); + + // sum += v * k; + sum[0].r += dot(v, k0); + sum[0].g += dot(v, k1); + sum[0].b += dot(v, k2); + sum[0].a += dot(v, k3); + sum[1].r += dot(v, k4); + sum[1].g += dot(v, k5); + sum[1].b += dot(v, k6); + sum[1].a += dot(v, k7); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolution_pack8.comp b/src/layer/vulkan/shader/convolution_pack8.comp new file mode 100644 index 000000000..af09f7af1 --- /dev/null +++ b/src/layer/vulkan/shader/convolution_pack8.comp @@ -0,0 +1,144 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gz * p.c * kernel_w * kernel_h; + + for (int z = 0; z < p.c; z++) + { + int v_offset = z * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + x) * 8 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + x) * 8 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + x) * 8 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + x) * 8 + 3); + afpvec8 k4 = buffer_ld8(weight_data, (w_offset + x) * 8 + 4); + afpvec8 k5 = buffer_ld8(weight_data, (w_offset + x) * 8 + 5); + afpvec8 k6 = buffer_ld8(weight_data, (w_offset + x) * 8 + 6); + afpvec8 k7 = buffer_ld8(weight_data, (w_offset + x) * 8 + 7); + + // sum += v * k + sum[0].r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum[0].g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum[0].b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum[0].a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + sum[1].r += dot(v[0], k4[0]) + dot(v[1], k4[1]); + sum[1].g += dot(v[0], k5[0]) + dot(v[1], k5[1]); + sum[1].b += dot(v[0], k6[0]) + dot(v[1], k6[1]); + sum[1].a += dot(v[0], k7[0]) + dot(v[1], k7[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolution_pack8to1.comp b/src/layer/vulkan/shader/convolution_pack8to1.comp new file mode 100644 index 000000000..2f40ed7b2 --- /dev/null +++ b/src/layer/vulkan/shader/convolution_pack8to1.comp @@ -0,0 +1,126 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfp bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afp sum; + + if (bias_term == 1) + { + sum = buffer_ld1(bias_data, gz); + } + else + { + sum = afp(0.f); + } + + int w_offset = gz * p.c * kernel_w * kernel_h; + + for (int z = 0; z < p.c; z++) + { + int v_offset = z * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k = buffer_ld8(weight_data, w_offset + x); + + // sum += dot(v, k); + sum += dot(v[0], k[0]) + dot(v[1], k[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = sum < afp(0.f) ? sum * slope : sum; + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st1(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolution_pack8to4.comp b/src/layer/vulkan/shader/convolution_pack8to4.comp new file mode 100644 index 000000000..7f1a9e95d --- /dev/null +++ b/src/layer/vulkan/shader/convolution_pack8to4.comp @@ -0,0 +1,132 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec4 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec4 sum; + + if (bias_term == 1) + { + sum = buffer_ld4(bias_data, gz); + } + else + { + sum = afpvec4(0.f); + } + + int w_offset = gz * p.c * kernel_w * kernel_h; + + for (int z = 0; z < p.c; z++) + { + int v_offset = z * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + x) * 4 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + x) * 4 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + x) * 4 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + x) * 4 + 3); + + // sum += v * k + sum.r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum.g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum.b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum.a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = mix(sum, sum * afp(slope), lessThan(sum, afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st4(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_group_pack1to8.comp b/src/layer/vulkan/shader/convolutiondepthwise_group_pack1to8.comp new file mode 100644 index 000000000..e78e0ed93 --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_group_pack1to8.comp @@ -0,0 +1,142 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int z = 0; z < channels_g; z++) + { + int v_offset = v_offset_0 + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afp v = buffer_ld1(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k = buffer_ld8(weight_data, w_offset + x); + + // sum += v * k; + sum[0] += v * k[0]; + sum[1] += v * k[1]; + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + v_offset_0 += p.cstep; + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_group_pack4to8.comp b/src/layer/vulkan/shader/convolutiondepthwise_group_pack4to8.comp new file mode 100644 index 000000000..7df4bde97 --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_group_pack4to8.comp @@ -0,0 +1,155 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec4 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int z = 0; z < channels_g; z++) + { + int v_offset = v_offset_0 + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset + x * dilation_w); + + afpvec4 k0 = buffer_ld4(weight_data, (w_offset + x) * 8 + 0); + afpvec4 k1 = buffer_ld4(weight_data, (w_offset + x) * 8 + 1); + afpvec4 k2 = buffer_ld4(weight_data, (w_offset + x) * 8 + 2); + afpvec4 k3 = buffer_ld4(weight_data, (w_offset + x) * 8 + 3); + afpvec4 k4 = buffer_ld4(weight_data, (w_offset + x) * 8 + 4); + afpvec4 k5 = buffer_ld4(weight_data, (w_offset + x) * 8 + 5); + afpvec4 k6 = buffer_ld4(weight_data, (w_offset + x) * 8 + 6); + afpvec4 k7 = buffer_ld4(weight_data, (w_offset + x) * 8 + 7); + + // sum += v * k; + sum[0].r += dot(v, k0); + sum[0].g += dot(v, k1); + sum[0].b += dot(v, k2); + sum[0].a += dot(v, k3); + sum[1].r += dot(v, k4); + sum[1].g += dot(v, k5); + sum[1].b += dot(v, k6); + sum[1].a += dot(v, k7); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + v_offset_0 += p.cstep; + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_group_pack8.comp b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8.comp new file mode 100644 index 000000000..197f67061 --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8.comp @@ -0,0 +1,155 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int z = 0; z < channels_g; z++) + { + int v_offset = v_offset_0 + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + x) * 8 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + x) * 8 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + x) * 8 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + x) * 8 + 3); + afpvec8 k4 = buffer_ld8(weight_data, (w_offset + x) * 8 + 4); + afpvec8 k5 = buffer_ld8(weight_data, (w_offset + x) * 8 + 5); + afpvec8 k6 = buffer_ld8(weight_data, (w_offset + x) * 8 + 6); + afpvec8 k7 = buffer_ld8(weight_data, (w_offset + x) * 8 + 7); + + // sum += v * k + sum[0].r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum[0].g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum[0].b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum[0].a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + sum[1].r += dot(v[0], k4[0]) + dot(v[1], k4[1]); + sum[1].g += dot(v[0], k5[0]) + dot(v[1], k5[1]); + sum[1].b += dot(v[0], k6[0]) + dot(v[1], k6[1]); + sum[1].a += dot(v[0], k7[0]) + dot(v[1], k7[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + v_offset_0 += p.cstep; + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to1.comp b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to1.comp new file mode 100644 index 000000000..42f809bcd --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to1.comp @@ -0,0 +1,137 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfp bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afp sum; + + if (bias_term == 1) + { + sum = buffer_ld1(bias_data, gz); + } + else + { + sum = afp(0.f); + } + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int z = 0; z < channels_g; z++) + { + int v_offset = v_offset_0 + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k = buffer_ld8(weight_data, w_offset + x); + + // sum += dot(v, k); + sum += dot(v[0], k[0]) + dot(v[1], k[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + v_offset_0 += p.cstep; + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = sum < afp(0.f) ? sum * slope : sum; + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st1(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to4.comp b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to4.comp new file mode 100644 index 000000000..521339101 --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_group_pack8to4.comp @@ -0,0 +1,143 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec4 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec4 sum; + + if (bias_term == 1) + { + sum = buffer_ld4(bias_data, gz); + } + else + { + sum = afpvec4(0.f); + } + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int z = 0; z < channels_g; z++) + { + int v_offset = v_offset_0 + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + x) * 4 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + x) * 4 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + x) * 4 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + x) * 4 + 3); + + // sum += v * k + sum.r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum.g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum.b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum.a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + v_offset_0 += p.cstep; + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = mix(sum, sum * afp(slope), lessThan(sum, afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st4(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/convolutiondepthwise_pack8.comp b/src/layer/vulkan/shader/convolutiondepthwise_pack8.comp new file mode 100644 index 000000000..4985e2a54 --- /dev/null +++ b/src/layer/vulkan/shader/convolutiondepthwise_pack8.comp @@ -0,0 +1,129 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + // depth-wise convolution + int w_offset = gz * kernel_w * kernel_h; + int v_offset = gz * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x * dilation_w); + + afpvec8 k = buffer_ld8(weight_data, w_offset + x); + + // sum += v * k; + sum[0] += v[0] * k[0]; + sum[1] += v[1] * k[1]; + } + + v_offset += dilation_h * p.w; + w_offset += kernel_w; + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/crop_pack1to8.comp b/src/layer/vulkan/shader/crop_pack1to8.comp new file mode 100644 index 000000000..ebd86e781 --- /dev/null +++ b/src/layer/vulkan/shader/crop_pack1to8.comp @@ -0,0 +1,69 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int woffset; + int hoffset; + int coffset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + int gi = gz * p.outcstep + gy * p.outw + gx; + + int x = gx + p.woffset; + int y = gy + p.hoffset; + ivec4 z4 = gz * 8 + ivec4(0, 1, 2, 3) + p.coffset; + ivec4 v_offset = z4 * p.cstep + y * p.w + x; + ivec4 vv_offset = (z4 + 4) * p.cstep + y * p.w + x; + + buffer_cp1to8(top_blob_data, gi, bottom_blob_data, v_offset, vv_offset); +} diff --git a/src/layer/vulkan/shader/crop_pack8to1.comp b/src/layer/vulkan/shader/crop_pack8to1.comp new file mode 100644 index 000000000..949e2d804 --- /dev/null +++ b/src/layer/vulkan/shader/crop_pack8to1.comp @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int woffset; + int hoffset; + int coffset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + int gi = gz * p.outcstep + gy * p.outw + gx; + + int x = gx + p.woffset; + int y = gy + p.hoffset; + int z = gz + p.coffset; + + int sz = z / 8; + int sz4 = (z % 8) / 4; + int lane = z % 4; + int v_offset = sz * p.cstep + y * p.w + x; + + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + buffer_st1(top_blob_data, gi, v[sz4][lane]); +} diff --git a/src/layer/vulkan/shader/deconvolution_pack1to8.comp b/src/layer/vulkan/shader/deconvolution_pack1to8.comp new file mode 100644 index 000000000..b56f6e582 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolution_pack1to8.comp @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + int w_offset_0 = gz * p.c * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < p.c; z++) + { + afp v = buffer_ld1(bottom_blob_data, v_offset); + + afpvec8 k = buffer_ld8(weight_data, w_offset); + + // sum += v * k; + sum[0] += v * k[0]; + sum[1] += v * k[1]; + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolution_pack4to8.comp b/src/layer/vulkan/shader/deconvolution_pack4to8.comp new file mode 100644 index 000000000..904fe4d1d --- /dev/null +++ b/src/layer/vulkan/shader/deconvolution_pack4to8.comp @@ -0,0 +1,164 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec4 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + int w_offset_0 = gz * p.c * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < p.c; z++) + { + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset); + + afpvec4 k0 = buffer_ld4(weight_data, w_offset * 8 + 0); + afpvec4 k1 = buffer_ld4(weight_data, w_offset * 8 + 1); + afpvec4 k2 = buffer_ld4(weight_data, w_offset * 8 + 2); + afpvec4 k3 = buffer_ld4(weight_data, w_offset * 8 + 3); + afpvec4 k4 = buffer_ld4(weight_data, w_offset * 8 + 4); + afpvec4 k5 = buffer_ld4(weight_data, w_offset * 8 + 5); + afpvec4 k6 = buffer_ld4(weight_data, w_offset * 8 + 6); + afpvec4 k7 = buffer_ld4(weight_data, w_offset * 8 + 7); + + // sum += v * k; + sum[0].r += dot(v, k0); + sum[0].g += dot(v, k1); + sum[0].b += dot(v, k2); + sum[0].a += dot(v, k3); + sum[1].r += dot(v, k4); + sum[1].g += dot(v, k5); + sum[1].b += dot(v, k6); + sum[1].a += dot(v, k7); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolution_pack8.comp b/src/layer/vulkan/shader/deconvolution_pack8.comp new file mode 100644 index 000000000..88b14694d --- /dev/null +++ b/src/layer/vulkan/shader/deconvolution_pack8.comp @@ -0,0 +1,164 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + int w_offset_0 = gz * p.c * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < p.c; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k0 = buffer_ld8(weight_data, w_offset * 8 + 0); + afpvec8 k1 = buffer_ld8(weight_data, w_offset * 8 + 1); + afpvec8 k2 = buffer_ld8(weight_data, w_offset * 8 + 2); + afpvec8 k3 = buffer_ld8(weight_data, w_offset * 8 + 3); + afpvec8 k4 = buffer_ld8(weight_data, w_offset * 8 + 4); + afpvec8 k5 = buffer_ld8(weight_data, w_offset * 8 + 5); + afpvec8 k6 = buffer_ld8(weight_data, w_offset * 8 + 6); + afpvec8 k7 = buffer_ld8(weight_data, w_offset * 8 + 7); + + // sum += v * k + sum[0].r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum[0].g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum[0].b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum[0].a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + sum[1].r += dot(v[0], k4[0]) + dot(v[1], k4[1]); + sum[1].g += dot(v[0], k5[0]) + dot(v[1], k5[1]); + sum[1].b += dot(v[0], k6[0]) + dot(v[1], k6[1]); + sum[1].a += dot(v[0], k7[0]) + dot(v[1], k7[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolution_pack8to1.comp b/src/layer/vulkan/shader/deconvolution_pack8to1.comp new file mode 100644 index 000000000..00cd87371 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolution_pack8to1.comp @@ -0,0 +1,146 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfp bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afp sum; + + if (bias_term == 1) + { + sum = buffer_ld1(bias_data, gz); + } + else + { + sum = afp(0.f); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + int w_offset_0 = gz * p.c * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < p.c; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k = buffer_ld8(weight_data, w_offset); + + // sum += dot(v, k); + sum += dot(v[0], k[0]) + dot(v[1], k[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = sum < afp(0.f) ? sum * slope : sum; + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st1(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolution_pack8to4.comp b/src/layer/vulkan/shader/deconvolution_pack8to4.comp new file mode 100644 index 000000000..af6f4e7cd --- /dev/null +++ b/src/layer/vulkan/shader/deconvolution_pack8to4.comp @@ -0,0 +1,152 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int activation_type = 0; +layout (constant_id = 8) const float activation_param_0 = 0; +layout (constant_id = 9) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec4 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec4 sum; + + if (bias_term == 1) + { + sum = buffer_ld4(bias_data, gz); + } + else + { + sum = afpvec4(0.f); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + int w_offset_0 = gz * p.c * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < p.c; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k0 = buffer_ld8(weight_data, w_offset * 4 + 0); + afpvec8 k1 = buffer_ld8(weight_data, w_offset * 4 + 1); + afpvec8 k2 = buffer_ld8(weight_data, w_offset * 4 + 2); + afpvec8 k3 = buffer_ld8(weight_data, w_offset * 4 + 3); + + // sum += v * k + sum.r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum.g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum.b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum.a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = mix(sum, sum * afp(slope), lessThan(sum, afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st4(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack1to8.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack1to8.comp new file mode 100644 index 000000000..2e01d9555 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack1to8.comp @@ -0,0 +1,160 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset_0 = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < channels_g; z++) + { + afp v = buffer_ld1(bottom_blob_data, v_offset); + + afpvec8 k = buffer_ld8(weight_data, w_offset); + + // sum += v * k; + sum[0] += v * k[0]; + sum[1] += v * k[1]; + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack4to8.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack4to8.comp new file mode 100644 index 000000000..80bf9a273 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack4to8.comp @@ -0,0 +1,173 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec4 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset_0 = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < channels_g; z++) + { + afpvec4 v = buffer_ld4(bottom_blob_data, v_offset); + + afpvec4 k0 = buffer_ld4(weight_data, w_offset * 8 + 0); + afpvec4 k1 = buffer_ld4(weight_data, w_offset * 8 + 1); + afpvec4 k2 = buffer_ld4(weight_data, w_offset * 8 + 2); + afpvec4 k3 = buffer_ld4(weight_data, w_offset * 8 + 3); + afpvec4 k4 = buffer_ld4(weight_data, w_offset * 8 + 4); + afpvec4 k5 = buffer_ld4(weight_data, w_offset * 8 + 5); + afpvec4 k6 = buffer_ld4(weight_data, w_offset * 8 + 6); + afpvec4 k7 = buffer_ld4(weight_data, w_offset * 8 + 7); + + // sum += v * k; + sum[0].r += dot(v, k0); + sum[0].g += dot(v, k1); + sum[0].b += dot(v, k2); + sum[0].a += dot(v, k3); + sum[1].r += dot(v, k4); + sum[1].g += dot(v, k5); + sum[1].b += dot(v, k6); + sum[1].a += dot(v, k7); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8.comp new file mode 100644 index 000000000..630b5d071 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8.comp @@ -0,0 +1,173 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset_0 = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < channels_g; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k0 = buffer_ld8(weight_data, w_offset * 8 + 0); + afpvec8 k1 = buffer_ld8(weight_data, w_offset * 8 + 1); + afpvec8 k2 = buffer_ld8(weight_data, w_offset * 8 + 2); + afpvec8 k3 = buffer_ld8(weight_data, w_offset * 8 + 3); + afpvec8 k4 = buffer_ld8(weight_data, w_offset * 8 + 4); + afpvec8 k5 = buffer_ld8(weight_data, w_offset * 8 + 5); + afpvec8 k6 = buffer_ld8(weight_data, w_offset * 8 + 6); + afpvec8 k7 = buffer_ld8(weight_data, w_offset * 8 + 7); + + // sum += v * k + sum[0].r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum[0].g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum[0].b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum[0].a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + sum[1].r += dot(v[0], k4[0]) + dot(v[1], k4[1]); + sum[1].g += dot(v[0], k5[0]) + dot(v[1], k5[1]); + sum[1].b += dot(v[0], k6[0]) + dot(v[1], k6[1]); + sum[1].a += dot(v[0], k7[0]) + dot(v[1], k7[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to1.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to1.comp new file mode 100644 index 000000000..99b65c922 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to1.comp @@ -0,0 +1,155 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfp bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afp sum; + + if (bias_term == 1) + { + sum = buffer_ld1(bias_data, gz); + } + else + { + sum = afp(0.f); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset_0 = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < channels_g; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k = buffer_ld8(weight_data, w_offset); + + // sum += dot(v, k); + sum += dot(v[0], k[0]) + dot(v[1], k[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = sum < afp(0.f) ? sum * slope : sum; + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st1(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to4.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to4.comp new file mode 100644 index 000000000..ff889e6ab --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_group_pack8to4.comp @@ -0,0 +1,161 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec4 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec4 sum; + + if (bias_term == 1) + { + sum = buffer_ld4(bias_data, gz); + } + else + { + sum = afpvec4(0.f); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // group convolution + const int channels_g = p.c / group; + const int num_output_g = p.outc / group; + + // group id + const int gg = gz / num_output_g; + + int w_offset_0 = gz * channels_g * kernel_w * kernel_h; + int v_offset_0 = gg * channels_g * p.cstep; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + for (int z = 0; z < channels_g; z++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k0 = buffer_ld8(weight_data, w_offset * 4 + 0); + afpvec8 k1 = buffer_ld8(weight_data, w_offset * 4 + 1); + afpvec8 k2 = buffer_ld8(weight_data, w_offset * 4 + 2); + afpvec8 k3 = buffer_ld8(weight_data, w_offset * 4 + 3); + + // sum += v * k + sum.r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum.g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum.b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum.a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + + v_offset += p.cstep; + w_offset += kernel_w * kernel_h; + } + } + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = mix(sum, sum * afp(slope), lessThan(sum, afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st4(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/deconvolutiondepthwise_pack8.comp b/src/layer/vulkan/shader/deconvolutiondepthwise_pack8.comp new file mode 100644 index 000000000..fc2db1e21 --- /dev/null +++ b/src/layer/vulkan/shader/deconvolutiondepthwise_pack8.comp @@ -0,0 +1,148 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int kernel_w = 1; +layout (constant_id = 1) const int kernel_h = 1; +layout (constant_id = 2) const int dilation_w = 1; +layout (constant_id = 3) const int dilation_h = 1; +layout (constant_id = 4) const int stride_w = 1; +layout (constant_id = 5) const int stride_h = 1; +layout (constant_id = 6) const int bias_term = 0; +layout (constant_id = 7) const int group = 1; +layout (constant_id = 8) const int activation_type = 0; +layout (constant_id = 9) const float activation_param_0 = 0; +layout (constant_id = 10) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1; + const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1; + + // depth-wise deconvolution + int v_offset_0 = gz * p.cstep; + int w_offset_0 = gz * kernel_w * kernel_h; + + for (int y = 0; y < kernel_h; y++) + { + int sys = (gy + y * dilation_h - (kernel_extent_h - 1)); + if (sys < 0 || sys % stride_h != 0) + continue; + + int sy = sys / stride_h; + if (sy >= p.h) + continue; + + for (int x = 0; x < kernel_w; x++) + { + int sxs = (gx + x * dilation_w - (kernel_extent_w - 1)); + if (sxs < 0 || sxs % stride_w != 0) + continue; + + int sx = sxs / stride_w; + if (sx >= p.w) + continue; + + int v_offset = v_offset_0 + sy * p.w + sx; + int w_offset = w_offset_0 + y * kernel_w + x; + + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset); + + afpvec8 k = buffer_ld8(weight_data, w_offset); + + // sum += v * k; + sum[0] += v[0] * k[0]; + sum[1] += v[1] * k[1]; + } + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, sum); +} diff --git a/src/layer/vulkan/shader/dropout_pack8.comp b/src/layer/vulkan/shader/dropout_pack8.comp new file mode 100644 index 000000000..03c6b4fde --- /dev/null +++ b/src/layer/vulkan/shader/dropout_pack8.comp @@ -0,0 +1,59 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const float scale = 1; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = v[0] * afp(scale); + v[1] = v[1] * afp(scale); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/eltwise_pack8.comp b/src/layer/vulkan/shader/eltwise_pack8.comp new file mode 100644 index 000000000..7c85d0ee1 --- /dev/null +++ b/src/layer/vulkan/shader/eltwise_pack8.comp @@ -0,0 +1,106 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; +layout (constant_id = 1) const int coeff_term = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob1 { sfpvec8 bottom_blob1_data[]; }; +layout (binding = 1) readonly buffer bottom_blob2 { sfpvec8 bottom_blob2_data[]; }; +layout (binding = 2) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + float coeff0; + float coeff1; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v1 = buffer_ld8(bottom_blob1_data, gi); + afpvec8 v2 = buffer_ld8(bottom_blob2_data, gi); + + afpvec8 res; + + if (coeff_term == 0) + { + if (op_type == 0) + { + res[0] = v1[0] * v2[0]; + res[1] = v1[1] * v2[1]; + } + + if (op_type == 1) + { + res[0] = v1[0] + v2[0]; + res[1] = v1[1] + v2[1]; + } + + if (op_type == 2) + { + res[0] = max(v1[0], v2[0]); + res[1] = max(v1[1], v2[1]); + } + } + else + { + if (op_type == 0) + { + res[0] = v1[0] * afp(p.coeff0) * v2[0] * afp(p.coeff1); + res[1] = v1[1] * afp(p.coeff0) * v2[1] * afp(p.coeff1); + } + + if (op_type == 1) + { + res[0] = v1[0] * afp(p.coeff0) + v2[0] * afp(p.coeff1); + res[1] = v1[1] * afp(p.coeff0) + v2[1] * afp(p.coeff1); + } + + if (op_type == 2) + { + res[0] = max(v1[0] * afp(p.coeff0), v2[0] * afp(p.coeff1)); + res[1] = max(v1[1] * afp(p.coeff0), v2[1] * afp(p.coeff1)); + } + } + + buffer_st8(top_blob_data, gi, res); +} diff --git a/src/layer/vulkan/shader/flatten_pack1to8.comp b/src/layer/vulkan/shader/flatten_pack1to8.comp new file mode 100644 index 000000000..b87c1ece1 --- /dev/null +++ b/src/layer/vulkan/shader/flatten_pack1to8.comp @@ -0,0 +1,73 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + ivec4 i4 = gx * 8 + ivec4(0, 1, 2, 3); + ivec4 ii4 = i4 + 4; + + int size = p.w * p.h; + + ivec4 z4 = i4 / size; + ivec4 y4 = i4 % size / p.w; + ivec4 x4 = i4 % size % p.w; + + ivec4 zz4 = ii4 / size; + ivec4 yy4 = ii4 % size / p.w; + ivec4 xx4 = ii4 % size % p.w; + + ivec4 v_offset = z4 * p.cstep + y4 * p.w + x4; + ivec4 vv_offset = zz4 * p.cstep + yy4 * p.w + xx4; + + buffer_cp1to8(top_blob_data, gx, bottom_blob_data, v_offset, vv_offset); +} diff --git a/src/layer/vulkan/shader/flatten_pack4.comp b/src/layer/vulkan/shader/flatten_pack4.comp index 7a79bb36b..49215a6eb 100644 --- a/src/layer/vulkan/shader/flatten_pack4.comp +++ b/src/layer/vulkan/shader/flatten_pack4.comp @@ -26,7 +26,7 @@ layout (local_size_y_id = 234) in; layout (local_size_z_id = 235) in; #if NCNN_fp16_packed -layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 0) readonly buffer bottom_blob { sfpvec2 bottom_blob_data[]; }; #else layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; #endif @@ -60,15 +60,15 @@ void main() ivec4 i4 = gx * 4 + ivec4(0, 1, 2, 3); ivec4 v_offset; - ivec4 lane4; + ivec4 lane2; if (p.dims == 2) { ivec4 y4 = i4 / p.w; ivec4 x4 = i4 % p.w; - v_offset = (y4 / 4) * p.w + x4; - lane4 = y4 % 4; + v_offset = ((y4 / 4) * p.w + x4) * 2 + (y4 % 4) / 2; + lane2 = y4 % 2; } else // if (p.dims == 3) { @@ -78,16 +78,16 @@ void main() ivec4 y4 = i4 % size / p.w; ivec4 x4 = i4 % size % p.w; - v_offset = (z4 / 4) * p.cstep + y4 * p.w + x4; - lane4 = z4 % 4; + v_offset = ((z4 / 4) * p.cstep + y4 * p.w + x4) * 2 + (z4 % 4) / 2; + lane2 = z4 % 2; } - vec4 vr = buffer_ld4(bottom_blob_data, v_offset.r); - vec4 vg = buffer_ld4(bottom_blob_data, v_offset.g); - vec4 vb = buffer_ld4(bottom_blob_data, v_offset.b); - vec4 va = buffer_ld4(bottom_blob_data, v_offset.a); + afpvec2 vr = buffer_ld2(bottom_blob_data, v_offset.r); + afpvec2 vg = buffer_ld2(bottom_blob_data, v_offset.g); + afpvec2 vb = buffer_ld2(bottom_blob_data, v_offset.b); + afpvec2 va = buffer_ld2(bottom_blob_data, v_offset.a); - vec4 v = vec4(vr[lane4.r], vg[lane4.g], vb[lane4.b], va[lane4.a]); + afpvec4 v = afpvec4(vr[lane2.r], vg[lane2.g], vb[lane2.b], va[lane2.a]); buffer_st4(top_blob_data, gx, v); #else diff --git a/src/layer/vulkan/shader/flatten_pack8.comp b/src/layer/vulkan/shader/flatten_pack8.comp new file mode 100644 index 000000000..5e8ea4e21 --- /dev/null +++ b/src/layer/vulkan/shader/flatten_pack8.comp @@ -0,0 +1,152 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +#if NCNN_fp16_packed +layout (binding = 0) readonly buffer bottom_blob { sfpvec2 bottom_blob_data[]; }; +#else +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +#endif +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + +#if NCNN_fp16_packed + ivec4 i4 = gx * 8 + ivec4(0, 1, 2, 3); + ivec4 ii4 = i4 + 4; + + ivec4 v_offset; + ivec4 lane4; + ivec4 vv_offset; + ivec4 lane8; + + if (p.dims == 2) + { + ivec4 y4 = i4 / p.w; + ivec4 x4 = i4 % p.w; + + ivec4 yy4 = ii4 / p.w; + ivec4 xx4 = ii4 % p.w; + + v_offset = ((y4 / 8) * p.w + x4) * 2 + (y4 % 8) / 2; + lane4 = y4 % 2; + + vv_offset = ((yy4 / 8) * p.w + xx4) * 2 + (yy4 % 8) / 2; + lane8 = yy4 % 2; + } + else // if (p.dims == 3) + { + int size = p.w * p.h; + + ivec4 z4 = i4 / size; + ivec4 y4 = i4 % size / p.w; + ivec4 x4 = i4 % size % p.w; + + ivec4 zz4 = ii4 / size; + ivec4 yy4 = ii4 % size / p.w; + ivec4 xx4 = ii4 % size % p.w; + + v_offset = ((z4 / 8) * p.cstep + y4 * p.w + x4) * 2 + (z4 % 4) / 2; + lane4 = z4 % 2; + + vv_offset = ((zz4 / 8) * p.cstep + yy4 * p.w + xx4) * 2 + (zz4 % 4) / 2; + lane8 = zz4 % 2; + } + + afpvec2 vr = buffer_ld2(bottom_blob_data, v_offset.r); + afpvec2 vg = buffer_ld2(bottom_blob_data, v_offset.g); + afpvec2 vb = buffer_ld2(bottom_blob_data, v_offset.b); + afpvec2 va = buffer_ld2(bottom_blob_data, v_offset.a); + + afpvec2 vvr = buffer_ld2(bottom_blob_data, vv_offset.r); + afpvec2 vvg = buffer_ld2(bottom_blob_data, vv_offset.g); + afpvec2 vvb = buffer_ld2(bottom_blob_data, vv_offset.b); + afpvec2 vva = buffer_ld2(bottom_blob_data, vv_offset.a); + + afpvec8 v = afpvec8(vr[lane4.r], vg[lane4.g], vb[lane4.b], va[lane4.a], vvr[lane8.r], vvg[lane8.g], vvb[lane8.b], vva[lane8.a]); + + buffer_st8(top_blob_data, gx, v); +#else + ivec4 v_offset; + ivec4 vv_offset; + + if (p.dims == 2) + { + int y = gx / p.w; + ivec4 i4 = (gx % p.w) * 8 + ivec4(0, 1, 2, 3); + ivec4 ii4 = i4 + 4; + + ivec4 k4 = i4 / p.w; + ivec4 si4 = i4 % p.w; + ivec4 kk4 = ii4 / p.w; + ivec4 sii4 = ii4 % p.w; + + v_offset = ivec4(y * 8 * p.w) + si4 * 8 + k4; + vv_offset = ivec4(y * 8 * p.w) + sii4 * 8 + kk4; + } + else // if (p.dims == 3) + { + int size = p.w * p.h; + + int z = gx / size; + ivec4 i4 = (gx % size) * 8 + ivec4(0, 1, 2, 3); + ivec4 ii4 = i4 + 4; + + ivec4 k4 = i4 / size; + ivec4 si4 = i4 % size; + ivec4 kk4 = ii4 / size; + ivec4 sii4 = ii4 % size; + + v_offset = ivec4(z * 8 * p.cstep) + si4 * 8 + k4; + vv_offset = ivec4(z * 8 * p.cstep) + sii4 * 8 + kk4; + } + + buffer_cp1to8(top_blob_data, gx, bottom_blob_data, v_offset, vv_offset); +#endif +} diff --git a/src/layer/vulkan/shader/hardsigmoid_pack8.comp b/src/layer/vulkan/shader/hardsigmoid_pack8.comp new file mode 100644 index 000000000..84e4a3fd4 --- /dev/null +++ b/src/layer/vulkan/shader/hardsigmoid_pack8.comp @@ -0,0 +1,60 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const float alpha = 0; +layout (constant_id = 1) const float beta = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = clamp(v[0] * afp(alpha) + afp(beta), afp(0.f), afp(1.f)); + v[1] = clamp(v[1] * afp(alpha) + afp(beta), afp(0.f), afp(1.f)); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/hardswish_pack8.comp b/src/layer/vulkan/shader/hardswish_pack8.comp new file mode 100644 index 000000000..fe5d7882f --- /dev/null +++ b/src/layer/vulkan/shader/hardswish_pack8.comp @@ -0,0 +1,60 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const float alpha = 0; +layout (constant_id = 1) const float beta = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = v[0] * clamp(v[0] * afp(alpha) + afp(beta), afp(0.f), afp(1.f)); + v[1] = v[1] * clamp(v[1] * afp(alpha) + afp(beta), afp(0.f), afp(1.f)); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/innerproduct_pack1to8.comp b/src/layer/vulkan/shader/innerproduct_pack1to8.comp new file mode 100644 index 000000000..2b06e3f6e --- /dev/null +++ b/src/layer/vulkan/shader/innerproduct_pack1to8.comp @@ -0,0 +1,112 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; +layout (constant_id = 1) const int activation_type = 0; +layout (constant_id = 2) const float activation_param_0 = 0; +layout (constant_id = 3) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gx * p.w; + + for (int i = 0; i < p.w; i++) + { + afp v = buffer_ld1(bottom_blob_data, i); + + afpvec8 k = buffer_ld8(weight_data, w_offset + i); + + // sum += v * k; + sum[0] += v * k[0]; + sum[1] += v * k[1]; + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + buffer_st8(top_blob_data, gx, sum); +} diff --git a/src/layer/vulkan/shader/innerproduct_pack4to8.comp b/src/layer/vulkan/shader/innerproduct_pack4to8.comp new file mode 100644 index 000000000..0b66b5174 --- /dev/null +++ b/src/layer/vulkan/shader/innerproduct_pack4to8.comp @@ -0,0 +1,125 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; +layout (constant_id = 1) const int activation_type = 0; +layout (constant_id = 2) const float activation_param_0 = 0; +layout (constant_id = 3) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec4 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gz); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gx * p.w; + + for (int i = 0; i < p.w; i++) + { + afpvec4 v = buffer_ld4(bottom_blob_data, i); + + afpvec4 k0 = buffer_ld4(weight_data, (w_offset + i) * 8 + 0); + afpvec4 k1 = buffer_ld4(weight_data, (w_offset + i) * 8 + 1); + afpvec4 k2 = buffer_ld4(weight_data, (w_offset + i) * 8 + 2); + afpvec4 k3 = buffer_ld4(weight_data, (w_offset + i) * 8 + 3); + afpvec4 k4 = buffer_ld4(weight_data, (w_offset + i) * 8 + 4); + afpvec4 k5 = buffer_ld4(weight_data, (w_offset + i) * 8 + 5); + afpvec4 k6 = buffer_ld4(weight_data, (w_offset + i) * 8 + 6); + afpvec4 k7 = buffer_ld4(weight_data, (w_offset + i) * 8 + 7); + + // sum += v * k; + sum[0].r += dot(v, k0); + sum[0].g += dot(v, k1); + sum[0].b += dot(v, k2); + sum[0].a += dot(v, k3); + sum[1].r += dot(v, k4); + sum[1].g += dot(v, k5); + sum[1].b += dot(v, k6); + sum[1].a += dot(v, k7); + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + buffer_st8(top_blob_data, gx, sum); +} diff --git a/src/layer/vulkan/shader/innerproduct_pack8.comp b/src/layer/vulkan/shader/innerproduct_pack8.comp new file mode 100644 index 000000000..f7bb49eeb --- /dev/null +++ b/src/layer/vulkan/shader/innerproduct_pack8.comp @@ -0,0 +1,125 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; +layout (constant_id = 1) const int activation_type = 0; +layout (constant_id = 2) const float activation_param_0 = 0; +layout (constant_id = 3) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec8 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + afpvec8 sum; + + if (bias_term == 1) + { + sum = buffer_ld8(bias_data, gx); + } + else + { + sum = afpvec8(afpvec4(0.f), afpvec4(0.f)); + } + + int w_offset = gx * p.w; + + for (int i = 0; i < p.w; i++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, i); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + i) * 8 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + i) * 8 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + i) * 8 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + i) * 8 + 3); + afpvec8 k4 = buffer_ld8(weight_data, (w_offset + i) * 8 + 4); + afpvec8 k5 = buffer_ld8(weight_data, (w_offset + i) * 8 + 5); + afpvec8 k6 = buffer_ld8(weight_data, (w_offset + i) * 8 + 6); + afpvec8 k7 = buffer_ld8(weight_data, (w_offset + i) * 8 + 7); + + // sum += v * k + sum[0].r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum[0].g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum[0].b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum[0].a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + sum[1].r += dot(v[0], k4[0]) + dot(v[1], k4[1]); + sum[1].g += dot(v[0], k5[0]) + dot(v[1], k5[1]); + sum[1].b += dot(v[0], k6[0]) + dot(v[1], k6[1]); + sum[1].a += dot(v[0], k7[0]) + dot(v[1], k7[1]); + } + + if (activation_type == 1) + { + sum[0] = max(sum[0], afp(0.f)); + sum[1] = max(sum[1], afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum[0] = mix(sum[0], sum[0] * afp(slope), lessThan(sum[0], afpvec4(0.f))); + sum[1] = mix(sum[1], sum[1] * afp(slope), lessThan(sum[1], afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum[0] = clamp(sum[0], const_min, const_max); + sum[1] = clamp(sum[1], const_min, const_max); + } + if (activation_type == 4) + { + sum[0] = afp(1.f) / (afp(1.f) + exp(-sum[0])); + sum[1] = afp(1.f) / (afp(1.f) + exp(-sum[1])); + } + + buffer_st8(top_blob_data, gx, sum); +} diff --git a/src/layer/vulkan/shader/innerproduct_pack8to1.comp b/src/layer/vulkan/shader/innerproduct_pack8to1.comp new file mode 100644 index 000000000..e7e3912cb --- /dev/null +++ b/src/layer/vulkan/shader/innerproduct_pack8to1.comp @@ -0,0 +1,107 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; +layout (constant_id = 1) const int activation_type = 0; +layout (constant_id = 2) const float activation_param_0 = 0; +layout (constant_id = 3) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfp bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + afp sum; + + if (bias_term == 1) + { + sum = buffer_ld1(bias_data, gx); + } + else + { + sum = afp(0.f); + } + + int w_offset = gx * p.w; + + for (int i = 0; i < p.w; i++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, i); + + afpvec8 k = buffer_ld8(weight_data, w_offset + i); + + // sum += dot(v, k); + sum += dot(v[0], k[0]) + dot(v[1], k[1]); + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = sum < afp(0.f) ? sum * slope : sum; + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + buffer_st1(top_blob_data, gx, sum); +} diff --git a/src/layer/vulkan/shader/innerproduct_pack8to4.comp b/src/layer/vulkan/shader/innerproduct_pack8to4.comp new file mode 100644 index 000000000..d5f5edca4 --- /dev/null +++ b/src/layer/vulkan/shader/innerproduct_pack8to4.comp @@ -0,0 +1,113 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; +layout (constant_id = 1) const int activation_type = 0; +layout (constant_id = 2) const float activation_param_0 = 0; +layout (constant_id = 3) const float activation_param_1 = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; +layout (binding = 2) readonly buffer weight_blob { sfpvec8 weight_data[]; }; +layout (binding = 3) readonly buffer bias_blob { sfpvec4 bias_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= 1 || gz >= 1) + return; + + afpvec4 sum; + + if (bias_term == 1) + { + sum = buffer_ld4(bias_data, gz); + } + else + { + sum = afpvec4(0.f); + } + + int w_offset = gx * p.w; + + for (int i = 0; i < p.w; i++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, i); + + afpvec8 k0 = buffer_ld8(weight_data, (w_offset + i) * 4 + 0); + afpvec8 k1 = buffer_ld8(weight_data, (w_offset + i) * 4 + 1); + afpvec8 k2 = buffer_ld8(weight_data, (w_offset + i) * 4 + 2); + afpvec8 k3 = buffer_ld8(weight_data, (w_offset + i) * 4 + 3); + + // sum += v * k + sum.r += dot(v[0], k0[0]) + dot(v[1], k0[1]); + sum.g += dot(v[0], k1[0]) + dot(v[1], k1[1]); + sum.b += dot(v[0], k2[0]) + dot(v[1], k2[1]); + sum.a += dot(v[0], k3[0]) + dot(v[1], k3[1]); + } + + if (activation_type == 1) + { + sum = max(sum, afp(0.f)); + } + if (activation_type == 2) + { + const afp slope = afp(activation_param_0); + sum = mix(sum, sum * afp(slope), lessThan(sum, afpvec4(0.f))); + } + if (activation_type == 3) + { + const afp const_min = afp(activation_param_0); + const afp const_max = afp(activation_param_1); + sum = clamp(sum, const_min, const_max); + } + if (activation_type == 4) + { + sum = afp(1.f) / (afp(1.f) + exp(-sum)); + } + + buffer_st4(top_blob_data, gx, sum); +} diff --git a/src/layer/vulkan/shader/packing_1to8.comp b/src/layer/vulkan/shader/packing_1to8.comp new file mode 100644 index 000000000..622281dd4 --- /dev/null +++ b/src/layer/vulkan/shader/packing_1to8.comp @@ -0,0 +1,84 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + ivec4 v_offset; + ivec4 vv_offset; + + if (p.dims == 1) + { + ivec4 x4 = ivec4(gx * 8) + ivec4(0, 1, 2, 3); + + v_offset = x4; + vv_offset = x4 + 4; + } + else if (p.dims == 2) + { + ivec4 y4 = ivec4(gy * 8) + ivec4(0, 1, 2, 3); + + v_offset = y4 * p.w + gx; + vv_offset = (y4 + 4) * p.w + gx; + } + else // if (p.dims == 3) + { + ivec4 z4 = ivec4(gz * 8) + ivec4(0, 1, 2, 3); + + v_offset = z4 * p.cstep + ivec4(gy * p.w + gx); + vv_offset = (z4 + 4) * p.cstep + ivec4(gy * p.w + gx); + } + + int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_cp1to8(top_blob_data, gi, bottom_blob_data, v_offset, vv_offset); +} diff --git a/src/layer/vulkan/shader/packing_4to8.comp b/src/layer/vulkan/shader/packing_4to8.comp new file mode 100644 index 000000000..851600293 --- /dev/null +++ b/src/layer/vulkan/shader/packing_4to8.comp @@ -0,0 +1,80 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec4 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + ivec2 v_offset; + + if (p.dims == 1) + { + ivec2 x2 = ivec2(gx * 2) + ivec2(0, 1); + + v_offset = x2; + } + else if (p.dims == 2) + { + ivec2 y2 = ivec2(gy * 2) + ivec2(0, 1); + + v_offset = y2 * p.w + gx; + } + else // if (p.dims == 3) + { + ivec2 z2 = ivec2(gz * 2) + ivec2(0, 1); + + v_offset = z2 * p.cstep + ivec2(gy * p.w + gx); + } + + int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_cp4to8(top_blob_data, gi, bottom_blob_data, v_offset); +} diff --git a/src/layer/vulkan/shader/packing_8to1.comp b/src/layer/vulkan/shader/packing_8to1.comp new file mode 100644 index 000000000..6aa321c6a --- /dev/null +++ b/src/layer/vulkan/shader/packing_8to1.comp @@ -0,0 +1,84 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfp top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + ivec4 v_offset; + ivec4 vv_offset; + + if (p.dims == 1) + { + ivec4 x4 = ivec4(gx * 8) + ivec4(0, 1, 2, 3); + + v_offset = x4; + vv_offset = x4 + 4; + } + else if (p.dims == 2) + { + ivec4 y4 = ivec4(gy * 8) + ivec4(0, 1, 2, 3); + + v_offset = y4 * p.outw + gx; + vv_offset = (y4 + 4) * p.outw + gx; + } + else // if (p.dims == 3) + { + ivec4 z4 = ivec4(gz * 8) + ivec4(0, 1, 2, 3); + + v_offset = z4 * p.outcstep + ivec4(gy * p.outw + gx); + vv_offset = (z4 + 4) * p.outcstep + ivec4(gy * p.outw + gx); + } + + int gi = gz * p.cstep + gy * p.w + gx; + + buffer_cp8to1(top_blob_data, v_offset, vv_offset, bottom_blob_data, gi); +} diff --git a/src/layer/vulkan/shader/packing_8to4.comp b/src/layer/vulkan/shader/packing_8to4.comp new file mode 100644 index 000000000..474545a1e --- /dev/null +++ b/src/layer/vulkan/shader/packing_8to4.comp @@ -0,0 +1,80 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec4 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + ivec2 v_offset; + + if (p.dims == 1) + { + ivec2 x2 = ivec2(gx * 2) + ivec2(0, 1); + + v_offset = x2; + } + else if (p.dims == 2) + { + ivec2 y2 = ivec2(gy * 2) + ivec2(0, 1); + + v_offset = y2 * p.outw + gx; + } + else // if (p.dims == 3) + { + ivec2 z2 = ivec2(gz * 2) + ivec2(0, 1); + + v_offset = z2 * p.outcstep + ivec2(gy * p.outw + gx); + } + + int gi = gz * p.cstep + gy * p.w + gx; + + buffer_cp8to4(top_blob_data, v_offset, bottom_blob_data, gi); +} diff --git a/src/layer/vulkan/shader/padding_pack8.comp b/src/layer/vulkan/shader/padding_pack8.comp new file mode 100644 index 000000000..6798a11ee --- /dev/null +++ b/src/layer/vulkan/shader/padding_pack8.comp @@ -0,0 +1,104 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int type = 1; +layout (constant_id = 1) const float value = 0; +layout (constant_id = 2) const int per_channel_pad = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; +layout (binding = 2) readonly buffer per_channel_pad_blob { sfpvec8 per_channel_pad_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int left; + int top; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + int x = gx - p.left; + int y = gy - p.top; + + if (type == 0) + { + if (x >= 0 && x < p.w && y >= 0 && y < p.h) + { + int v_offset = gz * p.cstep + y * p.w + x; + buffer_cp8(top_blob_data, gi, bottom_blob_data, v_offset); + } + else if (per_channel_pad == 1) + { + buffer_cp8(top_blob_data, gi, per_channel_pad_blob_data, gz); + } + else + { + afpvec8 v = afpvec8(afpvec4(value), afpvec4(value)); + buffer_st8(top_blob_data, gi, v); + } + } + else if (type == 1) + { + x = clamp(x, 0, p.w - 1); + y = clamp(y, 0, p.h - 1); + + int v_offset = gz * p.cstep + y * p.w + x; + buffer_cp8(top_blob_data, gi, bottom_blob_data, v_offset); + } + else if (type == 2) + { + x = abs(x); + y = abs(y); + x = (p.w - 1) - abs(x - (p.w - 1)); + y = (p.h - 1) - abs(y - (p.h - 1)); + + int v_offset = gz * p.cstep + y * p.w + x; + buffer_cp8(top_blob_data, gi, bottom_blob_data, v_offset); + } +} diff --git a/src/layer/vulkan/shader/pooling_global_pack8.comp b/src/layer/vulkan/shader/pooling_global_pack8.comp new file mode 100644 index 000000000..63f747445 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_global_pack8.comp @@ -0,0 +1,93 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +layout (constant_id = 0) const int pooling_type = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + int size = p.w * p.h; + int v_offset = gx * p.cstep; + + afpvec8 res; + + if (pooling_type == 0) + { + res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); + + for (int i = 0; i < size; i++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); + res[0] = max(res[0], v[0]); + res[1] = max(res[1], v[1]); + } + } + else if (pooling_type == 1) + { + res = afpvec8(afpvec4(0.f), afpvec4(0.f)); + + for (int i = 0; i < size; i++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + i); + res[0] += v[0]; + res[1] += v[1]; + } + + afp area = afp(size); + res[0] /= area; + res[1] /= area; + } + + buffer_st8(top_blob_data, gx, res); +} diff --git a/src/layer/vulkan/shader/pooling_pack8.comp b/src/layer/vulkan/shader/pooling_pack8.comp new file mode 100644 index 000000000..195a29c24 --- /dev/null +++ b/src/layer/vulkan/shader/pooling_pack8.comp @@ -0,0 +1,163 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +#define FLT_MAX 3.402823466e+38 + +layout (constant_id = 0) const int pooling_type = 0; +layout (constant_id = 1) const int kernel_w = 1; +layout (constant_id = 2) const int kernel_h = 1; +layout (constant_id = 3) const int stride_w = 1; +layout (constant_id = 4) const int stride_h = 1; +layout (constant_id = 5) const int pad_left = 0; +layout (constant_id = 6) const int pad_right = 0; +layout (constant_id = 7) const int pad_top = 0; +layout (constant_id = 8) const int pad_bottom = 0; +layout (constant_id = 9) const int global_pooling = 0; +layout (constant_id = 10) const int pad_mode = 0; +layout (constant_id = 11) const int avgpool_count_include_pad = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int wtailpad; + int htailpad; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 res; + + if (pooling_type == 0) + { + res = afpvec8(afpvec4(-FLT_MAX), afpvec4(-FLT_MAX)); + + int v_offset = gz * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x); + res[0] = max(res[0], v[0]); + res[1] = max(res[1], v[1]); + } + + v_offset += p.w; + } + } + else if (pooling_type == 1 && avgpool_count_include_pad == 0) + { + res = afpvec8(afpvec4(0.f), afpvec4(0.f)); + + int sx = gx * stride_w; + int sy = gy * stride_h; + + int v_offset = gz * p.cstep + sy * p.w + sx; + int area = 0; + + for (int y = 0; y < kernel_h; y++) + { + if (sy + y < pad_top) + { + v_offset += p.w; + continue; + } + + if (sy + y >= p.h - pad_bottom - p.htailpad) + break; + + for (int x = 0; x < kernel_w; x++) + { + if (sx + x < pad_left) + { + continue; + } + + if (sx + x >= p.w - pad_right - p.wtailpad) + break; + + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x); + res[0] += v[0]; + res[1] += v[1]; + area += 1; + } + + v_offset += p.w; + } + + res[0] /= afp(area); + res[1] /= afp(area); + } + else if (pooling_type == 1 && avgpool_count_include_pad == 1) + { + res = afpvec8(afpvec4(0.f), afpvec4(0.f)); + + int v_offset = gz * p.cstep + gy * stride_h * p.w + gx * stride_w; + + for (int y = 0; y < kernel_h; y++) + { + for (int x = 0; x < kernel_w; x++) + { + afpvec8 v = buffer_ld8(bottom_blob_data, v_offset + x); + res[0] += v[0]; + res[1] += v[1]; + } + + v_offset += p.w; + } + + afp area = afp(kernel_w * kernel_h); + res[0] /= area; + res[1] /= area; + } + + const int gi = gz * p.outcstep + gy * p.outw + gx; + + buffer_st8(top_blob_data, gi, res); +} diff --git a/src/layer/vulkan/shader/prelu_pack8.comp b/src/layer/vulkan/shader/prelu_pack8.comp new file mode 100644 index 000000000..27cc60d17 --- /dev/null +++ b/src/layer/vulkan/shader/prelu_pack8.comp @@ -0,0 +1,64 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int num_slope = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) readonly buffer slope_blob { sfpvec8 slope_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + const ivec3 gxyz = ivec3(gx, gy, gz); + const int abi = num_slope > 1 ? gxyz[p.dims - 1] : 0; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + afpvec8 slope = buffer_ld8(slope_blob_data, abi); + + v[0] = mix(v[0], v[0] * slope[0], lessThan(v[0], afpvec4(0.f))); + v[1] = mix(v[1], v[1] * slope[1], lessThan(v[1], afpvec4(0.f))); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/relu_pack8.comp b/src/layer/vulkan/shader/relu_pack8.comp new file mode 100644 index 000000000..5d16f4e43 --- /dev/null +++ b/src/layer/vulkan/shader/relu_pack8.comp @@ -0,0 +1,67 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const float slope = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + if (slope == 0) + { + v[0] = max(v[0], afp(0.f)); + v[1] = max(v[1], afp(0.f)); + } + else + { + v[0] = mix(v[0], v[0] * afp(slope), lessThan(v[0], afpvec4(0.f))); + v[1] = mix(v[1], v[1] * afp(slope), lessThan(v[1], afpvec4(0.f))); + } + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/scale_pack8.comp b/src/layer/vulkan/shader/scale_pack8.comp new file mode 100644 index 000000000..f069e225f --- /dev/null +++ b/src/layer/vulkan/shader/scale_pack8.comp @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int bias_term = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) readonly buffer scale_blob { sfpvec8 scale_blob_data[]; }; +layout (binding = 2) readonly buffer bias_blob { sfpvec8 bias_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + const ivec3 gxyz = ivec3(gx, gy, gz); + const int abi = gxyz[p.dims - 1]; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + afpvec8 scale = buffer_ld8(scale_blob_data, abi); + + if (bias_term == 1) + { + afpvec8 bias = buffer_ld8(bias_blob_data, abi); + v[0] = scale[0] * v[0] + bias[0]; + v[1] = scale[1] * v[1] + bias[1]; + } + else + { + v[0] = scale[0] * v[0]; + v[1] = scale[1] * v[1]; + } + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/shufflechannel_pack8.comp b/src/layer/vulkan/shader/shufflechannel_pack8.comp new file mode 100644 index 000000000..4836b8d09 --- /dev/null +++ b/src/layer/vulkan/shader/shufflechannel_pack8.comp @@ -0,0 +1,91 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int group = 1; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int channels_g = p.c * 8 / group; + + ivec4 gz4 = ivec4(gz * 8) + ivec4(0, 1, 2, 3); + ivec4 gzz4 = gz4 + 4; + + ivec4 z4 = (gz4 % group) * channels_g + (gz4 / group); + ivec4 zz4 = (gzz4 % group) * channels_g + (gzz4 / group); + + ivec4 v_offset = (z4 / 8) * p.cstep + gy * p.w + gx; + ivec4 vv_offset = (zz4 / 8) * p.cstep + gy * p.w + gx; + + ivec4 sz4 = (z4 % 8) / 4; + ivec4 szz4 = (zz4 % 8) / 4; + + ivec4 lane4 = z4 % 4; + ivec4 lane8 = zz4 % 4; + + // v = v4[lane] + int gi = gz * p.outcstep + gy * p.outw + gx; + + afpvec8 vr = buffer_ld8(bottom_blob_data, v_offset.r); + afpvec8 vg = buffer_ld8(bottom_blob_data, v_offset.g); + afpvec8 vb = buffer_ld8(bottom_blob_data, v_offset.b); + afpvec8 va = buffer_ld8(bottom_blob_data, v_offset.a); + + afpvec8 vvr = buffer_ld8(bottom_blob_data, vv_offset.r); + afpvec8 vvg = buffer_ld8(bottom_blob_data, vv_offset.g); + afpvec8 vvb = buffer_ld8(bottom_blob_data, vv_offset.b); + afpvec8 vva = buffer_ld8(bottom_blob_data, vv_offset.a); + + afpvec8 v = afpvec8(vr[sz4.r][lane4.r], vg[sz4.g][lane4.g], vb[sz4.b][lane4.b], va[sz4.a][lane4.a], vvr[szz4.r][lane8.r], vvg[szz4.g][lane8.g], vvb[szz4.b][lane8.b], vva[szz4.a][lane8.a]); + + buffer_st8(top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/sigmoid_pack8.comp b/src/layer/vulkan/shader/sigmoid_pack8.comp new file mode 100644 index 000000000..e9bcc6d5f --- /dev/null +++ b/src/layer/vulkan/shader/sigmoid_pack8.comp @@ -0,0 +1,57 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = afp(1.f) / (afp(1.f) + exp(-v[0])); + v[1] = afp(1.f) / (afp(1.f) + exp(-v[1])); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/slice_pack1to8.comp b/src/layer/vulkan/shader/slice_pack1to8.comp new file mode 100644 index 000000000..0a9e5b91b --- /dev/null +++ b/src/layer/vulkan/shader/slice_pack1to8.comp @@ -0,0 +1,75 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfp bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int offset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + ivec3 gxyz = ivec3(gx, gy, gz); + + gxyz[p.dims - 1] *= 8; + gxyz[p.dims - 1 - axis] += p.offset; + + int v_offset_0 = gxyz.z * p.outcstep + gxyz.y * p.outw + gxyz.x; + + ivec3 gxyz4 = ivec3(1, p.outw, p.outcstep); + + ivec4 v_offset = v_offset_0 + ivec4(0, 1, 2, 3) * gxyz4[p.dims - 1 - axis]; + ivec4 vv_offset = v_offset + 4 * gxyz4[p.dims - 1 - axis]; + + buffer_cp1to8(top_blob_data, gi, bottom_blob_data, v_offset, vv_offset); +} diff --git a/src/layer/vulkan/shader/slice_pack8.comp b/src/layer/vulkan/shader/slice_pack8.comp new file mode 100644 index 000000000..39c1a722c --- /dev/null +++ b/src/layer/vulkan/shader/slice_pack8.comp @@ -0,0 +1,69 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_blob { sfpvec8 bottom_blob_data[]; }; +layout (binding = 1) writeonly buffer top_blob { sfpvec8 top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; + + int offset; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + ivec3 gxyz = ivec3(gx, gy, gz); + + gxyz[p.dims - 1 - axis] += p.offset; + + int v_offset = gxyz.z * p.outcstep + gxyz.y * p.outw + gxyz.x; + + buffer_cp8(top_blob_data, gi, bottom_blob_data, v_offset); +} diff --git a/src/layer/vulkan/shader/softmax_div_sum_pack8.comp b/src/layer/vulkan/shader/softmax_div_sum_pack8.comp new file mode 100644 index 000000000..48a15da3b --- /dev/null +++ b/src/layer/vulkan/shader/softmax_div_sum_pack8.comp @@ -0,0 +1,93 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) readonly buffer sum_workspace { sfpvec8 sum_workspace_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + afpvec8 sum; + + if (p.dims == 1) // axis == 0 + { + sum = buffer_ld8(sum_workspace_data, 0); + } + else if (p.dims == 2 && axis == 0) + { + sum = buffer_ld8(sum_workspace_data, gx); + } + else if (p.dims == 2 && axis == 1) + { + sum = buffer_ld8(sum_workspace_data, gy); + } + else if (p.dims == 3 && axis == 0) + { + sum = buffer_ld8(sum_workspace_data, gy * p.w + gx); + } + else if (p.dims == 3 && axis == 1) + { + sum = buffer_ld8(sum_workspace_data, gz * p.w + gx); + } + else if (p.dims == 3 && axis == 2) + { + sum = buffer_ld8(sum_workspace_data, gz * p.h + gy); + } + + v[0] /= sum[0]; + v[1] /= sum[1]; + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/softmax_exp_sub_max_pack8.comp b/src/layer/vulkan/shader/softmax_exp_sub_max_pack8.comp new file mode 100644 index 000000000..d292cf5ed --- /dev/null +++ b/src/layer/vulkan/shader/softmax_exp_sub_max_pack8.comp @@ -0,0 +1,93 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) readonly buffer max_workspace { sfpvec8 max_workspace_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + afpvec8 max_value; + + if (p.dims == 1) // axis == 0 + { + max_value = buffer_ld8(max_workspace_data, 0); + } + else if (p.dims == 2 && axis == 0) + { + max_value = buffer_ld8(max_workspace_data, gx); + } + else if (p.dims == 2 && axis == 1) + { + max_value = buffer_ld8(max_workspace_data, gy); + } + else if (p.dims == 3 && axis == 0) + { + max_value = buffer_ld8(max_workspace_data, gy * p.w + gx); + } + else if (p.dims == 3 && axis == 1) + { + max_value = buffer_ld8(max_workspace_data, gz * p.w + gx); + } + else if (p.dims == 3 && axis == 2) + { + max_value = buffer_ld8(max_workspace_data, gz * p.h + gy); + } + + v[0] = exp(v[0] - max_value[0]); + v[1] = exp(v[1] - max_value[1]); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/softmax_reduce_max_pack8.comp b/src/layer/vulkan/shader/softmax_reduce_max_pack8.comp new file mode 100644 index 000000000..ef766f22b --- /dev/null +++ b/src/layer/vulkan/shader/softmax_reduce_max_pack8.comp @@ -0,0 +1,148 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) writeonly buffer max_workspace { sfpvec8 max_workspace_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 max_value = afpvec8(afpvec4(-99999999.f), afpvec4(-99999999.f)); + + if (p.dims == 1) // axis == 0 + { + for (int i = 0; i < p.w; i++) + { + afpvec8 v = buffer_ld8(bottom_top_blob_data, i); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + afpvec4 max4 = max(max_value[0], max_value[1]); + afpvec2 max2 = max(max4.rg, max4.ba); + afp max1 = max(max2.r, max2.g); + max_value = afpvec8(afpvec4(max1), afpvec4(max1)); + buffer_st8(max_workspace_data, 0, max_value); + return; + } + + if (p.dims == 2 && axis == 0) + { + for (int i = 0; i < p.h; i++) + { + int v_offset = i * p.w + gx; + afpvec8 v = buffer_ld8(bottom_top_blob_data, v_offset); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + afpvec4 max4 = max(max_value[0], max_value[1]); + afpvec2 max2 = max(max4.rg, max4.ba); + afp max1 = max(max2.r, max2.g); + max_value = afpvec8(afpvec4(max1), afpvec4(max1)); + buffer_st8(max_workspace_data, gx, max_value); + return; + } + + if (p.dims == 2 && axis == 1) + { + for (int i = 0; i < p.w; i++) + { + int v_offset = gx * p.w + i; + afpvec8 v = buffer_ld8(bottom_top_blob_data, v_offset); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + buffer_st8(max_workspace_data, gx, max_value); + return; + } + + if (p.dims == 3 && axis == 0) + { + for (int i = 0; i < p.c; i++) + { + int v_offset = i * p.cstep + gy * p.w + gx; + afpvec8 v = buffer_ld8(bottom_top_blob_data, v_offset); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + afpvec4 max4 = max(max_value[0], max_value[1]); + afpvec2 max2 = max(max4.rg, max4.ba); + afp max1 = max(max2.r, max2.g); + max_value = afpvec8(afpvec4(max1), afpvec4(max1)); + buffer_st8(max_workspace_data, gy * p.w + gx, max_value); + return; + } + + if (p.dims == 3 && axis == 1) + { + for (int i = 0; i < p.h; i++) + { + int v_offset = gy * p.cstep + i * p.w + gx; + afpvec8 v = buffer_ld8(bottom_top_blob_data, v_offset); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + buffer_st8(max_workspace_data, gy * p.w + gx, max_value); + return; + } + + if (p.dims == 3 && axis == 2) + { + for (int i = 0; i < p.w; i++) + { + int v_offset = gy * p.cstep + gx * p.w + i; + afpvec8 v = buffer_ld8(bottom_top_blob_data, v_offset); + max_value[0] = max(max_value[0], v[0]); + max_value[1] = max(max_value[1], v[1]); + } + buffer_st8(max_workspace_data, gy * p.h + gx, max_value); + return; + } +} diff --git a/src/layer/vulkan/shader/softmax_reduce_sum_pack8.comp b/src/layer/vulkan/shader/softmax_reduce_sum_pack8.comp new file mode 100644 index 000000000..ea70e42f3 --- /dev/null +++ b/src/layer/vulkan/shader/softmax_reduce_sum_pack8.comp @@ -0,0 +1,136 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int axis = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) readonly buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; +layout (binding = 1) writeonly buffer sum_workspace { sfpvec8 sum_workspace_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; + + int outdims; + int outw; + int outh; + int outc; + int outcstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.outw || gy >= p.outh || gz >= p.outc) + return; + + afpvec8 sum_value = afpvec8(afpvec4(0.f), afpvec4(0.f)); + + if (p.dims == 1) // axis == 0 + { + for (int i = 0; i < p.w; i++) + { + sum_value += buffer_ld8(bottom_top_blob_data, i); + } + afpvec4 sum4 = sum_value[0] + sum_value[1]; + afpvec2 sum2 = sum4.rg + sum4.ba; + afp sum1 = sum2.r + sum2.g; + sum_value = afpvec8(afpvec4(sum1), afpvec4(sum1)); + buffer_st8(sum_workspace_data, 0, sum_value); + return; + } + + if (p.dims == 2 && axis == 0) + { + for (int i = 0; i < p.h; i++) + { + int v_offset = i * p.w + gx; + sum_value += buffer_ld8(bottom_top_blob_data, v_offset); + } + afpvec4 sum4 = sum_value[0] + sum_value[1]; + afpvec2 sum2 = sum4.rg + sum4.ba; + afp sum1 = sum2.r + sum2.g; + sum_value = afpvec8(afpvec4(sum1), afpvec4(sum1)); + buffer_st8(sum_workspace_data, gx, sum_value); + return; + } + + if (p.dims == 2 && axis == 1) + { + for (int i = 0; i < p.w; i++) + { + int v_offset = gx * p.w + i; + sum_value += buffer_ld8(bottom_top_blob_data, v_offset); + } + buffer_st8(sum_workspace_data, gx, sum_value); + return; + } + + if (p.dims == 3 && axis == 0) + { + for (int i = 0; i < p.c; i++) + { + int v_offset = i * p.cstep + gy * p.w + gx; + sum_value += buffer_ld8(bottom_top_blob_data, v_offset); + } + afpvec4 sum4 = sum_value[0] + sum_value[1]; + afpvec2 sum2 = sum4.rg + sum4.ba; + afp sum1 = sum2.r + sum2.g; + sum_value = afpvec8(afpvec4(sum1), afpvec4(sum1)); + buffer_st8(sum_workspace_data, gy * p.w + gx, sum_value); + return; + } + + if (p.dims == 3 && axis == 1) + { + for (int i = 0; i < p.h; i++) + { + int v_offset = gy * p.cstep + i * p.w + gx; + sum_value += buffer_ld8(bottom_top_blob_data, v_offset); + } + buffer_st8(sum_workspace_data, gy * p.w + gx, sum_value); + return; + } + + if (p.dims == 3 && axis == 2) + { + for (int i = 0; i < p.w; i++) + { + int v_offset = gy * p.cstep + gx * p.w + i; + sum_value += buffer_ld8(bottom_top_blob_data, v_offset); + } + buffer_st8(sum_workspace_data, gy * p.h + gx, sum_value); + return; + } +} diff --git a/src/layer/vulkan/shader/tanh_pack8.comp b/src/layer/vulkan/shader/tanh_pack8.comp new file mode 100644 index 000000000..96eb050f1 --- /dev/null +++ b/src/layer/vulkan/shader/tanh_pack8.comp @@ -0,0 +1,57 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + v[0] = tanh(v[0]); + v[1] = tanh(v[1]); + + buffer_st8(bottom_top_blob_data, gi, v); +} diff --git a/src/layer/vulkan/shader/unaryop_pack8.comp b/src/layer/vulkan/shader/unaryop_pack8.comp new file mode 100644 index 000000000..bbc8f08ee --- /dev/null +++ b/src/layer/vulkan/shader/unaryop_pack8.comp @@ -0,0 +1,144 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#version 450 + +#if NCNN_fp16_storage +#extension GL_EXT_shader_16bit_storage: require +struct sfpvec8 { f16vec4 abcd; f16vec4 efgh; }; +#endif +#if NCNN_fp16_arithmetic +#extension GL_EXT_shader_explicit_arithmetic_types_float16: require +#endif + +layout (constant_id = 0) const int op_type = 0; + +layout (local_size_x_id = 233) in; +layout (local_size_y_id = 234) in; +layout (local_size_z_id = 235) in; + +layout (binding = 0) buffer bottom_top_blob { sfpvec8 bottom_top_blob_data[]; }; + +layout (push_constant) uniform parameter +{ + int dims; + int w; + int h; + int c; + int cstep; +} p; + +void main() +{ + int gx = int(gl_GlobalInvocationID.x); + int gy = int(gl_GlobalInvocationID.y); + int gz = int(gl_GlobalInvocationID.z); + + if (gx >= p.w || gy >= p.h || gz >= p.c) + return; + + const int gi = gz * p.cstep + gy * p.w + gx; + + afpvec8 v = buffer_ld8(bottom_top_blob_data, gi); + + afpvec8 res; + + if (op_type == 0) + { + res[0] = abs(v[0]); + res[1] = abs(v[1]); + } + if (op_type == 1) + { + res[0] = -v[0]; + res[1] = -v[1]; + } + if (op_type == 2) + { + res[0] = floor(v[0]); + res[1] = floor(v[1]); + } + if (op_type == 3) + { + res[0] = ceil(v[0]); + res[1] = ceil(v[1]); + } + if (op_type == 4) + { + res[0] = v[0] * v[0]; + res[1] = v[1] * v[1]; + } + if (op_type == 5) + { + res[0] = sqrt(v[0]); + res[1] = sqrt(v[1]); + } + if (op_type == 6) + { + res[0] = inversesqrt(v[0]); + res[1] = inversesqrt(v[1]); + } + if (op_type == 7) + { + res[0] = exp(v[0]); + res[1] = exp(v[1]); + } + if (op_type == 8) + { + res[0] = log(v[0]); + res[1] = log(v[1]); + } + if (op_type == 9) + { + res[0] = sin(v[0]); + res[1] = sin(v[1]); + } + if (op_type == 10) + { + res[0] = cos(v[0]); + res[1] = cos(v[1]); + } + if (op_type == 11) + { + res[0] = tan(v[0]); + res[1] = tan(v[1]); + } + if (op_type == 12) + { + res[0] = asin(v[0]); + res[1] = asin(v[1]); + } + if (op_type == 13) + { + res[0] = acos(v[0]); + res[1] = acos(v[1]); + } + if (op_type == 14) + { + res[0] = atan(v[0]); + res[1] = atan(v[1]); + } + if (op_type == 15) + { + res[0] = afp(1.f) / v[0]; + res[1] = afp(1.f) / v[1]; + } + if (op_type == 16) + { + res[0] = tanh(v[0]); + res[1] = tanh(v[1]); + } + + buffer_st8(bottom_top_blob_data, gi, res); +}