From 73aa99e83c818af75f7f00d84bd1ea2a5b5f5811 Mon Sep 17 00:00:00 2001 From: Tijmen Verhulsdonck Date: Wed, 1 Jul 2020 15:40:21 +0200 Subject: [PATCH] LSTM arm/x86 + fp16 innerproduct arm (#1881) * added fp16 weight storage version * Small changes * Fixed fp16 weight storage layers * fix innerproduct * fix loop error * Fix windows build. Disable fp 16 conversion when detecting int8 weights. Implement requested changes. * Restyled by clang-format * Restyled by astyle * Restyled by clang-format * Restyled by astyle * Update option.cpp Set fp16 storage based on vulkan being used or not. * added ability for storing state in lstm layer * added avx lstm * added arm lstm * fix innerproduct activation location and add 4 parallel channel version * Restyled by clang-format * Restyled by astyle * Restyled by clang-format * Restyled by astyle * revert arm file * commit before switch * implement requested changes * Restyled by clang-format * Restyled by astyle * Restyled by clang-format * Restyled by astyle * More x86 optimized implementations of common layers. Added LSTM layers for arm and x86 + a ctest to verify the layer accuracy Added fp16 innerproduct for arm * fix non avx build * Add fp16 arm compiler and cpu checks. Remove statefullness from LSTM implementation. * Fix build check for fp16 arm * Bypass lstm_fp16 if not supported * Build order was incorrect * fix std::min missing in windows build * Restyled by clang-format * Restyled by astyle * Restyled by clang-format * Restyled by astyle * attempting to fix gnu build by enabling: -mfp16-format=ieee to fix the missing __fp16 type * remove double "fix" * Specify ieee fp16 format * implement requested changes * fix arm non-fp16 build * fix arm lstm * Restyled/pull 1881 (#15) * Restyled by clang-format * Restyled by astyle * Restyled by clang-format * Restyled by astyle Co-authored-by: Restyled.io * Check blob size on arm lstm * fix styling Co-authored-by: Restyled.io --- src/layer/arm/innerproduct_arm.cpp | 256 ++++- src/layer/arm/innerproduct_arm.h | 8 + src/layer/arm/lstm_arm.cpp | 599 +++++++++++ src/layer/arm/lstm_arm.h | 38 + src/layer/arm/neon_activation.h | 20 +- src/layer/arm/neon_mathfun.h | 7 + src/layer/lstm.cpp | 95 +- src/layer/lstm.h | 2 + src/layer/x86/bias_x86.cpp | 69 ++ src/layer/x86/bias_x86.h | 30 + src/layer/x86/hardsigmoid_x86.cpp | 19 + src/layer/x86/innerproduct_x86.cpp | 1 - src/layer/x86/lstm_x86.cpp | 937 ++++++++++++++++++ src/layer/x86/lstm_x86.h | 40 + src/layer/x86/scale_x86.cpp | 261 +++++ src/layer/x86/scale_x86.h | 32 + src/layer/x86/sigmoid_x86.cpp | 4 +- src/layer/x86/slice_x86.cpp | 438 ++++++++ src/layer/x86/slice_x86.h | 38 + src/layer/x86/tanh_x86.cpp | 93 ++ src/layer/x86/tanh_x86.h | 32 + tests/CMakeLists.txt | 1 + tests/test_lstm.cpp | 217 ++++ tests/testutil.h | 26 +- .../arm-linux-gnueabihf.toolchain.cmake | 4 +- 25 files changed, 3205 insertions(+), 62 deletions(-) create mode 100644 src/layer/arm/lstm_arm.cpp create mode 100644 src/layer/arm/lstm_arm.h create mode 100644 src/layer/x86/bias_x86.cpp create mode 100644 src/layer/x86/bias_x86.h create mode 100644 src/layer/x86/lstm_x86.cpp create mode 100644 src/layer/x86/lstm_x86.h create mode 100644 src/layer/x86/scale_x86.cpp create mode 100644 src/layer/x86/scale_x86.h create mode 100644 src/layer/x86/slice_x86.cpp create mode 100644 src/layer/x86/slice_x86.h create mode 100644 src/layer/x86/tanh_x86.cpp create mode 100644 src/layer/x86/tanh_x86.h create mode 100644 tests/test_lstm.cpp diff --git a/src/layer/arm/innerproduct_arm.cpp b/src/layer/arm/innerproduct_arm.cpp index 63bab4415..4189b1309 100644 --- a/src/layer/arm/innerproduct_arm.cpp +++ b/src/layer/arm/innerproduct_arm.cpp @@ -20,7 +20,7 @@ #include #include "neon_mathfun.h" #endif // __ARM_NEON - +#include "cpu.h" #include "neon_activation.h" namespace ncnn { @@ -57,7 +57,12 @@ int InnerProduct_arm::create_pipeline(const Option& opt) { ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt); } - +#if __ARM_NEON && (__ARM_FP & 2) + else if (opt.use_fp16_storage && weight_data.elemsize == 4u && cpu_support_arm_vfpv4()) + { + ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); + } +#endif return 0; } @@ -72,6 +77,237 @@ int InnerProduct_arm::destroy_pipeline(const Option& opt) return 0; } +#if __ARM_NEON && (__ARM_FP & 2) +int InnerProduct_arm::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + int size = w * h; + + top_blob.create(num_output, elemsize, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + const unsigned short* weight_data_ptr = (const unsigned short*)weight_data_fp16; + + int nn_num_output = num_output >> 2; + int remain_num_output_start = nn_num_output << 2; + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < nn_num_output; pp++) + { + int p = pp * 4; + + float sum0 = 0.f; + float sum1 = 0.f; + float sum2 = 0.f; + float sum3 = 0.f; + + if (bias_term) + { + sum0 = bias_data[p]; + sum1 = bias_data[p + 1]; + sum2 = bias_data[p + 2]; + sum3 = bias_data[p + 3]; + } + + const unsigned short* w0 = (const unsigned short*)weight_data_ptr + size * channels * p; + const unsigned short* w1 = (const unsigned short*)weight_data_ptr + size * channels * (p + 1); + const unsigned short* w2 = (const unsigned short*)weight_data_ptr + size * channels * (p + 2); + const unsigned short* w3 = (const unsigned short*)weight_data_ptr + size * channels * (p + 3); + + float32x4_t _sum0 = vdupq_n_f32(0.f); + float32x4_t _sum1 = vdupq_n_f32(0.f); + float32x4_t _sum2 = vdupq_n_f32(0.f); + float32x4_t _sum3 = vdupq_n_f32(0.f); + + // channels + for (int q = 0; q < channels; q++) + { + const float* m = bottom_blob.channel(q); + + int nn = size >> 2; + int remain = size & 3; + + for (; nn > 0; nn--) + { + float32x4_t _m = vld1q_f32(m); + + float32x4_t _w0 = loadfp16(w0); + _sum0 = vmlaq_f32(_sum0, _m, _w0); + + float32x4_t _w1 = loadfp16(w1); + _sum1 = vmlaq_f32(_sum1, _m, _w1); + + float32x4_t _w2 = loadfp16(w2); + _sum2 = vmlaq_f32(_sum2, _m, _w2); + + float32x4_t _w3 = loadfp16(w3); + _sum3 = vmlaq_f32(_sum3, _m, _w3); + + m += 4; + w0 += 4; + w1 += 4; + w2 += 4; + w3 += 4; + } + if (remain != 0) + { + unsigned short fp16_weights[4][4] = {{0}}; + float _m_f[4] = {0}; + int i = 0; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (; remain > 0; remain--) + { + _m_f[i] = *m; + fp16_weights[0][i] = *w0; + fp16_weights[1][i] = *w1; + fp16_weights[2][i] = *w2; + fp16_weights[3][i] = *w3; + i++; + m++; + w0++; + w1++; + w2++; + w3++; + } + float32x4_t _m = vld1q_f32(_m_f); + + float32x4_t _w0 = loadfp16(fp16_weights[0]); + _sum0 = vmlaq_f32(_sum0, _m, _w0); + + float32x4_t _w1 = loadfp16(fp16_weights[1]); + _sum1 = vmlaq_f32(_sum1, _m, _w1); + + float32x4_t _w2 = loadfp16(fp16_weights[2]); + _sum2 = vmlaq_f32(_sum2, _m, _w2); + + float32x4_t _w3 = loadfp16(fp16_weights[3]); + _sum3 = vmlaq_f32(_sum3, _m, _w3); + } + } + + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); + float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1)); + float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2)); + float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3)); + + float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); + float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); + + sum0 += vget_lane_f32(_sum01ss, 0); + sum1 += vget_lane_f32(_sum01ss, 1); + sum2 += vget_lane_f32(_sum23ss, 0); + sum3 += vget_lane_f32(_sum23ss, 1); + + if (activation_type == 1) + { + sum0 = std::max(sum0, 0.f); + sum1 = std::max(sum1, 0.f); + sum2 = std::max(sum2, 0.f); + sum3 = std::max(sum3, 0.f); + } + else if (activation_type == 2) + { + float slope = activation_params[0]; + sum0 = sum0 > 0.f ? sum0 : sum0 * slope; + sum1 = sum1 > 0.f ? sum1 : sum1 * slope; + sum2 = sum2 > 0.f ? sum2 : sum2 * slope; + sum3 = sum3 > 0.f ? sum3 : sum3 * slope; + } + else if (activation_type == 3) + { + float min = activation_params[0]; + float max = activation_params[1]; + if (sum0 < min) sum0 = min; + if (sum0 > max) sum0 = max; + if (sum1 < min) sum1 = min; + if (sum1 > max) sum1 = max; + if (sum2 < min) sum2 = min; + if (sum2 > max) sum2 = max; + if (sum3 < min) sum3 = min; + if (sum3 > max) sum3 = max; + } + else if (activation_type == 4) + { + sum0 = static_cast(1.f / (1.f + exp(-sum0))); + sum1 = static_cast(1.f / (1.f + exp(-sum1))); + sum2 = static_cast(1.f / (1.f + exp(-sum2))); + sum3 = static_cast(1.f / (1.f + exp(-sum3))); + } + + top_blob[p] = sum0; + top_blob[p + 1] = sum1; + top_blob[p + 2] = sum2; + top_blob[p + 3] = sum3; + } + // num_output + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = remain_num_output_start; p < num_output; p++) + { + float sum0 = 0.f; + if (bias_term) + { + sum0 = bias_data[p]; + } + + const unsigned short* w0 = (const unsigned short*)weight_data_ptr + size * channels * p; + + float32x4_t _sum0 = vdupq_n_f32(0.f); + // channels + for (int q = 0; q < channels; q++) + { + const float* m = bottom_blob.channel(q); + + int nn = size >> 2; + int remain = size & 3; + + for (; nn > 0; nn--) + { + float32x4_t _m = vld1q_f32(m); + + float32x4_t _w0 = loadfp16(w0); + _sum0 = vmlaq_f32(_sum0, _m, _w0); + + m += 4; + w0 += 4; + } + if (remain != 0) + { + unsigned short fp16_weights[4] = {0}; + float _m_f[4] = {0}; + int i = 0; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (; remain > 0; remain--) + { + _m_f[i] = *m; + fp16_weights[i] = *w0; + i++; + m++; + w0++; + } + float32x4_t _m = vld1q_f32(_m_f); + + float32x4_t _w0 = loadfp16(fp16_weights); + _sum0 = vmlaq_f32(_sum0, _m, _w0); + } + } + + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0)); + + sum0 += vget_lane_f32(_sum0ss, 0) + vget_lane_f32(_sum0ss, 1); + + sum0 = activation_ss(sum0, activation_type, activation_params); + + top_blob[p] = sum0; + } + return 0; +} +#endif int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { @@ -111,11 +347,21 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio bottom_blob_flattened.elemsize = 4u; bottom_blob_flattened.elempack = 1; } - +#if (__ARM_FP & 2) + if (opt.use_fp16_storage && cpu_support_arm_vfpv4()) + { + return forward_fp16(bottom_blob_flattened, top_blob, opt); + } +#endif return forward(bottom_blob_flattened, top_blob, opt); } -#endif // __ARM_NEON - +#if (__ARM_FP & 2) + if (opt.use_fp16_storage && cpu_support_arm_vfpv4()) + { + return forward_fp16(bottom_blob, top_blob, opt); + } +#endif +#endif top_blob.create(num_output, elemsize, opt.blob_allocator); if (top_blob.empty()) return -100; diff --git a/src/layer/arm/innerproduct_arm.h b/src/layer/arm/innerproduct_arm.h index d55dbc48a..7875725f9 100644 --- a/src/layer/arm/innerproduct_arm.h +++ b/src/layer/arm/innerproduct_arm.h @@ -30,6 +30,11 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; protected: +#if __ARM_NEON + int forward_fp16(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + +#endif + int forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; public: @@ -37,6 +42,9 @@ public: // bf16 Mat weight_data_bf16; + + // fp16 + Mat weight_data_fp16; }; } // namespace ncnn diff --git a/src/layer/arm/lstm_arm.cpp b/src/layer/arm/lstm_arm.cpp new file mode 100644 index 000000000..0959b1091 --- /dev/null +++ b/src/layer/arm/lstm_arm.cpp @@ -0,0 +1,599 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "lstm_arm.h" + +#include + +#if __ARM_NEON +#include "neon_mathfun.h" +#include "neon_activation.h" +#endif // __ARM_NEON + +#include "cpu.h" + +namespace ncnn { + +DEFINE_LAYER_CREATOR(LSTM_arm) + +LSTM_arm::LSTM_arm() +{ + one_blob_only = false; + support_inplace = false; +} +int LSTM_arm::create_pipeline(const Option& opt) +{ +#if __ARM_NEON + if (opt.use_fp16_storage) + { + ncnn::cast_float32_to_float16(weight_xc_data, weight_xc_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_hc_data, weight_hc_data_fp16, opt); + } +#endif // __ARM_NEON + + return 0; +} + +#if __ARM_NEON +static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) +{ + int size = bottom_blob.w; + int T = bottom_blob.h; + + int num_output = top_blob.w; + + // 4 x num_output + Mat gates(num_output, 4, 4u, opt.workspace_allocator); + if (gates.empty()) + return -100; + + // unroll + for (int t = 0; t < T; t++) + { + // clip hidden by continuation indicator + // h_cont_{t-1} = cont_t * h_{t-1} + // h_cont_{t-1} = h_{t-1} if cont_t == 1 + // 0 otherwise + // calculate hidden + // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c + + int ti = reverse ? T - 1 - t : t; + for (int q = 0; q < num_output; q++) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const float* weight_xc_I = weight_xc.row(num_output * 0 + q); + const float* weight_xc_F = weight_xc.row(num_output * 1 + q); + const float* weight_xc_O = weight_xc.row(num_output * 2 + q); + const float* weight_xc_G = weight_xc.row(num_output * 3 + q); + + const float* weight_hc_I = weight_hc.row(num_output * 0 + q); + const float* weight_hc_F = weight_hc.row(num_output * 1 + q); + const float* weight_hc_O = weight_hc.row(num_output * 2 + q); + const float* weight_hc_G = weight_hc.row(num_output * 3 + q); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + float32x4_t _sumI = vdupq_n_f32(0.0f); + float32x4_t _sumF = vdupq_n_f32(0.0f); + float32x4_t _sumO = vdupq_n_f32(0.0f); + float32x4_t _sumG = vdupq_n_f32(0.0f); + int nn_num_size = size >> 2; + int remain_size = size & 3; + for (; nn_num_size > 0; nn_num_size--) + { + float32x4_t xi = vld1q_f32(x); + _sumI = vmlaq_f32(_sumI, vld1q_f32(weight_xc_I), xi); + _sumF = vmlaq_f32(_sumF, vld1q_f32(weight_xc_F), xi); + _sumO = vmlaq_f32(_sumO, vld1q_f32(weight_xc_O), xi); + _sumG = vmlaq_f32(_sumG, vld1q_f32(weight_xc_G), xi); + x += 4; + weight_xc_I += 4; + weight_xc_F += 4; + weight_xc_O += 4; + weight_xc_G += 4; + } + int nn_num_output = num_output >> 2; + int remain_num_output = num_output & 3; + for (; nn_num_output > 0; nn_num_output--) + { + float32x4_t h_cont = vld1q_f32(hidden_ptr_r); + + _sumI = vmlaq_f32(_sumI, vld1q_f32(weight_hc_I), h_cont); + _sumF = vmlaq_f32(_sumF, vld1q_f32(weight_hc_F), h_cont); + _sumO = vmlaq_f32(_sumO, vld1q_f32(weight_hc_O), h_cont); + _sumG = vmlaq_f32(_sumG, vld1q_f32(weight_hc_G), h_cont); + hidden_ptr_r += 4; + weight_hc_I += 4; + weight_hc_F += 4; + weight_hc_O += 4; + weight_hc_G += 4; + } + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sumI), vget_high_f32(_sumI)); + float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sumF), vget_high_f32(_sumF)); + float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sumO), vget_high_f32(_sumO)); + float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sumG), vget_high_f32(_sumG)); + + float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); + float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); + + float sums0 = vget_lane_f32(_sum01ss, 0) + bias_c_I[q]; + float sums1 = vget_lane_f32(_sum01ss, 1) + bias_c_F[q]; + float sums2 = vget_lane_f32(_sum23ss, 0) + bias_c_O[q]; + float sums3 = vget_lane_f32(_sum23ss, 1) + bias_c_G[q]; + + for (; remain_size > 0; remain_size--) + { + float xi = *x; + sums0 += *weight_xc_I * xi; + sums1 += *weight_xc_F * xi; + sums2 += *weight_xc_O * xi; + sums3 += *weight_xc_G * xi; + x++; + weight_xc_I++; + weight_xc_F++; + weight_xc_O++; + weight_xc_G++; + } + + for (; remain_num_output > 0; remain_num_output--) + { + float h_cont = *hidden_ptr_r; + sums0 += *weight_hc_I * h_cont; + sums1 += *weight_hc_F * h_cont; + sums2 += *weight_hc_O * h_cont; + sums3 += *weight_hc_G * h_cont; + hidden_ptr_r++; + weight_hc_I++; + weight_hc_F++; + weight_hc_O++; + weight_hc_G++; + } + gates_data_I[q] = sums0; + gates_data_F[q] = sums1; + gates_data_O[q] = sums2; + gates_data_G[q] = sums3; + } + + // lstm unit + // sigmoid(I) + // sigmoid(F) + // sigmoid(O) + // tanh(G) + // c_t := f_t .* c_{t-1} + i_t .* g_t + // h_t := o_t .* tanh[c_t] + float* output_data = top_blob.row(ti); + float* cell_ptr = cell_state; + float* hidden_ptr = hidden_state; + const float* gates_data_I = gates.row(0); + const float* gates_data_F = gates.row(1); + const float* gates_data_O = gates.row(2); + const float* gates_data_G = gates.row(3); + int nn_activation = num_output >> 2; + int remain_activations = num_output & 3; + for (; nn_activation > 0; nn_activation--) + { + float32x4_t I = sigmoid_ps(vld1q_f32(gates_data_I)); + float32x4_t F = sigmoid_ps(vld1q_f32(gates_data_F)); + float32x4_t O = sigmoid_ps(vld1q_f32(gates_data_O)); + float32x4_t G = tanh_ps(vld1q_f32(gates_data_G)); + float32x4_t cell2 = vaddq_f32(vmulq_f32(F, vld1q_f32(cell_ptr)), vmulq_f32(I, G)); + float32x4_t H = vmulq_f32(O, tanh_ps(cell2)); + vst1q_f32(cell_ptr, cell2); + vst1q_f32(hidden_ptr, H); + vst1q_f32(output_data, H); + cell_ptr += 4; + output_data += 4; + hidden_ptr += 4; + gates_data_I += 4; + gates_data_F += 4; + gates_data_O += 4; + gates_data_G += 4; + } + for (; remain_activations > 0; remain_activations--) + { + float I = *gates_data_I; + float F = *gates_data_F; + float O = *gates_data_O; + float G = *gates_data_G; + + I = 1.f / (1.f + exp(-I)); + F = 1.f / (1.f + exp(-F)); + O = 1.f / (1.f + exp(-O)); + G = tanh(G); + float cell2 = F * *cell_ptr + I * G; + float H = O * tanh(cell2); + *cell_ptr = cell2; + *hidden_ptr = H; + *output_data = H; + cell_ptr++; + output_data++; + hidden_ptr++; + gates_data_I++; + gates_data_F++; + gates_data_O++; + gates_data_G++; + } + + // no cell output here + } + + return 0; +} +#if (__ARM_FP & 2) +static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) +{ + int size = bottom_blob.w; + int T = bottom_blob.h; + + int num_output = top_blob.w; + + // 4 x num_output + Mat gates(num_output, 4, 4u, opt.workspace_allocator); + if (gates.empty()) + return -100; + + // unroll + for (int t = 0; t < T; t++) + { + // clip hidden by continuation indicator + // h_cont_{t-1} = cont_t * h_{t-1} + // h_cont_{t-1} = h_{t-1} if cont_t == 1 + // 0 otherwise + // calculate hidden + // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c + + int ti = reverse ? T - 1 - t : t; + for (int q = 0; q < num_output; q++) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const unsigned short* weight_xc_I = (const unsigned short*)weight_xc.row(num_output * 0 + q); + const unsigned short* weight_xc_F = (const unsigned short*)weight_xc.row(num_output * 1 + q); + const unsigned short* weight_xc_O = (const unsigned short*)weight_xc.row(num_output * 2 + q); + const unsigned short* weight_xc_G = (const unsigned short*)weight_xc.row(num_output * 3 + q); + + const unsigned short* weight_hc_I = (const unsigned short*)weight_hc.row(num_output * 0 + q); + const unsigned short* weight_hc_F = (const unsigned short*)weight_hc.row(num_output * 1 + q); + const unsigned short* weight_hc_O = (const unsigned short*)weight_hc.row(num_output * 2 + q); + const unsigned short* weight_hc_G = (const unsigned short*)weight_hc.row(num_output * 3 + q); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + float32x4_t _sumI = vdupq_n_f32(0.0f); + float32x4_t _sumF = vdupq_n_f32(0.0f); + float32x4_t _sumO = vdupq_n_f32(0.0f); + float32x4_t _sumG = vdupq_n_f32(0.0f); + int nn_num_size = size >> 2; + int remain_size = size & 3; + for (; nn_num_size > 0; nn_num_size--) + { + float32x4_t xi = vld1q_f32(x); + _sumI = vmlaq_f32(_sumI, loadfp16(weight_xc_I), xi); + _sumF = vmlaq_f32(_sumF, loadfp16(weight_xc_F), xi); + _sumO = vmlaq_f32(_sumO, loadfp16(weight_xc_O), xi); + _sumG = vmlaq_f32(_sumG, loadfp16(weight_xc_G), xi); + x += 4; + weight_xc_I += 4; + weight_xc_F += 4; + weight_xc_O += 4; + weight_xc_G += 4; + } + int nn_num_output = num_output >> 2; + int remain_num_output = num_output & 3; + for (; nn_num_output > 0; nn_num_output--) + { + float32x4_t h_cont = vld1q_f32(hidden_ptr_r); + + _sumI = vmlaq_f32(_sumI, loadfp16(weight_hc_I), h_cont); + _sumF = vmlaq_f32(_sumF, loadfp16(weight_hc_F), h_cont); + _sumO = vmlaq_f32(_sumO, loadfp16(weight_hc_O), h_cont); + _sumG = vmlaq_f32(_sumG, loadfp16(weight_hc_G), h_cont); + hidden_ptr_r += 4; + weight_hc_I += 4; + weight_hc_F += 4; + weight_hc_O += 4; + weight_hc_G += 4; + } + if (remain_size) + { + unsigned short fp16_weights[4][4] = {{0}}; + float _xi_f[4] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_size; i++) + { + _xi_f[i] = *x; + fp16_weights[0][i] = *weight_xc_I; + fp16_weights[1][i] = *weight_xc_F; + fp16_weights[2][i] = *weight_xc_O; + fp16_weights[3][i] = *weight_xc_G; + x++; + weight_xc_I++; + weight_xc_F++; + weight_xc_O++; + weight_xc_G++; + } + float32x4_t xi = vld1q_f32(_xi_f); + _sumI = vmlaq_f32(_sumI, loadfp16(fp16_weights[0]), xi); + _sumF = vmlaq_f32(_sumF, loadfp16(fp16_weights[1]), xi); + _sumO = vmlaq_f32(_sumO, loadfp16(fp16_weights[2]), xi); + _sumG = vmlaq_f32(_sumG, loadfp16(fp16_weights[3]), xi); + } + if (remain_num_output) + { + unsigned short fp16_weights[4][4] = {{0}}; + float _hcont_f[4] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_num_output; i++) + { + _hcont_f[i] = *hidden_ptr_r; + fp16_weights[0][i] = *weight_hc_I; + fp16_weights[1][i] = *weight_hc_F; + fp16_weights[2][i] = *weight_hc_O; + fp16_weights[3][i] = *weight_hc_G; + hidden_ptr_r++; + weight_hc_I++; + weight_hc_F++; + weight_hc_O++; + weight_hc_G++; + } + float32x4_t h_cont = vld1q_f32(_hcont_f); + _sumI = vmlaq_f32(_sumI, loadfp16(fp16_weights[0]), h_cont); + _sumF = vmlaq_f32(_sumF, loadfp16(fp16_weights[1]), h_cont); + _sumO = vmlaq_f32(_sumO, loadfp16(fp16_weights[2]), h_cont); + _sumG = vmlaq_f32(_sumG, loadfp16(fp16_weights[3]), h_cont); + } + float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sumI), vget_high_f32(_sumI)); + float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sumF), vget_high_f32(_sumF)); + float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sumO), vget_high_f32(_sumO)); + float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sumG), vget_high_f32(_sumG)); + + float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss); + float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss); + + float sums0 = vget_lane_f32(_sum01ss, 0) + bias_c_I[q]; + float sums1 = vget_lane_f32(_sum01ss, 1) + bias_c_F[q]; + float sums2 = vget_lane_f32(_sum23ss, 0) + bias_c_O[q]; + float sums3 = vget_lane_f32(_sum23ss, 1) + bias_c_G[q]; + + gates_data_I[q] = sums0; + gates_data_F[q] = sums1; + gates_data_O[q] = sums2; + gates_data_G[q] = sums3; + } + + // lstm unit + // sigmoid(I) + // sigmoid(F) + // sigmoid(O) + // tanh(G) + // c_t := f_t .* c_{t-1} + i_t .* g_t + // h_t := o_t .* tanh[c_t] + float* output_data = top_blob.row(ti); + float* cell_ptr = cell_state; + float* hidden_ptr = hidden_state; + const float* gates_data_I = gates.row(0); + const float* gates_data_F = gates.row(1); + const float* gates_data_O = gates.row(2); + const float* gates_data_G = gates.row(3); + int nn_activation = num_output >> 2; + int remain_activations = num_output & 3; + for (; nn_activation > 0; nn_activation--) + { + float32x4_t I = sigmoid_ps(vld1q_f32(gates_data_I)); + float32x4_t F = sigmoid_ps(vld1q_f32(gates_data_F)); + float32x4_t O = sigmoid_ps(vld1q_f32(gates_data_O)); + float32x4_t G = tanh_ps(vld1q_f32(gates_data_G)); + float32x4_t cell2 = vaddq_f32(vmulq_f32(F, vld1q_f32(cell_ptr)), vmulq_f32(I, G)); + float32x4_t H = vmulq_f32(O, tanh_ps(cell2)); + vst1q_f32(cell_ptr, cell2); + vst1q_f32(hidden_ptr, H); + vst1q_f32(output_data, H); + cell_ptr += 4; + output_data += 4; + hidden_ptr += 4; + gates_data_I += 4; + gates_data_F += 4; + gates_data_O += 4; + gates_data_G += 4; + } + for (; remain_activations > 0; remain_activations--) + { + float I = *gates_data_I; + float F = *gates_data_F; + float O = *gates_data_O; + float G = *gates_data_G; + + I = 1.f / (1.f + exp(-I)); + F = 1.f / (1.f + exp(-F)); + O = 1.f / (1.f + exp(-O)); + G = tanh(G); + float cell2 = F * *cell_ptr + I * G; + float H = O * tanh(cell2); + *cell_ptr = cell2; + *hidden_ptr = H; + *output_data = H; + cell_ptr++; + output_data++; + hidden_ptr++; + gates_data_I++; + gates_data_F++; + gates_data_O++; + gates_data_G++; + } + + // no cell output here + } + + return 0; +} +#endif +#endif +int LSTM_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ +#if __ARM_NEON + int T = bottom_blob.h; + int num_directions = direction == 2 ? 2 : 1; + + // initial hidden state + Mat hidden(num_output, 4u, opt.workspace_allocator); + if (hidden.empty()) + return -100; + hidden.fill(0.f); + // internal cell state + Mat cell(num_output, 4u, opt.workspace_allocator); + if (cell.empty()) + return -100; + cell.fill(0.f); + + top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // Uni directional + if (direction == 0 || direction == 1) + { +#if (__ARM_FP & 2) + if (opt.use_fp16_storage && cpu_support_arm_vfpv4()) + { + // Uni directional + return lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt); + } +#endif + // Uni directional + return lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); + } + + if (direction == 2) + { + Mat top_blob_forward(num_output, T, 4u, opt.workspace_allocator); + if (top_blob_forward.empty()) + return -100; + + Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator); + if (top_blob_reverse.empty()) + return -100; +#if (__ARM_FP & 2) + if (opt.use_fp16_storage && cpu_support_arm_vfpv4()) + { + // Uni directional + int ret0 = lstm_fp16(bottom_blob, top_blob_forward, 0, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt); + if (ret0 != 0) + return ret0; + hidden.fill(0.0f); + cell.fill(0.0f); + // Uni directional + int ret1 = lstm_fp16(bottom_blob, top_blob_reverse, 1, weight_xc_data_fp16.channel(1), bias_c_data.channel(1), weight_hc_data_fp16.channel(1), hidden, cell, opt); + if (ret1 != 0) + return ret1; + } + else + { +#endif + // Uni directional + int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); + if (ret0 != 0) + return ret0; + + hidden.fill(0.0f); + cell.fill(0.0f); + + // Uni directional + int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt); + if (ret1 != 0) + return ret1; +#if (__ARM_FP & 2) + } +#endif + + // concat w + for (int i = 0; i < T; i++) + { + const float* pf = top_blob_forward.row(i); + const float* pr = top_blob_reverse.row(i); + float* ptr = top_blob.row(i); + + memcpy(ptr, pf, num_output * sizeof(float)); + memcpy(ptr + num_output, pr, num_output * sizeof(float)); + } + } + + return 0; +#else + return LSTM::forward(bottom_blob, top_blob, opt); +#endif +} + +int LSTM_arm::forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const +{ +#if __ARM_NEON + if (bottom_blobs.size() != 3 || top_blobs.size() != 3) + { + return forward(bottom_blobs[0], top_blobs[0], opt); + } + const Mat& bottom_blob = bottom_blobs[0]; + + int T = bottom_blob.h; + Mat& top_blob = top_blobs[0]; + Mat& hidden_state = top_blobs[1]; + Mat& cell_state = top_blobs[2]; + + //Copy previous states + hidden_state = bottom_blobs[1].clone(opt.blob_allocator); + cell_state = bottom_blobs[2].clone(opt.blob_allocator); + + top_blob.create(num_output, T, 4u, opt.blob_allocator); + if (top_blob.empty()) + return -100; +#if (__ARM_FP & 2) + if (opt.use_fp16_storage && cpu_support_arm_vfpv4()) + { + // Uni directional + return lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden_state, cell_state, opt); + } +#endif + // Uni directional + return lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt); +#else + return LSTM::forward(bottom_blobs, top_blobs, opt); +#endif +} + +} // namespace ncnn diff --git a/src/layer/arm/lstm_arm.h b/src/layer/arm/lstm_arm.h new file mode 100644 index 000000000..1c5b934c4 --- /dev/null +++ b/src/layer/arm/lstm_arm.h @@ -0,0 +1,38 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_LSTM_ARM_H +#define LAYER_LSTM_ARM_H + +#include "lstm.h" + +namespace ncnn { + +class LSTM_arm : virtual public LSTM +{ +public: + LSTM_arm(); + + virtual int create_pipeline(const Option& opt); + virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + virtual int forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const; + +public: + Mat weight_hc_data_fp16; + Mat weight_xc_data_fp16; +}; + +} // namespace ncnn + +#endif // LAYER_LSTM_ARM_H diff --git a/src/layer/arm/neon_activation.h b/src/layer/arm/neon_activation.h index 19c2b9890..9879746aa 100644 --- a/src/layer/arm/neon_activation.h +++ b/src/layer/arm/neon_activation.h @@ -49,6 +49,17 @@ static inline float activation_ss(float v, int activation_type, const ncnn::Mat& } #if __ARM_NEON + +static inline float32x4_t sigmoid_ps(float32x4_t _v) +{ + float32x4_t _one = vdupq_n_f32(1.f); + _v = vnegq_f32(_v); + _v = exp_ps(_v); + _v = vaddq_f32(_v, _one); + float32x4_t _outp = vrecpeq_f32(_v); + return vmulq_f32(vrecpsq_f32(_v, _outp), _outp); +} + static inline float32x4_t activation_ps(float32x4_t _v, int activation_type, const ncnn::Mat& activation_params) { if (activation_type == 1) @@ -73,14 +84,7 @@ static inline float32x4_t activation_ps(float32x4_t _v, int activation_type, con } else if (activation_type == 4) { - float32x4_t _one = vdupq_n_f32(1.f); - _v = vnegq_f32(_v); - _v = exp_ps(_v); - _v = vaddq_f32(_v, _one); - float32x4_t _outp = vrecpeq_f32(_v); - _outp = vmulq_f32(vrecpsq_f32(_v, _outp), _outp); - // _outp = vmulq_f32(vrecpsq_f32(_v, _outp), _outp); - _v = _outp; + _v = sigmoid_ps(_v); } else if (activation_type == 5) { diff --git a/src/layer/arm/neon_mathfun.h b/src/layer/arm/neon_mathfun.h index 4abfa3268..83b301c22 100644 --- a/src/layer/arm/neon_mathfun.h +++ b/src/layer/arm/neon_mathfun.h @@ -27,6 +27,13 @@ #include +#if (__ARM_FP & 2) +static inline float32x4_t loadfp16(const void* ptr) +{ + return vcvt_f32_f16(vld1_f16((const __fp16*)ptr)); +} +#endif + #define c_inv_mant_mask ~0x7f800000u #define c_cephes_SQRTHF 0.707106781186547524 #define c_cephes_log_p0 7.0376836292E-2 diff --git a/src/layer/lstm.cpp b/src/layer/lstm.cpp index f2deb46d7..eb5d81845 100644 --- a/src/layer/lstm.cpp +++ b/src/layer/lstm.cpp @@ -22,7 +22,7 @@ DEFINE_LAYER_CREATOR(LSTM) LSTM::LSTM() { - one_blob_only = true; + one_blob_only = false; support_inplace = false; } @@ -31,7 +31,8 @@ int LSTM::load_param(const ParamDict& pd) num_output = pd.get(0, 0); weight_data_size = pd.get(1, 0); direction = pd.get(2, 0); - + if (direction == 2) + one_blob_only = true; return 0; } @@ -57,31 +58,18 @@ int LSTM::load_model(const ModelBin& mb) return 0; } -static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, const Option& opt) +static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) { int size = bottom_blob.w; int T = bottom_blob.h; int num_output = top_blob.w; - // initial hidden state - Mat hidden(num_output, 4u, opt.workspace_allocator); - if (hidden.empty()) - return -100; - - // internal cell state - Mat cell(num_output, 4u, opt.workspace_allocator); - if (cell.empty()) - return -100; - // 4 x num_output Mat gates(4, num_output, 4u, opt.workspace_allocator); if (gates.empty()) return -100; - hidden.fill(0.f); - cell.fill(0.f); - // unroll for (int t = 0; t < T; t++) { @@ -91,7 +79,6 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w // 0 otherwise // calculate hidden // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c - int cont = t > 0; int ti = reverse ? T - 1 - t : t; @@ -133,7 +120,7 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w for (int i = 0; i < num_output; i++) { - float h_cont = cont ? hidden[i] : 0.f; + float h_cont = hidden_state[i]; I += weight_hc_I[i] * h_cont; F += weight_hc_F[i] * h_cont; @@ -165,18 +152,16 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w float G = gates_data[3]; I = 1.f / (1.f + exp(-I)); - F = cont ? 1.f / (1.f + exp(-F)) : 0.f; + F = 1.f / (1.f + exp(-F)); O = 1.f / (1.f + exp(-O)); G = tanh(G); - float cell2 = F * cell[q] + I * G; + float cell2 = F * cell_state[q] + I * G; float H = O * tanh(cell2); - cell[q] = cell2; - hidden[q] = H; + cell_state[q] = cell2; + hidden_state[q] = H; output_data[q] = H; } - - // no cell output here } return 0; @@ -188,21 +173,25 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons int num_directions = direction == 2 ? 2 : 1; + // initial hidden state + Mat hidden(num_output, 4u, opt.workspace_allocator); + if (hidden.empty()) + return -100; + hidden.fill(0.f); + + Mat cell(num_output, 4u, opt.workspace_allocator); + if (cell.empty()) + return -100; + cell.fill(0.f); + top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator); if (top_blob.empty()) return -100; - // forward - if (direction == 0) - { - int ret = lstm(bottom_blob, top_blob, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt); - if (ret != 0) - return ret; - } - - if (direction == 1) + // Uni directional + if (direction == 0 || direction == 1) { - int ret = lstm(bottom_blob, top_blob, 1, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt); + int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); if (ret != 0) return ret; } @@ -217,11 +206,14 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons if (top_blob_reverse.empty()) return -100; - int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt); + int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); if (ret0 != 0) return ret0; - int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), opt); + hidden.fill(0.0f); + cell.fill(0.0f); + + int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt); if (ret1 != 0) return ret1; @@ -240,4 +232,35 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons return 0; } +int LSTM::forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const +{ + if (bottom_blobs.size() != 3 || top_blobs.size() != 3) + { + return forward(bottom_blobs[0], top_blobs[0], opt); + } + const Mat& bottom_blob = bottom_blobs[0]; + int T = bottom_blob.h; + Mat& top_blob = top_blobs[0]; + Mat& hidden_state = top_blobs[1]; + Mat& cell_state = top_blobs[2]; + + //Copy previous states + hidden_state = bottom_blobs[1].clone(opt.blob_allocator); + cell_state = bottom_blobs[2].clone(opt.blob_allocator); + + top_blob.create(num_output, T, 4u, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // Uni directional + if (direction == 0 || direction == 1) + { + int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt); + if (ret != 0) + return ret; + } + + return 0; +} + } // namespace ncnn diff --git a/src/layer/lstm.h b/src/layer/lstm.h index bbee45109..78d8366a0 100644 --- a/src/layer/lstm.h +++ b/src/layer/lstm.h @@ -30,6 +30,8 @@ public: virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + virtual int forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const; + public: int num_output; int weight_data_size; diff --git a/src/layer/x86/bias_x86.cpp b/src/layer/x86/bias_x86.cpp new file mode 100644 index 000000000..fa0686393 --- /dev/null +++ b/src/layer/x86/bias_x86.cpp @@ -0,0 +1,69 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#if __AVX__ +#include +#endif // __AVX__ + +#include "bias_x86.h" + +namespace ncnn { + +DEFINE_LAYER_CREATOR(Bias_x86) + +int Bias_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const +{ + int w = bottom_top_blob.w; + int h = bottom_top_blob.h; + int channels = bottom_top_blob.c; + int size = w * h; + + const float* bias_ptr = bias_data; + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + + float bias = bias_ptr[q]; + +#if __AVX__ + int nn = size >> 3; + int remain = size & 7; +#else + int remain = size; +#endif // __AVX__ + +#if __AVX__ + __m256 _bias = _mm256_set1_ps(bias); + for (; nn > 0; nn--) + { + __m256 _p = _mm256_loadu_ps(ptr); + __m256 _outp = _mm256_add_ps(_p, _bias); + _mm256_storeu_ps(ptr, _outp); + + ptr += 8; + } +#endif // __AVX__ + + for (; remain > 0; remain--) + { + *ptr = *ptr + bias; + + ptr++; + } + } + + return 0; +} + +} // namespace ncnn diff --git a/src/layer/x86/bias_x86.h b/src/layer/x86/bias_x86.h new file mode 100644 index 000000000..39d1bcef4 --- /dev/null +++ b/src/layer/x86/bias_x86.h @@ -0,0 +1,30 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_BIAS_X86_H +#define LAYER_BIAS_X86_H + +#include "bias.h" + +namespace ncnn { + +class Bias_x86 : virtual public Bias +{ +public: + virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const; +}; + +} // namespace ncnn + +#endif // LAYER_BIAS_X86_H diff --git a/src/layer/x86/hardsigmoid_x86.cpp b/src/layer/x86/hardsigmoid_x86.cpp index 9c13f18ea..81e5cc37c 100644 --- a/src/layer/x86/hardsigmoid_x86.cpp +++ b/src/layer/x86/hardsigmoid_x86.cpp @@ -68,7 +68,26 @@ int HardSigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) co for (int q = 0; q < channels; q++) { float* ptr = bottom_top_blob.channel(q); +#if __AVX__ + int nn_size = size >> 3; + int remain = size & 7; + + __m256 _zero = _mm256_set1_ps(0.f); + __m256 _one = _mm256_set1_ps(1.f); + for (; nn_size > 0; nn_size--) + { + __m256 _p = _mm256_loadu_ps(ptr); + __m256 _ans = _mm256_set1_ps(beta); + _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); + _ans = _mm256_max_ps(_ans, _zero); + _ans = _mm256_min_ps(_ans, _one); + _mm256_storeu_ps(ptr, _ans); + + ptr += 8; + } +#else int remain = size; +#endif for (; remain > 0; remain--) { if (*ptr < lower) diff --git a/src/layer/x86/innerproduct_x86.cpp b/src/layer/x86/innerproduct_x86.cpp index 81c946fc7..434c37e2a 100644 --- a/src/layer/x86/innerproduct_x86.cpp +++ b/src/layer/x86/innerproduct_x86.cpp @@ -87,7 +87,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, size_t elemsize = bottom_blob.elemsize; int elempack = bottom_blob.elempack; int size = w * h; - // fprintf(stderr, "bottom_blob %d x %d x %d, elempack = %d \n", w,h,channels,elempack); #if __AVX__ if (elempack == 8) { diff --git a/src/layer/x86/lstm_x86.cpp b/src/layer/x86/lstm_x86.cpp new file mode 100644 index 000000000..6bde5e5d6 --- /dev/null +++ b/src/layer/x86/lstm_x86.cpp @@ -0,0 +1,937 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#ifdef __AVX__ +#include "avx_activation.h" +#include "avx_usability.h" +#endif // NCNN_AVX2 + +#include "lstm_x86.h" + +#include +#include "layer_type.h" + +namespace ncnn { + +DEFINE_LAYER_CREATOR(LSTM_x86) + +LSTM_x86::LSTM_x86() +{ + one_blob_only = false; + support_inplace = false; +} +int LSTM_x86::create_pipeline(const Option& opt) +{ +#if __AVX__ + if (opt.use_fp16_storage) + { + ncnn::cast_float32_to_float16(weight_xc_data, weight_xc_data_fp16, opt); + ncnn::cast_float32_to_float16(weight_hc_data, weight_hc_data_fp16, opt); + } +#endif // __AVX__ + + return 0; +} +#ifdef __AVX__ + +static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) +{ + int size = bottom_blob.w; + int T = bottom_blob.h; + + int num_output = top_blob.w; + // fprintf(stderr, "bottom_blob = %d x %d x %d num_output = %d \n", bottom_blob.w,bottom_blob.h,bottom_blob.c,num_output); + // 4 x num_output + Mat gates(num_output, 4, 4u, opt.workspace_allocator); + if (gates.empty()) + return -100; + // unroll + for (int t = 0; t < T; t++) + { + // clip hidden by continuation indicator + // h_cont_{t-1} = cont_t * h_{t-1} + // h_cont_{t-1} = h_{t-1} if cont_t == 1 + // 0 otherwise + // calculate hidden + // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c + int ti = reverse ? T - 1 - t : t; + int remain_output = (num_output >> 1) << 1; + for (int q = 0; q + 1 < num_output; q += 2) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const unsigned short* weight_xc_I_0 = (const unsigned short*)weight_xc.row(num_output * 0 + q); + const unsigned short* weight_xc_F_0 = (const unsigned short*)weight_xc.row(num_output * 1 + q); + const unsigned short* weight_xc_O_0 = (const unsigned short*)weight_xc.row(num_output * 2 + q); + const unsigned short* weight_xc_G_0 = (const unsigned short*)weight_xc.row(num_output * 3 + q); + const unsigned short* weight_xc_I_1 = (const unsigned short*)weight_xc.row(num_output * 0 + (q + 1)); + const unsigned short* weight_xc_F_1 = (const unsigned short*)weight_xc.row(num_output * 1 + (q + 1)); + const unsigned short* weight_xc_O_1 = (const unsigned short*)weight_xc.row(num_output * 2 + (q + 1)); + const unsigned short* weight_xc_G_1 = (const unsigned short*)weight_xc.row(num_output * 3 + (q + 1)); + + const unsigned short* weight_hc_I_0 = (const unsigned short*)weight_hc.row(num_output * 0 + q); + const unsigned short* weight_hc_F_0 = (const unsigned short*)weight_hc.row(num_output * 1 + q); + const unsigned short* weight_hc_O_0 = (const unsigned short*)weight_hc.row(num_output * 2 + q); + const unsigned short* weight_hc_G_0 = (const unsigned short*)weight_hc.row(num_output * 3 + q); + const unsigned short* weight_hc_I_1 = (const unsigned short*)weight_hc.row(num_output * 0 + (q + 1)); + const unsigned short* weight_hc_F_1 = (const unsigned short*)weight_hc.row(num_output * 1 + (q + 1)); + const unsigned short* weight_hc_O_1 = (const unsigned short*)weight_hc.row(num_output * 2 + (q + 1)); + const unsigned short* weight_hc_G_1 = (const unsigned short*)weight_hc.row(num_output * 3 + (q + 1)); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + __m256 _sumI_0 = _mm256_setzero_ps(); + __m256 _sumF_0 = _mm256_setzero_ps(); + __m256 _sumO_0 = _mm256_setzero_ps(); + __m256 _sumG_0 = _mm256_setzero_ps(); + __m256 _sumI_1 = _mm256_setzero_ps(); + __m256 _sumF_1 = _mm256_setzero_ps(); + __m256 _sumO_1 = _mm256_setzero_ps(); + __m256 _sumG_1 = _mm256_setzero_ps(); + int nn_num_size = size >> 3; + int remain_size = size & 7; + for (; nn_num_size > 0; nn_num_size--) + { + __m256 xi = _mm256_loadu_ps(x); + _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_xc_I_0), xi, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_xc_F_0), xi, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_xc_O_0), xi, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_xc_G_0), xi, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_xc_I_1), xi, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_xc_F_1), xi, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_xc_O_1), xi, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_xc_G_1), xi, _sumG_1); + x += 8; + weight_xc_I_0 += 8; + weight_xc_F_0 += 8; + weight_xc_O_0 += 8; + weight_xc_G_0 += 8; + weight_xc_I_1 += 8; + weight_xc_F_1 += 8; + weight_xc_O_1 += 8; + weight_xc_G_1 += 8; + } + int nn_num_output = num_output >> 3; + int remain_num_output = num_output & 7; + for (; nn_num_output > 0; nn_num_output--) + { + __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); + + _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_hc_I_0), h_cont, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_hc_F_0), h_cont, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_hc_O_0), h_cont, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_hc_G_0), h_cont, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_hc_I_1), h_cont, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_hc_F_1), h_cont, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_hc_O_1), h_cont, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_hc_G_1), h_cont, _sumG_1); + hidden_ptr_r += 8; + weight_hc_I_0 += 8; + weight_hc_F_0 += 8; + weight_hc_O_0 += 8; + weight_hc_G_0 += 8; + weight_hc_I_1 += 8; + weight_hc_F_1 += 8; + weight_hc_O_1 += 8; + weight_hc_G_1 += 8; + } + if (remain_size != 0) + { + unsigned short fp16_weights[8][8] = {{0}}; + float _xi_f[8] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_size; i++) + { + _xi_f[i] = *x; + fp16_weights[0][i] = *weight_xc_I_0; + fp16_weights[1][i] = *weight_xc_F_0; + fp16_weights[2][i] = *weight_xc_O_0; + fp16_weights[3][i] = *weight_xc_G_0; + fp16_weights[4][i] = *weight_xc_I_1; + fp16_weights[5][i] = *weight_xc_F_1; + fp16_weights[6][i] = *weight_xc_O_1; + fp16_weights[7][i] = *weight_xc_G_1; + x++; + weight_xc_I_0++; + weight_xc_F_0++; + weight_xc_O_0++; + weight_xc_G_0++; + weight_xc_I_1++; + weight_xc_F_1++; + weight_xc_O_1++; + weight_xc_G_1++; + } + __m256 xi = _mm256_loadu_ps(_xi_f); + _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), xi, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), xi, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), xi, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), xi, _sumG_1); + } + if (remain_num_output != 0) + { + unsigned short fp16_weights[8][8] = {{0}}; + float _hcont_f[8] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_num_output; i++) + { + _hcont_f[i] = *hidden_ptr_r; + fp16_weights[0][i] = *weight_hc_I_0; + fp16_weights[1][i] = *weight_hc_F_0; + fp16_weights[2][i] = *weight_hc_O_0; + fp16_weights[3][i] = *weight_hc_G_0; + fp16_weights[4][i] = *weight_hc_I_1; + fp16_weights[5][i] = *weight_hc_F_1; + fp16_weights[6][i] = *weight_hc_O_1; + fp16_weights[7][i] = *weight_hc_G_1; + hidden_ptr_r++; + weight_hc_I_0++; + weight_hc_F_0++; + weight_hc_O_0++; + weight_hc_G_0++; + weight_hc_I_1++; + weight_hc_F_1++; + weight_hc_O_1++; + weight_hc_G_1++; + } + __m256 h_cont = _mm256_loadu_ps(_hcont_f); + _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), h_cont, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), h_cont, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), h_cont, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), h_cont, _sumG_1); + } + float sums[8]; + _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1)); + sums[0] += bias_c_I[q]; + sums[1] += bias_c_F[q]; + sums[2] += bias_c_O[q]; + sums[3] += bias_c_G[q]; + sums[4] += bias_c_I[q + 1]; + sums[5] += bias_c_F[q + 1]; + sums[6] += bias_c_O[q + 1]; + sums[7] += bias_c_G[q + 1]; + gates_data_I[q] = sums[0]; + gates_data_F[q] = sums[1]; + gates_data_O[q] = sums[2]; + gates_data_G[q] = sums[3]; + gates_data_I[q + 1] = sums[4]; + gates_data_F[q + 1] = sums[5]; + gates_data_O[q + 1] = sums[6]; + gates_data_G[q + 1] = sums[7]; + } + + for (int q = remain_output; q < num_output; q++) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const unsigned short* weight_xc_I = (const unsigned short*)weight_xc.row(num_output * 0 + q); + const unsigned short* weight_xc_F = (const unsigned short*)weight_xc.row(num_output * 1 + q); + const unsigned short* weight_xc_O = (const unsigned short*)weight_xc.row(num_output * 2 + q); + const unsigned short* weight_xc_G = (const unsigned short*)weight_xc.row(num_output * 3 + q); + + const unsigned short* weight_hc_I = (const unsigned short*)weight_hc.row(num_output * 0 + q); + const unsigned short* weight_hc_F = (const unsigned short*)weight_hc.row(num_output * 1 + q); + const unsigned short* weight_hc_O = (const unsigned short*)weight_hc.row(num_output * 2 + q); + const unsigned short* weight_hc_G = (const unsigned short*)weight_hc.row(num_output * 3 + q); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + __m256 _sumI = _mm256_setzero_ps(); + __m256 _sumF = _mm256_setzero_ps(); + __m256 _sumO = _mm256_setzero_ps(); + __m256 _sumG = _mm256_setzero_ps(); + int nn_num_size = size >> 3; + int remain_size = size & 7; + for (; nn_num_size > 0; nn_num_size--) + { + __m256 xi = _mm256_loadu_ps(x); + _sumI = _mm256_fmadd_ps(loadfp16(weight_xc_I), xi, _sumI); + _sumF = _mm256_fmadd_ps(loadfp16(weight_xc_F), xi, _sumF); + _sumO = _mm256_fmadd_ps(loadfp16(weight_xc_O), xi, _sumO); + _sumG = _mm256_fmadd_ps(loadfp16(weight_xc_G), xi, _sumG); + x += 8; + weight_xc_I += 8; + weight_xc_F += 8; + weight_xc_O += 8; + weight_xc_G += 8; + } + int nn_num_output = num_output >> 3; + int remain_num_output = num_output & 7; + for (; nn_num_output > 0; nn_num_output--) + { + __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); + + _sumI = _mm256_fmadd_ps(loadfp16(weight_hc_I), h_cont, _sumI); + _sumF = _mm256_fmadd_ps(loadfp16(weight_hc_F), h_cont, _sumF); + _sumO = _mm256_fmadd_ps(loadfp16(weight_hc_O), h_cont, _sumO); + _sumG = _mm256_fmadd_ps(loadfp16(weight_hc_G), h_cont, _sumG); + hidden_ptr_r += 8; + weight_hc_I += 8; + weight_hc_F += 8; + weight_hc_O += 8; + weight_hc_G += 8; + } + if (remain_size != 0) + { + unsigned short fp16_weights[4][8] = {{0}}; + float _xi_f[8] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_size; i++) + { + _xi_f[i] = *x; + fp16_weights[0][i] = *weight_xc_I; + fp16_weights[1][i] = *weight_xc_F; + fp16_weights[2][i] = *weight_xc_O; + fp16_weights[3][i] = *weight_xc_G; + x++; + weight_xc_I++; + weight_xc_F++; + weight_xc_O++; + weight_xc_G++; + } + __m256 xi = _mm256_loadu_ps(_xi_f); + _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI); + _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF); + _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO); + _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG); + } + if (remain_num_output != 0) + { + unsigned short fp16_weights[4][8] = {{0}}; + float _hcont_f[8] = {0}; + // No fast way to convert to fp32 one element at the time + // so batch an 8 lane vector. + for (int i = 0; i < remain_num_output; i++) + { + _hcont_f[i] = *hidden_ptr_r; + fp16_weights[0][i] = *weight_hc_I; + fp16_weights[1][i] = *weight_hc_F; + fp16_weights[2][i] = *weight_hc_O; + fp16_weights[3][i] = *weight_hc_G; + hidden_ptr_r++; + weight_hc_I++; + weight_hc_F++; + weight_hc_O++; + weight_hc_G++; + } + __m256 h_cont = _mm256_loadu_ps(_hcont_f); + _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI); + _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF); + _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO); + _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG); + } + + float sums[4]; + _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG)); + sums[0] += bias_c_I[q]; + sums[1] += bias_c_F[q]; + sums[2] += bias_c_O[q]; + sums[3] += bias_c_G[q]; + gates_data_I[q] = sums[0]; + gates_data_F[q] = sums[1]; + gates_data_O[q] = sums[2]; + gates_data_G[q] = sums[3]; + } + + // lstm unit + // sigmoid(I) + // sigmoid(F) + // sigmoid(O) + // tanh(G) + // c_t := f_t .* c_{t-1} + i_t .* g_t + // h_t := o_t .* tanh[c_t] + float* output_data = top_blob.row(ti); + float* cell_ptr = cell_state; + float* hidden_ptr = hidden_state; + const float* gates_data_I = gates.row(0); + const float* gates_data_F = gates.row(1); + const float* gates_data_O = gates.row(2); + const float* gates_data_G = gates.row(3); + int nn_activation = num_output >> 3; + int remain_activations = num_output & 7; + for (; nn_activation > 0; nn_activation--) + { + __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I)); + __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F)); + __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O)); + __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G)); + __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G)); + __m256 H = _mm256_mul_ps(O, tanh_avx(cell2)); + _mm256_storeu_ps(cell_ptr, cell2); + _mm256_storeu_ps(hidden_ptr, H); + _mm256_storeu_ps(output_data, H); + cell_ptr += 8; + output_data += 8; + hidden_ptr += 8; + gates_data_I += 8; + gates_data_F += 8; + gates_data_O += 8; + gates_data_G += 8; + } + for (; remain_activations > 0; remain_activations--) + { + float I = *gates_data_I; + float F = *gates_data_F; + float O = *gates_data_O; + float G = *gates_data_G; + + I = 1.f / (1.f + exp(-I)); + F = 1.f / (1.f + exp(-F)); + O = 1.f / (1.f + exp(-O)); + G = tanh(G); + float cell2 = F * *cell_ptr + I * G; + float H = O * tanh(cell2); + *cell_ptr = cell2; + *hidden_ptr = H; + *output_data = H; + cell_ptr++; + output_data++; + hidden_ptr++; + gates_data_I++; + gates_data_F++; + gates_data_O++; + gates_data_G++; + } + + // no cell output here + } + + return 0; +} + +static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) +{ + int size = bottom_blob.w; + int T = bottom_blob.h; + + int num_output = top_blob.w; + + // 4 x num_output + Mat gates(num_output, 4, 4u, opt.workspace_allocator); + if (gates.empty()) + return -100; + + // unroll + for (int t = 0; t < T; t++) + { + // clip hidden by continuation indicator + // h_cont_{t-1} = cont_t * h_{t-1} + // h_cont_{t-1} = h_{t-1} if cont_t == 1 + // 0 otherwise + // calculate hidden + // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c + + int ti = reverse ? T - 1 - t : t; + int remain_output = (num_output >> 1) << 1; + for (int q = 0; q + 1 < num_output; q += 2) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const float* weight_xc_I_0 = weight_xc.row(num_output * 0 + q); + const float* weight_xc_F_0 = weight_xc.row(num_output * 1 + q); + const float* weight_xc_O_0 = weight_xc.row(num_output * 2 + q); + const float* weight_xc_G_0 = weight_xc.row(num_output * 3 + q); + const float* weight_xc_I_1 = weight_xc.row(num_output * 0 + (q + 1)); + const float* weight_xc_F_1 = weight_xc.row(num_output * 1 + (q + 1)); + const float* weight_xc_O_1 = weight_xc.row(num_output * 2 + (q + 1)); + const float* weight_xc_G_1 = weight_xc.row(num_output * 3 + (q + 1)); + + const float* weight_hc_I_0 = weight_hc.row(num_output * 0 + q); + const float* weight_hc_F_0 = weight_hc.row(num_output * 1 + q); + const float* weight_hc_O_0 = weight_hc.row(num_output * 2 + q); + const float* weight_hc_G_0 = weight_hc.row(num_output * 3 + q); + const float* weight_hc_I_1 = weight_hc.row(num_output * 0 + (q + 1)); + const float* weight_hc_F_1 = weight_hc.row(num_output * 1 + (q + 1)); + const float* weight_hc_O_1 = weight_hc.row(num_output * 2 + (q + 1)); + const float* weight_hc_G_1 = weight_hc.row(num_output * 3 + (q + 1)); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + __m256 _sumI_0 = _mm256_setzero_ps(); + __m256 _sumF_0 = _mm256_setzero_ps(); + __m256 _sumO_0 = _mm256_setzero_ps(); + __m256 _sumG_0 = _mm256_setzero_ps(); + __m256 _sumI_1 = _mm256_setzero_ps(); + __m256 _sumF_1 = _mm256_setzero_ps(); + __m256 _sumO_1 = _mm256_setzero_ps(); + __m256 _sumG_1 = _mm256_setzero_ps(); + int nn_num_size = size >> 3; + int remain_size = size & 7; + for (; nn_num_size > 0; nn_num_size--) + { + __m256 xi = _mm256_loadu_ps(x); + _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_0), xi, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_0), xi, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_0), xi, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_0), xi, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_1), xi, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_1), xi, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_1), xi, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_1), xi, _sumG_1); + x += 8; + weight_xc_I_0 += 8; + weight_xc_F_0 += 8; + weight_xc_O_0 += 8; + weight_xc_G_0 += 8; + weight_xc_I_1 += 8; + weight_xc_F_1 += 8; + weight_xc_O_1 += 8; + weight_xc_G_1 += 8; + } + int nn_num_output = num_output >> 3; + int remain_num_output = num_output & 7; + for (; nn_num_output > 0; nn_num_output--) + { + __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); + + _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_0), h_cont, _sumI_0); + _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_0), h_cont, _sumF_0); + _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_0), h_cont, _sumO_0); + _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_0), h_cont, _sumG_0); + _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_1), h_cont, _sumI_1); + _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_1), h_cont, _sumF_1); + _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_1), h_cont, _sumO_1); + _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_1), h_cont, _sumG_1); + hidden_ptr_r += 8; + weight_hc_I_0 += 8; + weight_hc_F_0 += 8; + weight_hc_O_0 += 8; + weight_hc_G_0 += 8; + weight_hc_I_1 += 8; + weight_hc_F_1 += 8; + weight_hc_O_1 += 8; + weight_hc_G_1 += 8; + } + float sums[8]; + _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1)); + sums[0] += bias_c_I[q]; + sums[1] += bias_c_F[q]; + sums[2] += bias_c_O[q]; + sums[3] += bias_c_G[q]; + sums[4] += bias_c_I[q + 1]; + sums[5] += bias_c_F[q + 1]; + sums[6] += bias_c_O[q + 1]; + sums[7] += bias_c_G[q + 1]; + + for (; remain_size > 0; remain_size--) + { + float xi = *x; + sums[0] += *weight_xc_I_0 * xi; + sums[1] += *weight_xc_F_0 * xi; + sums[2] += *weight_xc_O_0 * xi; + sums[3] += *weight_xc_G_0 * xi; + sums[4] += *weight_xc_I_1 * xi; + sums[5] += *weight_xc_F_1 * xi; + sums[6] += *weight_xc_O_1 * xi; + sums[7] += *weight_xc_G_1 * xi; + x++; + weight_xc_I_0++; + weight_xc_F_0++; + weight_xc_O_0++; + weight_xc_G_0++; + weight_xc_I_1++; + weight_xc_F_1++; + weight_xc_O_1++; + weight_xc_G_1++; + } + + for (; remain_num_output > 0; remain_num_output--) + { + float h_cont = *hidden_ptr_r; + sums[0] += *weight_hc_I_0 * h_cont; + sums[1] += *weight_hc_F_0 * h_cont; + sums[2] += *weight_hc_O_0 * h_cont; + sums[3] += *weight_hc_G_0 * h_cont; + sums[4] += *weight_hc_I_1 * h_cont; + sums[5] += *weight_hc_F_1 * h_cont; + sums[6] += *weight_hc_O_1 * h_cont; + sums[7] += *weight_hc_G_1 * h_cont; + hidden_ptr_r++; + weight_hc_I_0++; + weight_hc_F_0++; + weight_hc_O_0++; + weight_hc_G_0++; + weight_hc_I_1++; + weight_hc_F_1++; + weight_hc_O_1++; + weight_hc_G_1++; + } + gates_data_I[q] = sums[0]; + gates_data_F[q] = sums[1]; + gates_data_O[q] = sums[2]; + gates_data_G[q] = sums[3]; + gates_data_I[q + 1] = sums[4]; + gates_data_F[q + 1] = sums[5]; + gates_data_O[q + 1] = sums[6]; + gates_data_G[q + 1] = sums[7]; + } + + for (int q = remain_output; q < num_output; q++) + { + const float* x = bottom_blob.row(ti); + const float* hidden_ptr_r = hidden_state; + const float* bias_c_I = bias_c.row(0); + const float* bias_c_F = bias_c.row(1); + const float* bias_c_O = bias_c.row(2); + const float* bias_c_G = bias_c.row(3); + + float* gates_data_I = gates.row(0); + float* gates_data_F = gates.row(1); + float* gates_data_O = gates.row(2); + float* gates_data_G = gates.row(3); + // gate I F O G + const float* weight_xc_I = weight_xc.row(num_output * 0 + q); + const float* weight_xc_F = weight_xc.row(num_output * 1 + q); + const float* weight_xc_O = weight_xc.row(num_output * 2 + q); + const float* weight_xc_G = weight_xc.row(num_output * 3 + q); + + const float* weight_hc_I = weight_hc.row(num_output * 0 + q); + const float* weight_hc_F = weight_hc.row(num_output * 1 + q); + const float* weight_hc_O = weight_hc.row(num_output * 2 + q); + const float* weight_hc_G = weight_hc.row(num_output * 3 + q); + + // float I = bias_c_I[q]; + // float F = bias_c_F[q]; + // float O = bias_c_O[q]; + // float G = bias_c_G[q]; + __m256 _sumI = _mm256_setzero_ps(); + __m256 _sumF = _mm256_setzero_ps(); + __m256 _sumO = _mm256_setzero_ps(); + __m256 _sumG = _mm256_setzero_ps(); + int nn_num_size = size >> 3; + int remain_size = size & 7; + for (; nn_num_size > 0; nn_num_size--) + { + __m256 xi = _mm256_loadu_ps(x); + _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I), xi, _sumI); + _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F), xi, _sumF); + _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O), xi, _sumO); + _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G), xi, _sumG); + x += 8; + weight_xc_I += 8; + weight_xc_F += 8; + weight_xc_O += 8; + weight_xc_G += 8; + } + int nn_num_output = num_output >> 3; + int remain_num_output = num_output & 7; + for (; nn_num_output > 0; nn_num_output--) + { + __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); + + _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I), h_cont, _sumI); + _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F), h_cont, _sumF); + _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O), h_cont, _sumO); + _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G), h_cont, _sumG); + hidden_ptr_r += 8; + weight_hc_I += 8; + weight_hc_F += 8; + weight_hc_O += 8; + weight_hc_G += 8; + } + float sums[4]; + _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG)); + sums[0] += bias_c_I[q]; + sums[1] += bias_c_F[q]; + sums[2] += bias_c_O[q]; + sums[3] += bias_c_G[q]; + + for (; remain_size > 0; remain_size--) + { + float xi = *x; + sums[0] += *weight_xc_I * xi; + sums[1] += *weight_xc_F * xi; + sums[2] += *weight_xc_O * xi; + sums[3] += *weight_xc_G * xi; + x++; + weight_xc_I++; + weight_xc_F++; + weight_xc_O++; + weight_xc_G++; + } + + for (; remain_num_output > 0; remain_num_output--) + { + float h_cont = *hidden_ptr_r; + sums[0] += *weight_hc_I * h_cont; + sums[1] += *weight_hc_F * h_cont; + sums[2] += *weight_hc_O * h_cont; + sums[3] += *weight_hc_G * h_cont; + hidden_ptr_r++; + weight_hc_I++; + weight_hc_F++; + weight_hc_O++; + weight_hc_G++; + } + gates_data_I[q] = sums[0]; + gates_data_F[q] = sums[1]; + gates_data_O[q] = sums[2]; + gates_data_G[q] = sums[3]; + } + + // lstm unit + // sigmoid(I) + // sigmoid(F) + // sigmoid(O) + // tanh(G) + // c_t := f_t .* c_{t-1} + i_t .* g_t + // h_t := o_t .* tanh[c_t] + float* output_data = top_blob.row(ti); + float* cell_ptr = cell_state; + float* hidden_ptr = hidden_state; + const float* gates_data_I = gates.row(0); + const float* gates_data_F = gates.row(1); + const float* gates_data_O = gates.row(2); + const float* gates_data_G = gates.row(3); + int nn_activation = num_output >> 3; + int remain_activations = num_output & 7; + for (; nn_activation > 0; nn_activation--) + { + __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I)); + __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F)); + __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O)); + __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G)); + __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G)); + __m256 H = _mm256_mul_ps(O, tanh_avx(cell2)); + _mm256_storeu_ps(cell_ptr, cell2); + _mm256_storeu_ps(hidden_ptr, H); + _mm256_storeu_ps(output_data, H); + cell_ptr += 8; + output_data += 8; + hidden_ptr += 8; + gates_data_I += 8; + gates_data_F += 8; + gates_data_O += 8; + gates_data_G += 8; + } + for (; remain_activations > 0; remain_activations--) + { + float I = *gates_data_I; + float F = *gates_data_F; + float O = *gates_data_O; + float G = *gates_data_G; + + I = 1.f / (1.f + exp(-I)); + F = 1.f / (1.f + exp(-F)); + O = 1.f / (1.f + exp(-O)); + G = tanh(G); + float cell2 = F * *cell_ptr + I * G; + float H = O * tanh(cell2); + *cell_ptr = cell2; + *hidden_ptr = H; + *output_data = H; + cell_ptr++; + output_data++; + hidden_ptr++; + gates_data_I++; + gates_data_F++; + gates_data_O++; + gates_data_G++; + } + + // no cell output here + } + + return 0; +} +#endif + +int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const +{ +#if __AVX__ + int T = bottom_blob.h; + int num_directions = direction == 2 ? 2 : 1; + + // initial hidden state + Mat hidden(num_output, 4u, opt.workspace_allocator); + if (hidden.empty()) + return -100; + hidden.fill(0.f); + // internal cell state + Mat cell(num_output, 4u, opt.workspace_allocator); + if (cell.empty()) + return -100; + cell.fill(0.f); + + top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + // Uni directional + if (direction == 0 || direction == 1) + { + if (opt.use_fp16_storage) + { + // Uni directional + int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt); + if (ret != 0) + return ret; + } + else + { + // Uni directional + int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); + if (ret != 0) + return ret; + } + } + + if (direction == 2) + { + Mat top_blob_forward(num_output, T, 4u, opt.workspace_allocator); + if (top_blob_forward.empty()) + return -100; + + Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator); + if (top_blob_reverse.empty()) + return -100; + + if (opt.use_fp16_storage) + { + // Uni directional + int ret0 = lstm_fp16(bottom_blob, top_blob_forward, 0, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt); + if (ret0 != 0) + return ret0; + } + else + { + // Uni directional + int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); + if (ret0 != 0) + return ret0; + } + + hidden.fill(0.0f); + cell.fill(0.0f); + if (opt.use_fp16_storage) + { + // Uni directional + int ret1 = lstm_fp16(bottom_blob, top_blob_reverse, 1, weight_xc_data_fp16.channel(1), bias_c_data.channel(1), weight_hc_data_fp16.channel(1), hidden, cell, opt); + if (ret1 != 0) + return ret1; + } + else + { + // Uni directional + int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt); + if (ret1 != 0) + return ret1; + } + + // concat w + for (int i = 0; i < T; i++) + { + const float* pf = top_blob_forward.row(i); + const float* pr = top_blob_reverse.row(i); + float* ptr = top_blob.row(i); + + memcpy(ptr, pf, num_output * sizeof(float)); + memcpy(ptr + num_output, pr, num_output * sizeof(float)); + } + } + + return 0; +#else + return LSTM::forward(bottom_blob, top_blob, opt); +#endif +} + +int LSTM_x86::forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const +{ +#if __AVX__ + if (bottom_blobs.size() != 3 || top_blobs.size() != 3) + { + return forward(bottom_blobs[0], top_blobs[0], opt); + } + const Mat& bottom_blob = bottom_blobs[0]; + + int T = bottom_blob.h; + Mat& top_blob = top_blobs[0]; + Mat& hidden_state = top_blobs[1]; + Mat& cell_state = top_blobs[2]; + + //Copy previous states + hidden_state = bottom_blobs[1].clone(opt.blob_allocator); + cell_state = bottom_blobs[2].clone(opt.blob_allocator); + + top_blob.create(num_output, T, 4u, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + if (opt.use_fp16_storage) + { + // Uni directional + int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden_state, cell_state, opt); + if (ret != 0) + return ret; + } + else + { + // Uni directional + int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt); + if (ret != 0) + return ret; + } + return 0; +#else + return LSTM::forward(bottom_blobs, top_blobs, opt); +#endif +} + +} // namespace ncnn diff --git a/src/layer/x86/lstm_x86.h b/src/layer/x86/lstm_x86.h new file mode 100644 index 000000000..2a48a41ed --- /dev/null +++ b/src/layer/x86/lstm_x86.h @@ -0,0 +1,40 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_LSTM_X86_H +#define LAYER_LSTM_X86_H + +#include "lstm.h" + +namespace ncnn { + +class LSTM_x86 : virtual public LSTM +{ +public: + LSTM_x86(); + + virtual int create_pipeline(const Option& opt); + + virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const; + + virtual int forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const; + +public: + Mat weight_hc_data_fp16; + Mat weight_xc_data_fp16; +}; + +} // namespace ncnn + +#endif // LAYER_LSTM_X86_H diff --git a/src/layer/x86/scale_x86.cpp b/src/layer/x86/scale_x86.cpp new file mode 100644 index 000000000..9b63144c2 --- /dev/null +++ b/src/layer/x86/scale_x86.cpp @@ -0,0 +1,261 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#if __AVX__ +#include +#endif // __AVX__ + +#include "scale_x86.h" + +namespace ncnn { + +DEFINE_LAYER_CREATOR(Scale_x86) + +Scale_x86::Scale_x86() +{ +#if __AVX__ + support_packing = true; +#endif // __AVX__ +} + +int Scale_x86::forward_inplace(std::vector& bottom_top_blobs, const Option& opt) const +{ + Mat& bottom_top_blob = bottom_top_blobs[0]; + const Mat& scale_blob = bottom_top_blobs[1]; + + int dims = bottom_top_blob.dims; + int elempack = bottom_top_blob.elempack; + +#if __AVX__ + if (elempack == 8) + { + if (dims == 1) + { + int w = bottom_top_blob.w; + + const float* scale = scale_blob; + if (bias_term) + { + const float* bias = bias_data; + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < w; i++) + { + float* ptr = (float*)bottom_top_blob + i * 8; + + __m256 _p = _mm256_loadu_ps(ptr); + __m256 _s = _mm256_loadu_ps(scale + i * 8); + __m256 _bias = _mm256_loadu_ps(bias + i * 8); + _p = _mm256_fmadd_ps(_p, _s, _bias); + _mm256_storeu_ps(ptr, _p); + } + } + else + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < w; i++) + { + float* ptr = (float*)bottom_top_blob + i * 8; + + __m256 _p = _mm256_loadu_ps(ptr); + __m256 _s = _mm256_loadu_ps(scale + i * 8); + _p = _mm256_mul_ps(_p, _s); + _mm256_storeu_ps(ptr, _p); + } + } + } + + if (dims == 2) + { + int w = bottom_top_blob.w; + int h = bottom_top_blob.h; + + if (bias_term) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < h; i++) + { + float* ptr = bottom_top_blob.row(i); + __m256 _s = _mm256_loadu_ps((const float*)scale_blob + i * 8); + __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); + + for (int j = 0; j < w; j++) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_fmadd_ps(_p, _s, _bias); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } + } + } + else + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int i = 0; i < h; i++) + { + float* ptr = bottom_top_blob.row(i); + __m256 _s = _mm256_loadu_ps((const float*)scale_blob + i * 8); + + for (int j = 0; j < w; j++) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_mul_ps(_p, _s); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } + } + } + } + + if (dims == 3) + { + int w = bottom_top_blob.w; + int h = bottom_top_blob.h; + int channels = bottom_top_blob.c; + int size = w * h; + + if (bias_term) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + __m256 _s = _mm256_loadu_ps((const float*)scale_blob + q * 8); + __m256 _bias = _mm256_loadu_ps((const float*)bias_data + q * 8); + + for (int i = 0; i < size; i++) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_fmadd_ps(_p, _s, _bias); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } + } + } + else + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + __m256 _s = _mm256_loadu_ps((const float*)scale_blob + q * 8); + + for (int i = 0; i < size; i++) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_mul_ps(_p, _s); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } + } + } + } + + return 0; + } +#endif // __AVX__ + + if (dims != 3) + return Scale::forward_inplace(bottom_top_blobs, opt); + + int w = bottom_top_blob.w; + int h = bottom_top_blob.h; + int channels = bottom_top_blob.c; + int size = w * h; + + if (bias_term) + { + const float* scale_ptr = scale_blob; + const float* bias_ptr = bias_data; + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + + float s = scale_ptr[q]; + float bias = bias_ptr[q]; + +#if __AVX__ + int nn = size >> 3; + int remain = size & 7; +#else + int remain = size; +#endif // __AVX__ + +#if __AVX__ + __m256 _s = _mm256_set1_ps(s); + __m256 _bias = _mm256_set1_ps(bias); + for (; nn > 0; nn--) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_fmadd_ps(_p, _s, _bias); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } +#endif // __AVX__ + + for (; remain > 0; remain--) + { + *ptr = *ptr * s + bias; + + ptr++; + } + } + } + else + { + const float* scale_ptr = scale_blob; + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + + float s = scale_ptr[q]; + +#if __AVX__ + int nn = size >> 3; + int remain = size & 7; +#else + int remain = size; +#endif // __AVX__ + +#if __AVX__ + __m256 _s = _mm256_set1_ps(s); + for (; nn > 0; nn--) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = _mm256_mul_ps(_p, _s); + _mm256_storeu_ps(ptr, _p); + + ptr += 8; + } +#endif // __AVX__ + + for (; remain > 0; remain--) + { + *ptr *= s; + + ptr++; + } + } + } + + return 0; +} + +} // namespace ncnn diff --git a/src/layer/x86/scale_x86.h b/src/layer/x86/scale_x86.h new file mode 100644 index 000000000..840e6903c --- /dev/null +++ b/src/layer/x86/scale_x86.h @@ -0,0 +1,32 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_SCALE_X86_H +#define LAYER_SCALE_X86_H + +#include "scale.h" + +namespace ncnn { + +class Scale_x86 : virtual public Scale +{ +public: + Scale_x86(); + + virtual int forward_inplace(std::vector& bottom_top_blobs, const Option& opt) const; +}; + +} // namespace ncnn + +#endif // LAYER_SCALE_X86_H diff --git a/src/layer/x86/sigmoid_x86.cpp b/src/layer/x86/sigmoid_x86.cpp index a97b95dce..9c6a83c00 100644 --- a/src/layer/x86/sigmoid_x86.cpp +++ b/src/layer/x86/sigmoid_x86.cpp @@ -67,7 +67,7 @@ int Sigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const int remain = size & 7; #else int remain = size; -#endif // __ARM_NEON +#endif // __AVX__ #if __AVX__ for (; nn > 0; nn--) @@ -76,7 +76,7 @@ int Sigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const _mm256_storeu_ps(ptr, sigmoid_avx(_p)); ptr += 8; } -#endif // __ARM_NEON +#endif // __AVX__ for (; remain > 0; remain--) { *ptr = 1.f / (1.f + exp(-*ptr)); diff --git a/src/layer/x86/slice_x86.cpp b/src/layer/x86/slice_x86.cpp new file mode 100644 index 000000000..bdc2ce9e5 --- /dev/null +++ b/src/layer/x86/slice_x86.cpp @@ -0,0 +1,438 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#include + +#if __AVX__ +#include +#endif // __AVX__ + +#include "slice_x86.h" + +#include "layer_type.h" + +namespace ncnn { + +DEFINE_LAYER_CREATOR(Slice_x86) + +Slice_x86::Slice_x86() +{ +#if __AVX__ + support_packing = true; + + packing_pack1 = 0; +#endif // __AVX__ +} + +int Slice_x86::create_pipeline(const Option& opt) +{ +#if __AVX__ + if (opt.use_packing_layout) + { + packing_pack1 = ncnn::create_layer(ncnn::LayerType::Packing); + + ncnn::ParamDict pd; + pd.set(0, 1); + + packing_pack1->load_param(pd); + + packing_pack1->create_pipeline(opt); + } +#endif // __AVX__ + + return 0; +} + +int Slice_x86::destroy_pipeline(const Option& opt) +{ +#if __AVX__ + if (opt.use_packing_layout) + { + if (packing_pack1) + { + packing_pack1->destroy_pipeline(opt); + delete packing_pack1; + packing_pack1 = 0; + } + } +#endif // __AVX__ + + return 0; +} + +int Slice_x86::forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const +{ + const Mat& bottom_blob = bottom_blobs[0]; + int dims = bottom_blob.dims; + size_t elemsize = bottom_blob.elemsize; + int elempack = bottom_blob.elempack; + const int* slices_ptr = slices; + +#if __AVX__ + if (opt.use_packing_layout) + { + if (dims == 1) // axis == 0 + { + // slice vector + int w = bottom_blob.w * elempack; + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (w - q) / (top_blobs.size() - i); + } + + int out_elempack = slice % 8 == 0 ? 8 : 1; + size_t out_elemsize = elemsize / elempack * out_elempack; + + Mat& top_blob = top_blobs[i]; + top_blob.create(slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + const float* ptr = (const float*)bottom_blob + q; + float* outptr = top_blob; + memcpy(outptr, ptr, top_blob.w * top_blob.elemsize); + + q += slice; + } + + return 0; + } + + if (dims == 2 && axis == 0) + { + // slice image height + int w = bottom_blob.w; + int h = bottom_blob.h * elempack; + + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (h - q) / (top_blobs.size() - i); + } + + int out_elempack = slice % 8 == 0 ? 8 : 1; + size_t out_elemsize = elemsize / elempack * out_elempack; + + Mat& top_blob = top_blobs[i]; + top_blob.create(w, slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + q += slice; + } + + size_t out_elemsize = top_blobs[0].elemsize; + int out_elempack = top_blobs[0].elempack; + for (size_t i = 0; i < top_blobs.size(); i++) + { + out_elemsize = std::min(out_elemsize, top_blobs[i].elemsize); + out_elempack = std::min(out_elempack, top_blobs[i].elempack); + } + + Mat bottom_blob_unpacked = bottom_blob; + if (elempack == 8 && out_elempack == 1) + { + packing_pack1->forward(bottom_blob, bottom_blob_unpacked, opt); + } + + const float* ptr = bottom_blob_unpacked; + for (size_t i = 0; i < top_blobs.size(); i++) + { + Mat& top_blob = top_blobs[i]; + + if (out_elempack == 1 && top_blob.elempack == 8) + { + for (int j = 0; j < top_blob.h; j++) + { + const float* r0 = ptr; + const float* r1 = ptr + w; + const float* r2 = ptr + w * 2; + const float* r3 = ptr + w * 3; + const float* r4 = ptr + w * 4; + const float* r5 = ptr + w * 5; + const float* r6 = ptr + w * 6; + const float* r7 = ptr + w * 7; + + float* outptr0 = top_blob.row(j); + + for (int j = 0; j < w; j++) + { + outptr0[0] = *r0++; + outptr0[1] = *r1++; + outptr0[2] = *r2++; + outptr0[3] = *r3++; + outptr0[4] = *r4++; + outptr0[5] = *r5++; + outptr0[6] = *r6++; + outptr0[7] = *r7++; + + outptr0 += 8; + } + + ptr += w * 8; + } + } + else // if (out_elempack == 1 && top_blob.elempack == 1) if (out_elempack == 8 && top_blob.elempack == 8) + { + int size = w * top_blob.h; + + float* outptr = top_blob; + memcpy(outptr, ptr, size * top_blob.elemsize); + + ptr += size * top_blob.elempack; + } + } + + return 0; + } + + if (dims == 2 && axis == 1) + { + // slice image width + int w = bottom_blob.w; + int h = bottom_blob.h; + + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (w - q) / (top_blobs.size() - i); + } + + Mat& top_blob = top_blobs[i]; + top_blob.create(slice, h, elemsize, elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + q += slice; + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int j = 0; j < h; j++) + { + const float* ptr = bottom_blob.row(j); + for (size_t i = 0; i < top_blobs.size(); i++) + { + Mat& top_blob = top_blobs[i]; + + float* outptr = top_blob.row(j); + memcpy(outptr, ptr, top_blob.w * elemsize); + + ptr += top_blob.w * elempack; + } + } + + return 0; + } + + if (dims == 3 && axis == 0) + { + // slice dim channel + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c * elempack; + + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (channels - q) / (top_blobs.size() - i); + } + + int out_elempack = slice % 8 == 0 ? 8 : 1; + size_t out_elemsize = elemsize / elempack * out_elempack; + + Mat& top_blob = top_blobs[i]; + top_blob.create(w, h, slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + q += slice; + } + + size_t out_elemsize = top_blobs[0].elemsize; + int out_elempack = top_blobs[0].elempack; + for (size_t i = 0; i < top_blobs.size(); i++) + { + out_elemsize = std::min(out_elemsize, top_blobs[i].elemsize); + out_elempack = std::min(out_elempack, top_blobs[i].elempack); + } + + Mat bottom_blob_unpacked = bottom_blob; + if (elempack == 8 && out_elempack == 1) + { + packing_pack1->forward(bottom_blob, bottom_blob_unpacked, opt); + } + + int p = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + Mat& top_blob = top_blobs[i]; + + if (out_elempack == 1 && top_blob.elempack == 8) + { + int size = top_blob.w * top_blob.h; + + for (int q = 0; q < top_blob.c; q++) + { + const float* r0 = bottom_blob_unpacked.channel(p); + const float* r1 = bottom_blob_unpacked.channel(p + 1); + const float* r2 = bottom_blob_unpacked.channel(p + 2); + const float* r3 = bottom_blob_unpacked.channel(p + 3); + const float* r4 = bottom_blob_unpacked.channel(p + 4); + const float* r5 = bottom_blob_unpacked.channel(p + 5); + const float* r6 = bottom_blob_unpacked.channel(p + 6); + const float* r7 = bottom_blob_unpacked.channel(p + 7); + + float* outptr0 = top_blob.channel(q); + + for (int j = 0; j < size; j++) + { + outptr0[0] = *r0++; + outptr0[1] = *r1++; + outptr0[2] = *r2++; + outptr0[3] = *r3++; + outptr0[4] = *r4++; + outptr0[5] = *r5++; + outptr0[6] = *r6++; + outptr0[7] = *r7++; + + outptr0 += 8; + } + + p += 8; + } + } + else // if (out_elempack == 1 && top_blob.elempack == 1) if (out_elempack == 8 && top_blob.elempack == 8) + { + int size = top_blob.total(); + + const float* ptr = bottom_blob_unpacked.channel(p); + float* outptr = top_blob; + memcpy(outptr, ptr, size * top_blob.elemsize); + + p += top_blob.c; + } + } + + return 0; + } + + if (dims == 3 && axis == 1) + { + // slice dim height + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (h - q) / (top_blobs.size() - i); + } + + Mat& top_blob = top_blobs[i]; + top_blob.create(w, slice, channels, elemsize, elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + q += slice; + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < channels; p++) + { + const float* ptr = bottom_blob.channel(p); + + for (size_t i = 0; i < top_blobs.size(); i++) + { + Mat& top_blob = top_blobs[i]; + + int size = top_blob.w * top_blob.h; + + float* outptr = top_blob.channel(p); + memcpy(outptr, ptr, size * elemsize); + + ptr += size * elempack; + } + } + + return 0; + } + + if (dims == 3 && axis == 2) + { + // slice dim width + int w = bottom_blob.w; + int h = bottom_blob.h; + int channels = bottom_blob.c; + + int q = 0; + for (size_t i = 0; i < top_blobs.size(); i++) + { + int slice = slices_ptr[i]; + if (slice == -233) + { + slice = (w - q) / (top_blobs.size() - i); + } + + Mat& top_blob = top_blobs[i]; + top_blob.create(slice, h, channels, elemsize, elempack, opt.blob_allocator); + if (top_blob.empty()) + return -100; + + q += slice; + } + + #pragma omp parallel for num_threads(opt.num_threads) + for (int p = 0; p < channels; p++) + { + const float* ptr = bottom_blob.channel(p); + + for (int j = 0; j < h; j++) + { + for (size_t i = 0; i < top_blobs.size(); i++) + { + Mat& top_blob = top_blobs[i]; + + float* outptr = top_blob.channel(p).row(j); + memcpy(outptr, ptr, top_blob.w * elemsize); + + ptr += top_blob.w * elempack; + } + } + } + + return 0; + } + + } // opt.use_packing_layout +#endif // __AVX__ + + return Slice::forward(bottom_blobs, top_blobs, opt); +} +} // namespace ncnn diff --git a/src/layer/x86/slice_x86.h b/src/layer/x86/slice_x86.h new file mode 100644 index 000000000..d5bc698d4 --- /dev/null +++ b/src/layer/x86/slice_x86.h @@ -0,0 +1,38 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_SLICE_x86_H +#define LAYER_SLICE_x86_H + +#include "slice.h" + +namespace ncnn { + +class Slice_x86 : virtual public Slice +{ +public: + Slice_x86(); + + virtual int create_pipeline(const Option& opt); + virtual int destroy_pipeline(const Option& opt); + + virtual int forward(const std::vector& bottom_blobs, std::vector& top_blobs, const Option& opt) const; + +public: + ncnn::Layer* packing_pack1; +}; + +} // namespace ncnn + +#endif // LAYER_SLICE_x86_H diff --git a/src/layer/x86/tanh_x86.cpp b/src/layer/x86/tanh_x86.cpp new file mode 100644 index 000000000..005e94aec --- /dev/null +++ b/src/layer/x86/tanh_x86.cpp @@ -0,0 +1,93 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#if __AVX__ +#include "avx_activation.h" +#endif // __AVX__ + +#include "tanh_x86.h" + +#include + +namespace ncnn { + +DEFINE_LAYER_CREATOR(TanH_x86) + +TanH_x86::TanH_x86() +{ +#if __AVX__ + support_packing = true; +#endif // __AVX__ +} + +int TanH_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const +{ + int w = bottom_top_blob.w; + int h = bottom_top_blob.h; + int channels = bottom_top_blob.c; + int size = w * h; + int elempack = bottom_top_blob.elempack; + +#if __AVX__ + if (elempack == 8) + { + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + + for (int i = 0; i < size; i++) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = tanh_avx(_p); + _mm256_storeu_ps(ptr, _p); + ptr += 8; + } + } + + return 0; + } +#endif // __AVX__ + + #pragma omp parallel for num_threads(opt.num_threads) + for (int q = 0; q < channels; q++) + { + float* ptr = bottom_top_blob.channel(q); + +#if __AVX__ + int nn = size >> 3; + int remain = size - (nn << 3); +#else + int remain = size; +#endif // __AVX__ + +#if __AVX__ + for (; nn > 0; nn--) + { + __m256 _p = _mm256_loadu_ps(ptr); + _p = tanh_avx(_p); + _mm256_storeu_ps(ptr, _p); + ptr += 8; + } +#endif // __AVX__ + for (; remain > 0; remain--) + { + *ptr = tanh(*ptr); + ptr++; + } + } + + return 0; +} + +} // namespace ncnn diff --git a/src/layer/x86/tanh_x86.h b/src/layer/x86/tanh_x86.h new file mode 100644 index 000000000..60913d49c --- /dev/null +++ b/src/layer/x86/tanh_x86.h @@ -0,0 +1,32 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef LAYER_TANH_X86_H +#define LAYER_TANH_X86_H + +#include "tanh.h" + +namespace ncnn { + +class TanH_x86 : virtual public TanH +{ +public: + TanH_x86(); + + virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const; +}; + +} // namespace ncnn + +#endif // LAYER_TANH_X86_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2d6b464f6..7ee9c56e1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -68,3 +68,4 @@ ncnn_add_layer_test(TanH) ncnn_add_layer_test(UnaryOp) ncnn_add_layer_test(Mish) ncnn_add_layer_test(Swish) +ncnn_add_layer_test(LSTM) diff --git a/tests/test_lstm.cpp b/tests/test_lstm.cpp new file mode 100644 index 000000000..f0189a26b --- /dev/null +++ b/tests/test_lstm.cpp @@ -0,0 +1,217 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "layer/lstm.h" +#include "testutil.h" + +static int test_lstm(const ncnn::Mat& a, int outch, int direction) +{ + int input_size = a.w * a.h * a.c; + int num_directions = direction == 2 ? 2 : 1; + + ncnn::ParamDict pd; + pd.set(0, outch); // num_output + pd.set(1, outch * input_size * 4 * num_directions); + pd.set(2, direction); // bias_term + + std::vector weights(3); + weights[0] = RandomMat(outch * input_size * 4 * num_directions); + weights[1] = RandomMat(outch * 4 * num_directions); + weights[2] = RandomMat(outch * outch * 4 * num_directions); + ncnn::Option opt; + opt.num_threads = 1; + opt.use_int8_inference = false; + + int ret = test_layer("LSTM", pd, weights, opt, a); + if (ret != 0) + { + fprintf(stderr, "test_lstm failed a.dims=%d a=(%d %d %d) outch=%d, direction = %d \n", a.dims, a.w, a.h, a.c, outch, direction); + } + + return ret; +} + +int test_lstm_layer(const ncnn::Mat& a, int outch, int direction, float epsilon = 0.01) +{ + int input_size = a.w * a.h * a.c; + ncnn::ParamDict pd; + pd.set(0, outch); // num_output + pd.set(1, outch * input_size * 4); + pd.set(2, direction); // bias_term + int num_directions = direction == 2 ? 2 : 1; + + std::vector weights(3); + weights[0] = RandomMat(outch * input_size * 4 * num_directions); + weights[1] = RandomMat(outch * 4 * num_directions); + weights[2] = RandomMat(outch * outch * 4 * num_directions); + + ncnn::Option opt; + opt.num_threads = 1; + opt.use_int8_inference = false; + + ncnn::LSTM* op = (ncnn::LSTM*)ncnn::create_layer(ncnn::layer_to_index("LSTM")); + + if (!op->support_vulkan) opt.use_vulkan_compute = false; + if (!op->support_packing) opt.use_packing_layout = false; + if (!op->support_bf16_storage) opt.use_bf16_storage = false; + if (!op->support_image_storage) opt.use_image_storage = false; + + op->load_param(pd); + + ncnn::ModelBinFromMatArray mb(weights.data()); + + op->load_model(mb); + + op->create_pipeline(opt); + + ncnn::Mat b; + op->LSTM::forward(a, b, opt); + + std::vector _c1(3); + std::vector _c2(3); + std::vector a1(3); + std::vector a2(3); + if (direction == 0) + { + a1[0] = a.row_range(0, a.h / 2).clone(); + a2[0] = a.row_range(a.h / 2, a.h - a.h / 2).clone(); + } + else + { + a2[0] = a.row_range(0, a.h / 2).clone(); + a1[0] = a.row_range(a.h / 2, a.h - a.h / 2).clone(); + } + + // initial hidden state + ncnn::Mat hidden(outch); + if (hidden.empty()) + return -100; + hidden.fill(0.f); + + ncnn::Mat cell(outch); + if (cell.empty()) + return -100; + cell.fill(0.f); + + a1[1] = hidden; + a1[2] = cell; + op->forward(a1, _c1, opt); + a2[1] = _c1[1]; + a2[2] = _c1[2]; + op->forward(a2, _c2, opt); + + ncnn::Mat c1 = _c1[0]; + ncnn::Mat c2 = _c2[0]; + + if (direction == 1) + { + c2 = _c1[0]; + c1 = _c2[0]; + } + + // total height + ncnn::Mat c; + c.create(b.w, b.h, b.elemsize, opt.blob_allocator); + if (c.empty()) + return -100; + + unsigned char* outptr = c; + int c1_size = c1.w * c1.h; + const unsigned char* c1ptr = c1; + memcpy(outptr, c1ptr, c1_size * c1.elemsize); + outptr += c1_size * c1.elemsize; + int c2_size = c2.w * c2.h; + const unsigned char* c2ptr = c2; + memcpy(outptr, c2ptr, c2_size * c2.elemsize); + + op->destroy_pipeline(opt); + + delete op; + + if (CompareMat(b, c, epsilon) != 0) + { + fprintf(stderr, "test_lstm two step failed a.dims=%d a=(%d %d %d) outch=%d, direction = %d \n", a.dims, a.w, a.h, a.c, outch, direction); + return -1; + } + + return 0; +} + +static int test_lstm_0() +{ + return 0 + || test_lstm(RandomMat(4, 1), 2, 2) + || test_lstm(RandomMat(8, 2), 2, 2) + || test_lstm(RandomMat(16, 8), 7, 2) + || test_lstm(RandomMat(17, 8), 8, 2) + || test_lstm(RandomMat(19, 15), 8, 2) + || test_lstm(RandomMat(5, 16), 16, 2) + || test_lstm(RandomMat(3, 16), 8, 2) + || test_lstm(RandomMat(8, 16), 16, 2) + || test_lstm(RandomMat(2, 5), 17, 2); +} + +static int test_lstm_1() +{ + return 0 + || test_lstm_layer(RandomMat(4, 4), 1, 1) + || test_lstm_layer(RandomMat(8, 2), 2, 1) + || test_lstm_layer(RandomMat(16, 8), 7, 1) + || test_lstm_layer(RandomMat(17, 8), 8, 1) + || test_lstm_layer(RandomMat(19, 15), 8, 1) + || test_lstm_layer(RandomMat(5, 16), 16, 1) + || test_lstm_layer(RandomMat(3, 16), 8, 1) + || test_lstm_layer(RandomMat(2, 5), 99, 1) + || test_lstm_layer(RandomMat(4, 2), 1, 0) + || test_lstm_layer(RandomMat(8, 2), 2, 0) + || test_lstm_layer(RandomMat(16, 8), 7, 0) + || test_lstm_layer(RandomMat(17, 8), 8, 0) + || test_lstm_layer(RandomMat(19, 15), 8, 0) + || test_lstm_layer(RandomMat(5, 16), 16, 0) + || test_lstm_layer(RandomMat(3, 16), 8, 0) + || test_lstm_layer(RandomMat(2, 5), 17, 0); +} + +static int test_lstm_2() +{ + return 0 + || test_lstm(RandomMat(4, 1), 1, 0) + || test_lstm(RandomMat(8, 2), 2, 0) + || test_lstm(RandomMat(16, 8), 7, 0) + || test_lstm(RandomMat(17, 8), 8, 0) + || test_lstm(RandomMat(19, 15), 8, 0) + || test_lstm(RandomMat(5, 16), 16, 0) + || test_lstm(RandomMat(3, 16), 8, 0) + || test_lstm(RandomMat(8, 16), 16, 0) + || test_lstm(RandomMat(2, 5), 17, 0); +} +static int test_lstm_3() +{ + return 0 + || test_lstm(RandomMat(4, 1), 1, 1) + || test_lstm(RandomMat(8, 2), 2, 1) + || test_lstm(RandomMat(16, 8), 7, 1) + || test_lstm(RandomMat(17, 8), 8, 1) + || test_lstm(RandomMat(19, 15), 8, 1) + || test_lstm(RandomMat(5, 16), 16, 1) + || test_lstm(RandomMat(3, 16), 8, 1) + || test_lstm(RandomMat(8, 16), 16, 1) + || test_lstm(RandomMat(2, 5), 17, 1); +} + +int main() +{ + SRAND(7767517); + return 0 || test_lstm_0() || test_lstm_1() || test_lstm_2() || test_lstm_3(); +} diff --git a/tests/testutil.h b/tests/testutil.h index 5b071404a..6217e8856 100644 --- a/tests/testutil.h +++ b/tests/testutil.h @@ -515,7 +515,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vectorsupport_inplace) { @@ -663,7 +662,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vector int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector& weights, const ncnn::Option& _opt, const std::vector& a, int top_blob_count = 1, float epsilon = 0.001, void (*func)(T*) = 0) { - ncnn::Option opts[3]; + ncnn::Option opts[4]; opts[0] = _opt; opts[0].use_packing_layout = false; opts[0].use_fp16_packed = false; @@ -683,8 +682,14 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec opts[2].use_bf16_storage = true; opts[2].use_shader_pack8 = true; opts[2].use_image_storage = true; - - for (int i = 0; i < 3; i++) + opts[3] = _opt; + opts[3].use_packing_layout = true; + opts[3].use_fp16_packed = true; + opts[3].use_fp16_storage = true; + opts[3].use_bf16_storage = false; + opts[3].use_shader_pack8 = true; + opts[3].use_image_storage = true; + for (int i = 0; i < 4; i++) { const ncnn::Option& opt = opts[i]; @@ -750,7 +755,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec template int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector& weights, const ncnn::Option& _opt, const ncnn::Mat& a, float epsilon = 0.001, void (*func)(T*) = 0) { - ncnn::Option opts[3]; + ncnn::Option opts[4]; opts[0] = _opt; opts[0].use_packing_layout = false; opts[0].use_fp16_packed = false; @@ -770,11 +775,16 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec opts[2].use_bf16_storage = true; opts[2].use_shader_pack8 = true; opts[2].use_image_storage = true; - - for (int i = 0; i < 3; i++) + opts[3] = _opt; + opts[3].use_packing_layout = true; + opts[3].use_fp16_packed = true; + opts[3].use_fp16_storage = true; + opts[3].use_bf16_storage = false; + opts[3].use_shader_pack8 = true; + opts[3].use_image_storage = true; + for (int i = 0; i < 4; i++) { const ncnn::Option& opt = opts[i]; - // fp16 representation ncnn::Mat a_fp16; std::vector weights_fp16; diff --git a/toolchains/arm-linux-gnueabihf.toolchain.cmake b/toolchains/arm-linux-gnueabihf.toolchain.cmake index a705c51f9..c7ef24b62 100644 --- a/toolchains/arm-linux-gnueabihf.toolchain.cmake +++ b/toolchains/arm-linux-gnueabihf.toolchain.cmake @@ -8,8 +8,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_C_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4") -set(CMAKE_CXX_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4") +set(CMAKE_C_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4 -mfp16-format=ieee") +set(CMAKE_CXX_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4 -mfp16-format=ieee") # cache flags set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags")