LSTM arm/x86 + fp16 innerproduct arm (#1881)

* added fp16 weight storage version

* Small changes

* Fixed fp16 weight storage layers

* fix innerproduct

* fix loop error

* Fix windows build.

Disable fp 16 conversion when detecting int8 weights.

Implement requested changes.

* Restyled by clang-format

* Restyled by astyle

* Restyled by clang-format

* Restyled by astyle

* Update option.cpp

Set fp16 storage based on vulkan being used or not.

* added ability for storing state in lstm layer

* added avx lstm

* added arm lstm

* fix innerproduct activation location and add 4 parallel channel version

* Restyled by clang-format

* Restyled by astyle

* Restyled by clang-format

* Restyled by astyle

* revert arm file

* commit before switch

* implement requested changes

* Restyled by clang-format

* Restyled by astyle

* Restyled by clang-format

* Restyled by astyle

* More x86 optimized implementations of common layers.

Added LSTM layers for arm and x86 + a ctest to verify the layer accuracy

Added fp16 innerproduct for arm

* fix non avx build

* Add fp16 arm compiler and cpu checks. Remove statefullness from LSTM implementation.

* Fix build check for fp16 arm

* Bypass lstm_fp16 if not supported

* Build order was incorrect

* fix std::min missing in windows build

* Restyled by clang-format

* Restyled by astyle

* Restyled by clang-format

* Restyled by astyle

* attempting to fix gnu build by enabling: -mfp16-format=ieee to fix the missing __fp16 type

* remove double "fix"

* Specify ieee fp16 format

* implement requested changes

* fix arm non-fp16 build

* fix arm lstm

* Restyled/pull 1881 (#15)

* Restyled by clang-format

* Restyled by astyle

* Restyled by clang-format

* Restyled by astyle

Co-authored-by: Restyled.io <commits@restyled.io>

* Check blob size on arm lstm

* fix styling

Co-authored-by: Restyled.io <commits@restyled.io>

src/layer/arm/innerproduct_arm.cpp

@@ -20,7 +20,7 @@
 #include <arm_neon.h>
 #include "neon_mathfun.h"
 #endif // __ARM_NEON
-
+#include "cpu.h"
 #include "neon_activation.h"
 
 namespace ncnn {
@@ -57,7 +57,12 @@ int InnerProduct_arm::create_pipeline(const Option& opt)
     {
         ncnn::cast_float32_to_bfloat16(weight_data, weight_data_bf16, opt);
     }
-
+#if __ARM_NEON && (__ARM_FP & 2)
+    else if (opt.use_fp16_storage && weight_data.elemsize == 4u && cpu_support_arm_vfpv4())
+    {
+        ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt);
+    }
+#endif
     return 0;
 }
 
@@ -72,6 +77,237 @@ int InnerProduct_arm::destroy_pipeline(const Option& opt)
 
     return 0;
 }
+#if __ARM_NEON && (__ARM_FP & 2)
+int InnerProduct_arm::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    size_t elemsize = bottom_blob.elemsize;
+    int elempack = bottom_blob.elempack;
+    int size = w * h;
+
+    top_blob.create(num_output, elemsize, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    const unsigned short* weight_data_ptr = (const unsigned short*)weight_data_fp16;
+
+    int nn_num_output = num_output >> 2;
+    int remain_num_output_start = nn_num_output << 2;
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int pp = 0; pp < nn_num_output; pp++)
+    {
+        int p = pp * 4;
+
+        float sum0 = 0.f;
+        float sum1 = 0.f;
+        float sum2 = 0.f;
+        float sum3 = 0.f;
+
+        if (bias_term)
+        {
+            sum0 = bias_data[p];
+            sum1 = bias_data[p + 1];
+            sum2 = bias_data[p + 2];
+            sum3 = bias_data[p + 3];
+        }
+
+        const unsigned short* w0 = (const unsigned short*)weight_data_ptr + size * channels * p;
+        const unsigned short* w1 = (const unsigned short*)weight_data_ptr + size * channels * (p + 1);
+        const unsigned short* w2 = (const unsigned short*)weight_data_ptr + size * channels * (p + 2);
+        const unsigned short* w3 = (const unsigned short*)weight_data_ptr + size * channels * (p + 3);
+
+        float32x4_t _sum0 = vdupq_n_f32(0.f);
+        float32x4_t _sum1 = vdupq_n_f32(0.f);
+        float32x4_t _sum2 = vdupq_n_f32(0.f);
+        float32x4_t _sum3 = vdupq_n_f32(0.f);
+
+        // channels
+        for (int q = 0; q < channels; q++)
+        {
+            const float* m = bottom_blob.channel(q);
+
+            int nn = size >> 2;
+            int remain = size & 3;
+
+            for (; nn > 0; nn--)
+            {
+                float32x4_t _m = vld1q_f32(m);
+
+                float32x4_t _w0 = loadfp16(w0);
+                _sum0 = vmlaq_f32(_sum0, _m, _w0);
+
+                float32x4_t _w1 = loadfp16(w1);
+                _sum1 = vmlaq_f32(_sum1, _m, _w1);
+
+                float32x4_t _w2 = loadfp16(w2);
+                _sum2 = vmlaq_f32(_sum2, _m, _w2);
+
+                float32x4_t _w3 = loadfp16(w3);
+                _sum3 = vmlaq_f32(_sum3, _m, _w3);
+
+                m += 4;
+                w0 += 4;
+                w1 += 4;
+                w2 += 4;
+                w3 += 4;
+            }
+            if (remain != 0)
+            {
+                unsigned short fp16_weights[4][4] = {{0}};
+                float _m_f[4] = {0};
+                int i = 0;
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (; remain > 0; remain--)
+                {
+                    _m_f[i] = *m;
+                    fp16_weights[0][i] = *w0;
+                    fp16_weights[1][i] = *w1;
+                    fp16_weights[2][i] = *w2;
+                    fp16_weights[3][i] = *w3;
+                    i++;
+                    m++;
+                    w0++;
+                    w1++;
+                    w2++;
+                    w3++;
+                }
+                float32x4_t _m = vld1q_f32(_m_f);
+
+                float32x4_t _w0 = loadfp16(fp16_weights[0]);
+                _sum0 = vmlaq_f32(_sum0, _m, _w0);
+
+                float32x4_t _w1 = loadfp16(fp16_weights[1]);
+                _sum1 = vmlaq_f32(_sum1, _m, _w1);
+
+                float32x4_t _w2 = loadfp16(fp16_weights[2]);
+                _sum2 = vmlaq_f32(_sum2, _m, _w2);
+
+                float32x4_t _w3 = loadfp16(fp16_weights[3]);
+                _sum3 = vmlaq_f32(_sum3, _m, _w3);
+            }
+        }
+
+        float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));
+        float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sum1), vget_high_f32(_sum1));
+        float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sum2), vget_high_f32(_sum2));
+        float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sum3), vget_high_f32(_sum3));
+
+        float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss);
+        float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss);
+
+        sum0 += vget_lane_f32(_sum01ss, 0);
+        sum1 += vget_lane_f32(_sum01ss, 1);
+        sum2 += vget_lane_f32(_sum23ss, 0);
+        sum3 += vget_lane_f32(_sum23ss, 1);
+
+        if (activation_type == 1)
+        {
+            sum0 = std::max(sum0, 0.f);
+            sum1 = std::max(sum1, 0.f);
+            sum2 = std::max(sum2, 0.f);
+            sum3 = std::max(sum3, 0.f);
+        }
+        else if (activation_type == 2)
+        {
+            float slope = activation_params[0];
+            sum0 = sum0 > 0.f ? sum0 : sum0 * slope;
+            sum1 = sum1 > 0.f ? sum1 : sum1 * slope;
+            sum2 = sum2 > 0.f ? sum2 : sum2 * slope;
+            sum3 = sum3 > 0.f ? sum3 : sum3 * slope;
+        }
+        else if (activation_type == 3)
+        {
+            float min = activation_params[0];
+            float max = activation_params[1];
+            if (sum0 < min) sum0 = min;
+            if (sum0 > max) sum0 = max;
+            if (sum1 < min) sum1 = min;
+            if (sum1 > max) sum1 = max;
+            if (sum2 < min) sum2 = min;
+            if (sum2 > max) sum2 = max;
+            if (sum3 < min) sum3 = min;
+            if (sum3 > max) sum3 = max;
+        }
+        else if (activation_type == 4)
+        {
+            sum0 = static_cast<float>(1.f / (1.f + exp(-sum0)));
+            sum1 = static_cast<float>(1.f / (1.f + exp(-sum1)));
+            sum2 = static_cast<float>(1.f / (1.f + exp(-sum2)));
+            sum3 = static_cast<float>(1.f / (1.f + exp(-sum3)));
+        }
+
+        top_blob[p] = sum0;
+        top_blob[p + 1] = sum1;
+        top_blob[p + 2] = sum2;
+        top_blob[p + 3] = sum3;
+    }
+    // num_output
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int p = remain_num_output_start; p < num_output; p++)
+    {
+        float sum0 = 0.f;
+        if (bias_term)
+        {
+            sum0 = bias_data[p];
+        }
+
+        const unsigned short* w0 = (const unsigned short*)weight_data_ptr + size * channels * p;
+
+        float32x4_t _sum0 = vdupq_n_f32(0.f);
+        // channels
+        for (int q = 0; q < channels; q++)
+        {
+            const float* m = bottom_blob.channel(q);
+
+            int nn = size >> 2;
+            int remain = size & 3;
+
+            for (; nn > 0; nn--)
+            {
+                float32x4_t _m = vld1q_f32(m);
+
+                float32x4_t _w0 = loadfp16(w0);
+                _sum0 = vmlaq_f32(_sum0, _m, _w0);
+
+                m += 4;
+                w0 += 4;
+            }
+            if (remain != 0)
+            {
+                unsigned short fp16_weights[4] = {0};
+                float _m_f[4] = {0};
+                int i = 0;
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (; remain > 0; remain--)
+                {
+                    _m_f[i] = *m;
+                    fp16_weights[i] = *w0;
+                    i++;
+                    m++;
+                    w0++;
+                }
+                float32x4_t _m = vld1q_f32(_m_f);
+
+                float32x4_t _w0 = loadfp16(fp16_weights);
+                _sum0 = vmlaq_f32(_sum0, _m, _w0);
+            }
+        }
+
+        float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sum0), vget_high_f32(_sum0));
+
+        sum0 += vget_lane_f32(_sum0ss, 0) + vget_lane_f32(_sum0ss, 1);
+
+        sum0 = activation_ss(sum0, activation_type, activation_params);
+
+        top_blob[p] = sum0;
+    }
+    return 0;
+}
+#endif
 
 int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
 {
@@ -111,11 +347,21 @@ int InnerProduct_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Optio
             bottom_blob_flattened.elemsize = 4u;
             bottom_blob_flattened.elempack = 1;
         }
-
+#if (__ARM_FP & 2)
+        if (opt.use_fp16_storage && cpu_support_arm_vfpv4())
+        {
+            return forward_fp16(bottom_blob_flattened, top_blob, opt);
+        }
+#endif
         return forward(bottom_blob_flattened, top_blob, opt);
     }
-#endif // __ARM_NEON
-
+#if (__ARM_FP & 2)
+    if (opt.use_fp16_storage && cpu_support_arm_vfpv4())
+    {
+        return forward_fp16(bottom_blob, top_blob, opt);
+    }
+#endif
+#endif
     top_blob.create(num_output, elemsize, opt.blob_allocator);
     if (top_blob.empty())
         return -100;

src/layer/arm/innerproduct_arm.h

@@ -30,6 +30,11 @@ public:
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 protected:
+#if __ARM_NEON
+    int forward_fp16(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
+#endif
+
     int forward_bf16s(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
 public:
@@ -37,6 +42,9 @@ public:
 
     // bf16
     Mat weight_data_bf16;
+
+    // fp16
+    Mat weight_data_fp16;
 };
 
 } // namespace ncnn

src/layer/arm/lstm_arm.cpp

@@ -0,0 +1,599 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#include "lstm_arm.h"
+
+#include <math.h>
+
+#if __ARM_NEON
+#include "neon_mathfun.h"
+#include "neon_activation.h"
+#endif // __ARM_NEON
+
+#include "cpu.h"
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(LSTM_arm)
+
+LSTM_arm::LSTM_arm()
+{
+    one_blob_only = false;
+    support_inplace = false;
+}
+int LSTM_arm::create_pipeline(const Option& opt)
+{
+#if __ARM_NEON
+    if (opt.use_fp16_storage)
+    {
+        ncnn::cast_float32_to_float16(weight_xc_data, weight_xc_data_fp16, opt);
+        ncnn::cast_float32_to_float16(weight_hc_data, weight_hc_data_fp16, opt);
+    }
+#endif // __ARM_NEON
+
+    return 0;
+}
+
+#if __ARM_NEON
+static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)
+{
+    int size = bottom_blob.w;
+    int T = bottom_blob.h;
+
+    int num_output = top_blob.w;
+
+    // 4 x num_output
+    Mat gates(num_output, 4, 4u, opt.workspace_allocator);
+    if (gates.empty())
+        return -100;
+
+    // unroll
+    for (int t = 0; t < T; t++)
+    {
+        // clip hidden by continuation indicator
+        // h_cont_{t-1} = cont_t * h_{t-1}
+        // h_cont_{t-1} = h_{t-1} if cont_t == 1
+        //                0       otherwise
+        // calculate hidden
+        // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c
+
+        int ti = reverse ? T - 1 - t : t;
+        for (int q = 0; q < num_output; q++)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const float* weight_xc_I = weight_xc.row(num_output * 0 + q);
+            const float* weight_xc_F = weight_xc.row(num_output * 1 + q);
+            const float* weight_xc_O = weight_xc.row(num_output * 2 + q);
+            const float* weight_xc_G = weight_xc.row(num_output * 3 + q);
+
+            const float* weight_hc_I = weight_hc.row(num_output * 0 + q);
+            const float* weight_hc_F = weight_hc.row(num_output * 1 + q);
+            const float* weight_hc_O = weight_hc.row(num_output * 2 + q);
+            const float* weight_hc_G = weight_hc.row(num_output * 3 + q);
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            float32x4_t _sumI = vdupq_n_f32(0.0f);
+            float32x4_t _sumF = vdupq_n_f32(0.0f);
+            float32x4_t _sumO = vdupq_n_f32(0.0f);
+            float32x4_t _sumG = vdupq_n_f32(0.0f);
+            int nn_num_size = size >> 2;
+            int remain_size = size & 3;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                float32x4_t xi = vld1q_f32(x);
+                _sumI = vmlaq_f32(_sumI, vld1q_f32(weight_xc_I), xi);
+                _sumF = vmlaq_f32(_sumF, vld1q_f32(weight_xc_F), xi);
+                _sumO = vmlaq_f32(_sumO, vld1q_f32(weight_xc_O), xi);
+                _sumG = vmlaq_f32(_sumG, vld1q_f32(weight_xc_G), xi);
+                x += 4;
+                weight_xc_I += 4;
+                weight_xc_F += 4;
+                weight_xc_O += 4;
+                weight_xc_G += 4;
+            }
+            int nn_num_output = num_output >> 2;
+            int remain_num_output = num_output & 3;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                float32x4_t h_cont = vld1q_f32(hidden_ptr_r);
+
+                _sumI = vmlaq_f32(_sumI, vld1q_f32(weight_hc_I), h_cont);
+                _sumF = vmlaq_f32(_sumF, vld1q_f32(weight_hc_F), h_cont);
+                _sumO = vmlaq_f32(_sumO, vld1q_f32(weight_hc_O), h_cont);
+                _sumG = vmlaq_f32(_sumG, vld1q_f32(weight_hc_G), h_cont);
+                hidden_ptr_r += 4;
+                weight_hc_I += 4;
+                weight_hc_F += 4;
+                weight_hc_O += 4;
+                weight_hc_G += 4;
+            }
+            float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sumI), vget_high_f32(_sumI));
+            float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sumF), vget_high_f32(_sumF));
+            float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sumO), vget_high_f32(_sumO));
+            float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sumG), vget_high_f32(_sumG));
+
+            float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss);
+            float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss);
+
+            float sums0 = vget_lane_f32(_sum01ss, 0) + bias_c_I[q];
+            float sums1 = vget_lane_f32(_sum01ss, 1) + bias_c_F[q];
+            float sums2 = vget_lane_f32(_sum23ss, 0) + bias_c_O[q];
+            float sums3 = vget_lane_f32(_sum23ss, 1) + bias_c_G[q];
+
+            for (; remain_size > 0; remain_size--)
+            {
+                float xi = *x;
+                sums0 += *weight_xc_I * xi;
+                sums1 += *weight_xc_F * xi;
+                sums2 += *weight_xc_O * xi;
+                sums3 += *weight_xc_G * xi;
+                x++;
+                weight_xc_I++;
+                weight_xc_F++;
+                weight_xc_O++;
+                weight_xc_G++;
+            }
+
+            for (; remain_num_output > 0; remain_num_output--)
+            {
+                float h_cont = *hidden_ptr_r;
+                sums0 += *weight_hc_I * h_cont;
+                sums1 += *weight_hc_F * h_cont;
+                sums2 += *weight_hc_O * h_cont;
+                sums3 += *weight_hc_G * h_cont;
+                hidden_ptr_r++;
+                weight_hc_I++;
+                weight_hc_F++;
+                weight_hc_O++;
+                weight_hc_G++;
+            }
+            gates_data_I[q] = sums0;
+            gates_data_F[q] = sums1;
+            gates_data_O[q] = sums2;
+            gates_data_G[q] = sums3;
+        }
+
+        // lstm unit
+        // sigmoid(I)
+        // sigmoid(F)
+        // sigmoid(O)
+        // tanh(G)
+        // c_t := f_t .* c_{t-1} + i_t .* g_t
+        // h_t := o_t .* tanh[c_t]
+        float* output_data = top_blob.row(ti);
+        float* cell_ptr = cell_state;
+        float* hidden_ptr = hidden_state;
+        const float* gates_data_I = gates.row(0);
+        const float* gates_data_F = gates.row(1);
+        const float* gates_data_O = gates.row(2);
+        const float* gates_data_G = gates.row(3);
+        int nn_activation = num_output >> 2;
+        int remain_activations = num_output & 3;
+        for (; nn_activation > 0; nn_activation--)
+        {
+            float32x4_t I = sigmoid_ps(vld1q_f32(gates_data_I));
+            float32x4_t F = sigmoid_ps(vld1q_f32(gates_data_F));
+            float32x4_t O = sigmoid_ps(vld1q_f32(gates_data_O));
+            float32x4_t G = tanh_ps(vld1q_f32(gates_data_G));
+            float32x4_t cell2 = vaddq_f32(vmulq_f32(F, vld1q_f32(cell_ptr)), vmulq_f32(I, G));
+            float32x4_t H = vmulq_f32(O, tanh_ps(cell2));
+            vst1q_f32(cell_ptr, cell2);
+            vst1q_f32(hidden_ptr, H);
+            vst1q_f32(output_data, H);
+            cell_ptr += 4;
+            output_data += 4;
+            hidden_ptr += 4;
+            gates_data_I += 4;
+            gates_data_F += 4;
+            gates_data_O += 4;
+            gates_data_G += 4;
+        }
+        for (; remain_activations > 0; remain_activations--)
+        {
+            float I = *gates_data_I;
+            float F = *gates_data_F;
+            float O = *gates_data_O;
+            float G = *gates_data_G;
+
+            I = 1.f / (1.f + exp(-I));
+            F = 1.f / (1.f + exp(-F));
+            O = 1.f / (1.f + exp(-O));
+            G = tanh(G);
+            float cell2 = F * *cell_ptr + I * G;
+            float H = O * tanh(cell2);
+            *cell_ptr = cell2;
+            *hidden_ptr = H;
+            *output_data = H;
+            cell_ptr++;
+            output_data++;
+            hidden_ptr++;
+            gates_data_I++;
+            gates_data_F++;
+            gates_data_O++;
+            gates_data_G++;
+        }
+
+        // no cell output here
+    }
+
+    return 0;
+}
+#if (__ARM_FP & 2)
+static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)
+{
+    int size = bottom_blob.w;
+    int T = bottom_blob.h;
+
+    int num_output = top_blob.w;
+
+    // 4 x num_output
+    Mat gates(num_output, 4, 4u, opt.workspace_allocator);
+    if (gates.empty())
+        return -100;
+
+    // unroll
+    for (int t = 0; t < T; t++)
+    {
+        // clip hidden by continuation indicator
+        // h_cont_{t-1} = cont_t * h_{t-1}
+        // h_cont_{t-1} = h_{t-1} if cont_t == 1
+        //                0       otherwise
+        // calculate hidden
+        // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c
+
+        int ti = reverse ? T - 1 - t : t;
+        for (int q = 0; q < num_output; q++)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const unsigned short* weight_xc_I = (const unsigned short*)weight_xc.row(num_output * 0 + q);
+            const unsigned short* weight_xc_F = (const unsigned short*)weight_xc.row(num_output * 1 + q);
+            const unsigned short* weight_xc_O = (const unsigned short*)weight_xc.row(num_output * 2 + q);
+            const unsigned short* weight_xc_G = (const unsigned short*)weight_xc.row(num_output * 3 + q);
+
+            const unsigned short* weight_hc_I = (const unsigned short*)weight_hc.row(num_output * 0 + q);
+            const unsigned short* weight_hc_F = (const unsigned short*)weight_hc.row(num_output * 1 + q);
+            const unsigned short* weight_hc_O = (const unsigned short*)weight_hc.row(num_output * 2 + q);
+            const unsigned short* weight_hc_G = (const unsigned short*)weight_hc.row(num_output * 3 + q);
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            float32x4_t _sumI = vdupq_n_f32(0.0f);
+            float32x4_t _sumF = vdupq_n_f32(0.0f);
+            float32x4_t _sumO = vdupq_n_f32(0.0f);
+            float32x4_t _sumG = vdupq_n_f32(0.0f);
+            int nn_num_size = size >> 2;
+            int remain_size = size & 3;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                float32x4_t xi = vld1q_f32(x);
+                _sumI = vmlaq_f32(_sumI, loadfp16(weight_xc_I), xi);
+                _sumF = vmlaq_f32(_sumF, loadfp16(weight_xc_F), xi);
+                _sumO = vmlaq_f32(_sumO, loadfp16(weight_xc_O), xi);
+                _sumG = vmlaq_f32(_sumG, loadfp16(weight_xc_G), xi);
+                x += 4;
+                weight_xc_I += 4;
+                weight_xc_F += 4;
+                weight_xc_O += 4;
+                weight_xc_G += 4;
+            }
+            int nn_num_output = num_output >> 2;
+            int remain_num_output = num_output & 3;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                float32x4_t h_cont = vld1q_f32(hidden_ptr_r);
+
+                _sumI = vmlaq_f32(_sumI, loadfp16(weight_hc_I), h_cont);
+                _sumF = vmlaq_f32(_sumF, loadfp16(weight_hc_F), h_cont);
+                _sumO = vmlaq_f32(_sumO, loadfp16(weight_hc_O), h_cont);
+                _sumG = vmlaq_f32(_sumG, loadfp16(weight_hc_G), h_cont);
+                hidden_ptr_r += 4;
+                weight_hc_I += 4;
+                weight_hc_F += 4;
+                weight_hc_O += 4;
+                weight_hc_G += 4;
+            }
+            if (remain_size)
+            {
+                unsigned short fp16_weights[4][4] = {{0}};
+                float _xi_f[4] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_size; i++)
+                {
+                    _xi_f[i] = *x;
+                    fp16_weights[0][i] = *weight_xc_I;
+                    fp16_weights[1][i] = *weight_xc_F;
+                    fp16_weights[2][i] = *weight_xc_O;
+                    fp16_weights[3][i] = *weight_xc_G;
+                    x++;
+                    weight_xc_I++;
+                    weight_xc_F++;
+                    weight_xc_O++;
+                    weight_xc_G++;
+                }
+                float32x4_t xi = vld1q_f32(_xi_f);
+                _sumI = vmlaq_f32(_sumI, loadfp16(fp16_weights[0]), xi);
+                _sumF = vmlaq_f32(_sumF, loadfp16(fp16_weights[1]), xi);
+                _sumO = vmlaq_f32(_sumO, loadfp16(fp16_weights[2]), xi);
+                _sumG = vmlaq_f32(_sumG, loadfp16(fp16_weights[3]), xi);
+            }
+            if (remain_num_output)
+            {
+                unsigned short fp16_weights[4][4] = {{0}};
+                float _hcont_f[4] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_num_output; i++)
+                {
+                    _hcont_f[i] = *hidden_ptr_r;
+                    fp16_weights[0][i] = *weight_hc_I;
+                    fp16_weights[1][i] = *weight_hc_F;
+                    fp16_weights[2][i] = *weight_hc_O;
+                    fp16_weights[3][i] = *weight_hc_G;
+                    hidden_ptr_r++;
+                    weight_hc_I++;
+                    weight_hc_F++;
+                    weight_hc_O++;
+                    weight_hc_G++;
+                }
+                float32x4_t h_cont = vld1q_f32(_hcont_f);
+                _sumI = vmlaq_f32(_sumI, loadfp16(fp16_weights[0]), h_cont);
+                _sumF = vmlaq_f32(_sumF, loadfp16(fp16_weights[1]), h_cont);
+                _sumO = vmlaq_f32(_sumO, loadfp16(fp16_weights[2]), h_cont);
+                _sumG = vmlaq_f32(_sumG, loadfp16(fp16_weights[3]), h_cont);
+            }
+            float32x2_t _sum0ss = vadd_f32(vget_low_f32(_sumI), vget_high_f32(_sumI));
+            float32x2_t _sum1ss = vadd_f32(vget_low_f32(_sumF), vget_high_f32(_sumF));
+            float32x2_t _sum2ss = vadd_f32(vget_low_f32(_sumO), vget_high_f32(_sumO));
+            float32x2_t _sum3ss = vadd_f32(vget_low_f32(_sumG), vget_high_f32(_sumG));
+
+            float32x2_t _sum01ss = vpadd_f32(_sum0ss, _sum1ss);
+            float32x2_t _sum23ss = vpadd_f32(_sum2ss, _sum3ss);
+
+            float sums0 = vget_lane_f32(_sum01ss, 0) + bias_c_I[q];
+            float sums1 = vget_lane_f32(_sum01ss, 1) + bias_c_F[q];
+            float sums2 = vget_lane_f32(_sum23ss, 0) + bias_c_O[q];
+            float sums3 = vget_lane_f32(_sum23ss, 1) + bias_c_G[q];
+
+            gates_data_I[q] = sums0;
+            gates_data_F[q] = sums1;
+            gates_data_O[q] = sums2;
+            gates_data_G[q] = sums3;
+        }
+
+        // lstm unit
+        // sigmoid(I)
+        // sigmoid(F)
+        // sigmoid(O)
+        // tanh(G)
+        // c_t := f_t .* c_{t-1} + i_t .* g_t
+        // h_t := o_t .* tanh[c_t]
+        float* output_data = top_blob.row(ti);
+        float* cell_ptr = cell_state;
+        float* hidden_ptr = hidden_state;
+        const float* gates_data_I = gates.row(0);
+        const float* gates_data_F = gates.row(1);
+        const float* gates_data_O = gates.row(2);
+        const float* gates_data_G = gates.row(3);
+        int nn_activation = num_output >> 2;
+        int remain_activations = num_output & 3;
+        for (; nn_activation > 0; nn_activation--)
+        {
+            float32x4_t I = sigmoid_ps(vld1q_f32(gates_data_I));
+            float32x4_t F = sigmoid_ps(vld1q_f32(gates_data_F));
+            float32x4_t O = sigmoid_ps(vld1q_f32(gates_data_O));
+            float32x4_t G = tanh_ps(vld1q_f32(gates_data_G));
+            float32x4_t cell2 = vaddq_f32(vmulq_f32(F, vld1q_f32(cell_ptr)), vmulq_f32(I, G));
+            float32x4_t H = vmulq_f32(O, tanh_ps(cell2));
+            vst1q_f32(cell_ptr, cell2);
+            vst1q_f32(hidden_ptr, H);
+            vst1q_f32(output_data, H);
+            cell_ptr += 4;
+            output_data += 4;
+            hidden_ptr += 4;
+            gates_data_I += 4;
+            gates_data_F += 4;
+            gates_data_O += 4;
+            gates_data_G += 4;
+        }
+        for (; remain_activations > 0; remain_activations--)
+        {
+            float I = *gates_data_I;
+            float F = *gates_data_F;
+            float O = *gates_data_O;
+            float G = *gates_data_G;
+
+            I = 1.f / (1.f + exp(-I));
+            F = 1.f / (1.f + exp(-F));
+            O = 1.f / (1.f + exp(-O));
+            G = tanh(G);
+            float cell2 = F * *cell_ptr + I * G;
+            float H = O * tanh(cell2);
+            *cell_ptr = cell2;
+            *hidden_ptr = H;
+            *output_data = H;
+            cell_ptr++;
+            output_data++;
+            hidden_ptr++;
+            gates_data_I++;
+            gates_data_F++;
+            gates_data_O++;
+            gates_data_G++;
+        }
+
+        // no cell output here
+    }
+
+    return 0;
+}
+#endif
+#endif
+int LSTM_arm::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+#if __ARM_NEON
+    int T = bottom_blob.h;
+    int num_directions = direction == 2 ? 2 : 1;
+
+    // initial hidden state
+    Mat hidden(num_output, 4u, opt.workspace_allocator);
+    if (hidden.empty())
+        return -100;
+    hidden.fill(0.f);
+    // internal cell state
+    Mat cell(num_output, 4u, opt.workspace_allocator);
+    if (cell.empty())
+        return -100;
+    cell.fill(0.f);
+
+    top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // Uni directional
+    if (direction == 0 || direction == 1)
+    {
+#if (__ARM_FP & 2)
+        if (opt.use_fp16_storage && cpu_support_arm_vfpv4())
+        {
+            // Uni directional
+            return lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);
+        }
+#endif
+        // Uni directional
+        return lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
+    }
+
+    if (direction == 2)
+    {
+        Mat top_blob_forward(num_output, T, 4u, opt.workspace_allocator);
+        if (top_blob_forward.empty())
+            return -100;
+
+        Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator);
+        if (top_blob_reverse.empty())
+            return -100;
+#if (__ARM_FP & 2)
+        if (opt.use_fp16_storage && cpu_support_arm_vfpv4())
+        {
+            // Uni directional
+            int ret0 = lstm_fp16(bottom_blob, top_blob_forward, 0, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);
+            if (ret0 != 0)
+                return ret0;
+            hidden.fill(0.0f);
+            cell.fill(0.0f);
+            // Uni directional
+            int ret1 = lstm_fp16(bottom_blob, top_blob_reverse, 1, weight_xc_data_fp16.channel(1), bias_c_data.channel(1), weight_hc_data_fp16.channel(1), hidden, cell, opt);
+            if (ret1 != 0)
+                return ret1;
+        }
+        else
+        {
+#endif
+            // Uni directional
+            int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
+            if (ret0 != 0)
+                return ret0;
+
+            hidden.fill(0.0f);
+            cell.fill(0.0f);
+
+            // Uni directional
+            int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt);
+            if (ret1 != 0)
+                return ret1;
+#if (__ARM_FP & 2)
+        }
+#endif
+
+        // concat w
+        for (int i = 0; i < T; i++)
+        {
+            const float* pf = top_blob_forward.row(i);
+            const float* pr = top_blob_reverse.row(i);
+            float* ptr = top_blob.row(i);
+
+            memcpy(ptr, pf, num_output * sizeof(float));
+            memcpy(ptr + num_output, pr, num_output * sizeof(float));
+        }
+    }
+
+    return 0;
+#else
+    return LSTM::forward(bottom_blob, top_blob, opt);
+#endif
+}
+
+int LSTM_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
+{
+#if __ARM_NEON
+    if (bottom_blobs.size() != 3 || top_blobs.size() != 3)
+    {
+        return forward(bottom_blobs[0], top_blobs[0], opt);
+    }
+    const Mat& bottom_blob = bottom_blobs[0];
+
+    int T = bottom_blob.h;
+    Mat& top_blob = top_blobs[0];
+    Mat& hidden_state = top_blobs[1];
+    Mat& cell_state = top_blobs[2];
+
+    //Copy previous states
+    hidden_state = bottom_blobs[1].clone(opt.blob_allocator);
+    cell_state = bottom_blobs[2].clone(opt.blob_allocator);
+
+    top_blob.create(num_output, T, 4u, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+#if (__ARM_FP & 2)
+    if (opt.use_fp16_storage && cpu_support_arm_vfpv4())
+    {
+        // Uni directional
+        return lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden_state, cell_state, opt);
+    }
+#endif
+    // Uni directional
+    return lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt);
+#else
+    return LSTM::forward(bottom_blobs, top_blobs, opt);
+#endif
+}
+
+} // namespace ncnn

src/layer/arm/lstm_arm.h

@@ -0,0 +1,38 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_LSTM_ARM_H
+#define LAYER_LSTM_ARM_H
+
+#include "lstm.h"
+
+namespace ncnn {
+
+class LSTM_arm : virtual public LSTM
+{
+public:
+    LSTM_arm();
+
+    virtual int create_pipeline(const Option& opt);
+    virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+    virtual int forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const;
+
+public:
+    Mat weight_hc_data_fp16;
+    Mat weight_xc_data_fp16;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_LSTM_ARM_H

src/layer/arm/neon_activation.h

@@ -49,6 +49,17 @@ static inline float activation_ss(float v, int activation_type, const ncnn::Mat&
 }
 
 #if __ARM_NEON
+
+static inline float32x4_t sigmoid_ps(float32x4_t _v)
+{
+    float32x4_t _one = vdupq_n_f32(1.f);
+    _v = vnegq_f32(_v);
+    _v = exp_ps(_v);
+    _v = vaddq_f32(_v, _one);
+    float32x4_t _outp = vrecpeq_f32(_v);
+    return vmulq_f32(vrecpsq_f32(_v, _outp), _outp);
+}
+
 static inline float32x4_t activation_ps(float32x4_t _v, int activation_type, const ncnn::Mat& activation_params)
 {
     if (activation_type == 1)
@@ -73,14 +84,7 @@ static inline float32x4_t activation_ps(float32x4_t _v, int activation_type, con
     }
     else if (activation_type == 4)
     {
-        float32x4_t _one = vdupq_n_f32(1.f);
-        _v = vnegq_f32(_v);
-        _v = exp_ps(_v);
-        _v = vaddq_f32(_v, _one);
-        float32x4_t _outp = vrecpeq_f32(_v);
-        _outp = vmulq_f32(vrecpsq_f32(_v, _outp), _outp);
-        //         _outp = vmulq_f32(vrecpsq_f32(_v, _outp), _outp);
-        _v = _outp;
+        _v = sigmoid_ps(_v);
     }
     else if (activation_type == 5)
     {

src/layer/arm/neon_mathfun.h

@@ -27,6 +27,13 @@
 
 #include <arm_neon.h>
 
+#if (__ARM_FP & 2)
+static inline float32x4_t loadfp16(const void* ptr)
+{
+    return vcvt_f32_f16(vld1_f16((const __fp16*)ptr));
+}
+#endif
+
 #define c_inv_mant_mask ~0x7f800000u
 #define c_cephes_SQRTHF 0.707106781186547524
 #define c_cephes_log_p0 7.0376836292E-2

src/layer/lstm.cpp

@@ -22,7 +22,7 @@ DEFINE_LAYER_CREATOR(LSTM)
 
 LSTM::LSTM()
 {
-    one_blob_only = true;
+    one_blob_only = false;
     support_inplace = false;
 }
 
@@ -31,7 +31,8 @@ int LSTM::load_param(const ParamDict& pd)
     num_output = pd.get(0, 0);
     weight_data_size = pd.get(1, 0);
     direction = pd.get(2, 0);
-
+    if (direction == 2)
+        one_blob_only = true;
     return 0;
 }
 
@@ -57,31 +58,18 @@ int LSTM::load_model(const ModelBin& mb)
     return 0;
 }
 
-static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, const Option& opt)
+static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)
 {
     int size = bottom_blob.w;
     int T = bottom_blob.h;
 
     int num_output = top_blob.w;
 
-    // initial hidden state
-    Mat hidden(num_output, 4u, opt.workspace_allocator);
-    if (hidden.empty())
-        return -100;
-
-    // internal cell state
-    Mat cell(num_output, 4u, opt.workspace_allocator);
-    if (cell.empty())
-        return -100;
-
     // 4 x num_output
     Mat gates(4, num_output, 4u, opt.workspace_allocator);
     if (gates.empty())
         return -100;
 
-    hidden.fill(0.f);
-    cell.fill(0.f);
-
     // unroll
     for (int t = 0; t < T; t++)
     {
@@ -91,7 +79,6 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w
         //                0       otherwise
         // calculate hidden
         // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c
-        int cont = t > 0;
 
         int ti = reverse ? T - 1 - t : t;
 
@@ -133,7 +120,7 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w
 
             for (int i = 0; i < num_output; i++)
             {
-                float h_cont = cont ? hidden[i] : 0.f;
+                float h_cont = hidden_state[i];
 
                 I += weight_hc_I[i] * h_cont;
                 F += weight_hc_F[i] * h_cont;
@@ -165,18 +152,16 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w
             float G = gates_data[3];
 
             I = 1.f / (1.f + exp(-I));
-            F = cont ? 1.f / (1.f + exp(-F)) : 0.f;
+            F = 1.f / (1.f + exp(-F));
             O = 1.f / (1.f + exp(-O));
             G = tanh(G);
 
-            float cell2 = F * cell[q] + I * G;
+            float cell2 = F * cell_state[q] + I * G;
             float H = O * tanh(cell2);
-            cell[q] = cell2;
-            hidden[q] = H;
+            cell_state[q] = cell2;
+            hidden_state[q] = H;
             output_data[q] = H;
         }
-
-        // no cell output here
     }
 
     return 0;
@@ -188,21 +173,25 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons
 
     int num_directions = direction == 2 ? 2 : 1;
 
+    // initial hidden state
+    Mat hidden(num_output, 4u, opt.workspace_allocator);
+    if (hidden.empty())
+        return -100;
+    hidden.fill(0.f);
+
+    Mat cell(num_output, 4u, opt.workspace_allocator);
+    if (cell.empty())
+        return -100;
+    cell.fill(0.f);
+
     top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator);
     if (top_blob.empty())
         return -100;
 
-    // forward
-    if (direction == 0)
-    {
-        int ret = lstm(bottom_blob, top_blob, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt);
-        if (ret != 0)
-            return ret;
-    }
-
-    if (direction == 1)
+    // Uni directional
+    if (direction == 0 || direction == 1)
     {
-        int ret = lstm(bottom_blob, top_blob, 1, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt);
+        int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
         if (ret != 0)
             return ret;
     }
@@ -217,11 +206,14 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons
         if (top_blob_reverse.empty())
             return -100;
 
-        int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), opt);
+        int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
         if (ret0 != 0)
             return ret0;
 
-        int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), opt);
+        hidden.fill(0.0f);
+        cell.fill(0.0f);
+
+        int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt);
         if (ret1 != 0)
             return ret1;
 
@@ -240,4 +232,35 @@ int LSTM::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) cons
     return 0;
 }
 
+int LSTM::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
+{
+    if (bottom_blobs.size() != 3 || top_blobs.size() != 3)
+    {
+        return forward(bottom_blobs[0], top_blobs[0], opt);
+    }
+    const Mat& bottom_blob = bottom_blobs[0];
+    int T = bottom_blob.h;
+    Mat& top_blob = top_blobs[0];
+    Mat& hidden_state = top_blobs[1];
+    Mat& cell_state = top_blobs[2];
+
+    //Copy previous states
+    hidden_state = bottom_blobs[1].clone(opt.blob_allocator);
+    cell_state = bottom_blobs[2].clone(opt.blob_allocator);
+
+    top_blob.create(num_output, T, 4u, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // Uni directional
+    if (direction == 0 || direction == 1)
+    {
+        int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt);
+        if (ret != 0)
+            return ret;
+    }
+
+    return 0;
+}
+
 } // namespace ncnn

src/layer/lstm.h

@@ -30,6 +30,8 @@ public:
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+    virtual int forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const;
+
 public:
     int num_output;
     int weight_data_size;

src/layer/x86/bias_x86.cpp

@@ -0,0 +1,69 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#if __AVX__
+#include <immintrin.h>
+#endif // __AVX__
+
+#include "bias_x86.h"
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(Bias_x86)
+
+int Bias_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const
+{
+    int w = bottom_top_blob.w;
+    int h = bottom_top_blob.h;
+    int channels = bottom_top_blob.c;
+    int size = w * h;
+
+    const float* bias_ptr = bias_data;
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int q = 0; q < channels; q++)
+    {
+        float* ptr = bottom_top_blob.channel(q);
+
+        float bias = bias_ptr[q];
+
+#if __AVX__
+        int nn = size >> 3;
+        int remain = size & 7;
+#else
+        int remain = size;
+#endif // __AVX__
+
+#if __AVX__
+        __m256 _bias = _mm256_set1_ps(bias);
+        for (; nn > 0; nn--)
+        {
+            __m256 _p = _mm256_loadu_ps(ptr);
+            __m256 _outp = _mm256_add_ps(_p, _bias);
+            _mm256_storeu_ps(ptr, _outp);
+
+            ptr += 8;
+        }
+#endif // __AVX__
+
+        for (; remain > 0; remain--)
+        {
+            *ptr = *ptr + bias;
+
+            ptr++;
+        }
+    }
+
+    return 0;
+}
+
+} // namespace ncnn

src/layer/x86/bias_x86.h

@@ -0,0 +1,30 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_BIAS_X86_H
+#define LAYER_BIAS_X86_H
+
+#include "bias.h"
+
+namespace ncnn {
+
+class Bias_x86 : virtual public Bias
+{
+public:
+    virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_BIAS_X86_H

src/layer/x86/hardsigmoid_x86.cpp

@@ -68,7 +68,26 @@ int HardSigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) co
     for (int q = 0; q < channels; q++)
     {
         float* ptr = bottom_top_blob.channel(q);
+#if __AVX__
+        int nn_size = size >> 3;
+        int remain = size & 7;
+
+        __m256 _zero = _mm256_set1_ps(0.f);
+        __m256 _one = _mm256_set1_ps(1.f);
+        for (; nn_size > 0; nn_size--)
+        {
+            __m256 _p = _mm256_loadu_ps(ptr);
+            __m256 _ans = _mm256_set1_ps(beta);
+            _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans);
+            _ans = _mm256_max_ps(_ans, _zero);
+            _ans = _mm256_min_ps(_ans, _one);
+            _mm256_storeu_ps(ptr, _ans);
+
+            ptr += 8;
+        }
+#else
         int remain = size;
+#endif
         for (; remain > 0; remain--)
         {
             if (*ptr < lower)

src/layer/x86/innerproduct_x86.cpp

@@ -87,7 +87,6 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob,
     size_t elemsize = bottom_blob.elemsize;
     int elempack = bottom_blob.elempack;
     int size = w * h;
-    // fprintf(stderr, "bottom_blob %d x %d x %d, elempack = %d \n", w,h,channels,elempack);
 #if __AVX__
     if (elempack == 8)
     {

src/layer/x86/lstm_x86.cpp

@@ -0,0 +1,937 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#ifdef __AVX__
+#include "avx_activation.h"
+#include "avx_usability.h"
+#endif // NCNN_AVX2
+
+#include "lstm_x86.h"
+
+#include <math.h>
+#include "layer_type.h"
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(LSTM_x86)
+
+LSTM_x86::LSTM_x86()
+{
+    one_blob_only = false;
+    support_inplace = false;
+}
+int LSTM_x86::create_pipeline(const Option& opt)
+{
+#if __AVX__
+    if (opt.use_fp16_storage)
+    {
+        ncnn::cast_float32_to_float16(weight_xc_data, weight_xc_data_fp16, opt);
+        ncnn::cast_float32_to_float16(weight_hc_data, weight_hc_data_fp16, opt);
+    }
+#endif // __AVX__
+
+    return 0;
+}
+#ifdef __AVX__
+
+static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)
+{
+    int size = bottom_blob.w;
+    int T = bottom_blob.h;
+
+    int num_output = top_blob.w;
+    // fprintf(stderr, "bottom_blob = %d x %d x %d num_output = %d \n", bottom_blob.w,bottom_blob.h,bottom_blob.c,num_output);
+    // 4 x num_output
+    Mat gates(num_output, 4, 4u, opt.workspace_allocator);
+    if (gates.empty())
+        return -100;
+    // unroll
+    for (int t = 0; t < T; t++)
+    {
+        // clip hidden by continuation indicator
+        // h_cont_{t-1} = cont_t * h_{t-1}
+        // h_cont_{t-1} = h_{t-1} if cont_t == 1
+        //                0       otherwise
+        // calculate hidden
+        // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c
+        int ti = reverse ? T - 1 - t : t;
+        int remain_output = (num_output >> 1) << 1;
+        for (int q = 0; q + 1 < num_output; q += 2)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const unsigned short* weight_xc_I_0 = (const unsigned short*)weight_xc.row(num_output * 0 + q);
+            const unsigned short* weight_xc_F_0 = (const unsigned short*)weight_xc.row(num_output * 1 + q);
+            const unsigned short* weight_xc_O_0 = (const unsigned short*)weight_xc.row(num_output * 2 + q);
+            const unsigned short* weight_xc_G_0 = (const unsigned short*)weight_xc.row(num_output * 3 + q);
+            const unsigned short* weight_xc_I_1 = (const unsigned short*)weight_xc.row(num_output * 0 + (q + 1));
+            const unsigned short* weight_xc_F_1 = (const unsigned short*)weight_xc.row(num_output * 1 + (q + 1));
+            const unsigned short* weight_xc_O_1 = (const unsigned short*)weight_xc.row(num_output * 2 + (q + 1));
+            const unsigned short* weight_xc_G_1 = (const unsigned short*)weight_xc.row(num_output * 3 + (q + 1));
+
+            const unsigned short* weight_hc_I_0 = (const unsigned short*)weight_hc.row(num_output * 0 + q);
+            const unsigned short* weight_hc_F_0 = (const unsigned short*)weight_hc.row(num_output * 1 + q);
+            const unsigned short* weight_hc_O_0 = (const unsigned short*)weight_hc.row(num_output * 2 + q);
+            const unsigned short* weight_hc_G_0 = (const unsigned short*)weight_hc.row(num_output * 3 + q);
+            const unsigned short* weight_hc_I_1 = (const unsigned short*)weight_hc.row(num_output * 0 + (q + 1));
+            const unsigned short* weight_hc_F_1 = (const unsigned short*)weight_hc.row(num_output * 1 + (q + 1));
+            const unsigned short* weight_hc_O_1 = (const unsigned short*)weight_hc.row(num_output * 2 + (q + 1));
+            const unsigned short* weight_hc_G_1 = (const unsigned short*)weight_hc.row(num_output * 3 + (q + 1));
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            __m256 _sumI_0 = _mm256_setzero_ps();
+            __m256 _sumF_0 = _mm256_setzero_ps();
+            __m256 _sumO_0 = _mm256_setzero_ps();
+            __m256 _sumG_0 = _mm256_setzero_ps();
+            __m256 _sumI_1 = _mm256_setzero_ps();
+            __m256 _sumF_1 = _mm256_setzero_ps();
+            __m256 _sumO_1 = _mm256_setzero_ps();
+            __m256 _sumG_1 = _mm256_setzero_ps();
+            int nn_num_size = size >> 3;
+            int remain_size = size & 7;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                __m256 xi = _mm256_loadu_ps(x);
+                _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_xc_I_0), xi, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_xc_F_0), xi, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_xc_O_0), xi, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_xc_G_0), xi, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_xc_I_1), xi, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_xc_F_1), xi, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_xc_O_1), xi, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_xc_G_1), xi, _sumG_1);
+                x += 8;
+                weight_xc_I_0 += 8;
+                weight_xc_F_0 += 8;
+                weight_xc_O_0 += 8;
+                weight_xc_G_0 += 8;
+                weight_xc_I_1 += 8;
+                weight_xc_F_1 += 8;
+                weight_xc_O_1 += 8;
+                weight_xc_G_1 += 8;
+            }
+            int nn_num_output = num_output >> 3;
+            int remain_num_output = num_output & 7;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);
+
+                _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_hc_I_0), h_cont, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_hc_F_0), h_cont, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_hc_O_0), h_cont, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_hc_G_0), h_cont, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_hc_I_1), h_cont, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_hc_F_1), h_cont, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_hc_O_1), h_cont, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_hc_G_1), h_cont, _sumG_1);
+                hidden_ptr_r += 8;
+                weight_hc_I_0 += 8;
+                weight_hc_F_0 += 8;
+                weight_hc_O_0 += 8;
+                weight_hc_G_0 += 8;
+                weight_hc_I_1 += 8;
+                weight_hc_F_1 += 8;
+                weight_hc_O_1 += 8;
+                weight_hc_G_1 += 8;
+            }
+            if (remain_size != 0)
+            {
+                unsigned short fp16_weights[8][8] = {{0}};
+                float _xi_f[8] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_size; i++)
+                {
+                    _xi_f[i] = *x;
+                    fp16_weights[0][i] = *weight_xc_I_0;
+                    fp16_weights[1][i] = *weight_xc_F_0;
+                    fp16_weights[2][i] = *weight_xc_O_0;
+                    fp16_weights[3][i] = *weight_xc_G_0;
+                    fp16_weights[4][i] = *weight_xc_I_1;
+                    fp16_weights[5][i] = *weight_xc_F_1;
+                    fp16_weights[6][i] = *weight_xc_O_1;
+                    fp16_weights[7][i] = *weight_xc_G_1;
+                    x++;
+                    weight_xc_I_0++;
+                    weight_xc_F_0++;
+                    weight_xc_O_0++;
+                    weight_xc_G_0++;
+                    weight_xc_I_1++;
+                    weight_xc_F_1++;
+                    weight_xc_O_1++;
+                    weight_xc_G_1++;
+                }
+                __m256 xi = _mm256_loadu_ps(_xi_f);
+                _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), xi, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), xi, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), xi, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), xi, _sumG_1);
+            }
+            if (remain_num_output != 0)
+            {
+                unsigned short fp16_weights[8][8] = {{0}};
+                float _hcont_f[8] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_num_output; i++)
+                {
+                    _hcont_f[i] = *hidden_ptr_r;
+                    fp16_weights[0][i] = *weight_hc_I_0;
+                    fp16_weights[1][i] = *weight_hc_F_0;
+                    fp16_weights[2][i] = *weight_hc_O_0;
+                    fp16_weights[3][i] = *weight_hc_G_0;
+                    fp16_weights[4][i] = *weight_hc_I_1;
+                    fp16_weights[5][i] = *weight_hc_F_1;
+                    fp16_weights[6][i] = *weight_hc_O_1;
+                    fp16_weights[7][i] = *weight_hc_G_1;
+                    hidden_ptr_r++;
+                    weight_hc_I_0++;
+                    weight_hc_F_0++;
+                    weight_hc_O_0++;
+                    weight_hc_G_0++;
+                    weight_hc_I_1++;
+                    weight_hc_F_1++;
+                    weight_hc_O_1++;
+                    weight_hc_G_1++;
+                }
+                __m256 h_cont = _mm256_loadu_ps(_hcont_f);
+                _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), h_cont, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), h_cont, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), h_cont, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), h_cont, _sumG_1);
+            }
+            float sums[8];
+            _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1));
+            sums[0] += bias_c_I[q];
+            sums[1] += bias_c_F[q];
+            sums[2] += bias_c_O[q];
+            sums[3] += bias_c_G[q];
+            sums[4] += bias_c_I[q + 1];
+            sums[5] += bias_c_F[q + 1];
+            sums[6] += bias_c_O[q + 1];
+            sums[7] += bias_c_G[q + 1];
+            gates_data_I[q] = sums[0];
+            gates_data_F[q] = sums[1];
+            gates_data_O[q] = sums[2];
+            gates_data_G[q] = sums[3];
+            gates_data_I[q + 1] = sums[4];
+            gates_data_F[q + 1] = sums[5];
+            gates_data_O[q + 1] = sums[6];
+            gates_data_G[q + 1] = sums[7];
+        }
+
+        for (int q = remain_output; q < num_output; q++)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const unsigned short* weight_xc_I = (const unsigned short*)weight_xc.row(num_output * 0 + q);
+            const unsigned short* weight_xc_F = (const unsigned short*)weight_xc.row(num_output * 1 + q);
+            const unsigned short* weight_xc_O = (const unsigned short*)weight_xc.row(num_output * 2 + q);
+            const unsigned short* weight_xc_G = (const unsigned short*)weight_xc.row(num_output * 3 + q);
+
+            const unsigned short* weight_hc_I = (const unsigned short*)weight_hc.row(num_output * 0 + q);
+            const unsigned short* weight_hc_F = (const unsigned short*)weight_hc.row(num_output * 1 + q);
+            const unsigned short* weight_hc_O = (const unsigned short*)weight_hc.row(num_output * 2 + q);
+            const unsigned short* weight_hc_G = (const unsigned short*)weight_hc.row(num_output * 3 + q);
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            __m256 _sumI = _mm256_setzero_ps();
+            __m256 _sumF = _mm256_setzero_ps();
+            __m256 _sumO = _mm256_setzero_ps();
+            __m256 _sumG = _mm256_setzero_ps();
+            int nn_num_size = size >> 3;
+            int remain_size = size & 7;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                __m256 xi = _mm256_loadu_ps(x);
+                _sumI = _mm256_fmadd_ps(loadfp16(weight_xc_I), xi, _sumI);
+                _sumF = _mm256_fmadd_ps(loadfp16(weight_xc_F), xi, _sumF);
+                _sumO = _mm256_fmadd_ps(loadfp16(weight_xc_O), xi, _sumO);
+                _sumG = _mm256_fmadd_ps(loadfp16(weight_xc_G), xi, _sumG);
+                x += 8;
+                weight_xc_I += 8;
+                weight_xc_F += 8;
+                weight_xc_O += 8;
+                weight_xc_G += 8;
+            }
+            int nn_num_output = num_output >> 3;
+            int remain_num_output = num_output & 7;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);
+
+                _sumI = _mm256_fmadd_ps(loadfp16(weight_hc_I), h_cont, _sumI);
+                _sumF = _mm256_fmadd_ps(loadfp16(weight_hc_F), h_cont, _sumF);
+                _sumO = _mm256_fmadd_ps(loadfp16(weight_hc_O), h_cont, _sumO);
+                _sumG = _mm256_fmadd_ps(loadfp16(weight_hc_G), h_cont, _sumG);
+                hidden_ptr_r += 8;
+                weight_hc_I += 8;
+                weight_hc_F += 8;
+                weight_hc_O += 8;
+                weight_hc_G += 8;
+            }
+            if (remain_size != 0)
+            {
+                unsigned short fp16_weights[4][8] = {{0}};
+                float _xi_f[8] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_size; i++)
+                {
+                    _xi_f[i] = *x;
+                    fp16_weights[0][i] = *weight_xc_I;
+                    fp16_weights[1][i] = *weight_xc_F;
+                    fp16_weights[2][i] = *weight_xc_O;
+                    fp16_weights[3][i] = *weight_xc_G;
+                    x++;
+                    weight_xc_I++;
+                    weight_xc_F++;
+                    weight_xc_O++;
+                    weight_xc_G++;
+                }
+                __m256 xi = _mm256_loadu_ps(_xi_f);
+                _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI);
+                _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF);
+                _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO);
+                _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG);
+            }
+            if (remain_num_output != 0)
+            {
+                unsigned short fp16_weights[4][8] = {{0}};
+                float _hcont_f[8] = {0};
+                // No fast way to convert to fp32 one element at the time
+                // so batch an 8 lane vector.
+                for (int i = 0; i < remain_num_output; i++)
+                {
+                    _hcont_f[i] = *hidden_ptr_r;
+                    fp16_weights[0][i] = *weight_hc_I;
+                    fp16_weights[1][i] = *weight_hc_F;
+                    fp16_weights[2][i] = *weight_hc_O;
+                    fp16_weights[3][i] = *weight_hc_G;
+                    hidden_ptr_r++;
+                    weight_hc_I++;
+                    weight_hc_F++;
+                    weight_hc_O++;
+                    weight_hc_G++;
+                }
+                __m256 h_cont = _mm256_loadu_ps(_hcont_f);
+                _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI);
+                _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF);
+                _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO);
+                _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG);
+            }
+
+            float sums[4];
+            _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG));
+            sums[0] += bias_c_I[q];
+            sums[1] += bias_c_F[q];
+            sums[2] += bias_c_O[q];
+            sums[3] += bias_c_G[q];
+            gates_data_I[q] = sums[0];
+            gates_data_F[q] = sums[1];
+            gates_data_O[q] = sums[2];
+            gates_data_G[q] = sums[3];
+        }
+
+        // lstm unit
+        // sigmoid(I)
+        // sigmoid(F)
+        // sigmoid(O)
+        // tanh(G)
+        // c_t := f_t .* c_{t-1} + i_t .* g_t
+        // h_t := o_t .* tanh[c_t]
+        float* output_data = top_blob.row(ti);
+        float* cell_ptr = cell_state;
+        float* hidden_ptr = hidden_state;
+        const float* gates_data_I = gates.row(0);
+        const float* gates_data_F = gates.row(1);
+        const float* gates_data_O = gates.row(2);
+        const float* gates_data_G = gates.row(3);
+        int nn_activation = num_output >> 3;
+        int remain_activations = num_output & 7;
+        for (; nn_activation > 0; nn_activation--)
+        {
+            __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I));
+            __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F));
+            __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O));
+            __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G));
+            __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G));
+            __m256 H = _mm256_mul_ps(O, tanh_avx(cell2));
+            _mm256_storeu_ps(cell_ptr, cell2);
+            _mm256_storeu_ps(hidden_ptr, H);
+            _mm256_storeu_ps(output_data, H);
+            cell_ptr += 8;
+            output_data += 8;
+            hidden_ptr += 8;
+            gates_data_I += 8;
+            gates_data_F += 8;
+            gates_data_O += 8;
+            gates_data_G += 8;
+        }
+        for (; remain_activations > 0; remain_activations--)
+        {
+            float I = *gates_data_I;
+            float F = *gates_data_F;
+            float O = *gates_data_O;
+            float G = *gates_data_G;
+
+            I = 1.f / (1.f + exp(-I));
+            F = 1.f / (1.f + exp(-F));
+            O = 1.f / (1.f + exp(-O));
+            G = tanh(G);
+            float cell2 = F * *cell_ptr + I * G;
+            float H = O * tanh(cell2);
+            *cell_ptr = cell2;
+            *hidden_ptr = H;
+            *output_data = H;
+            cell_ptr++;
+            output_data++;
+            hidden_ptr++;
+            gates_data_I++;
+            gates_data_F++;
+            gates_data_O++;
+            gates_data_G++;
+        }
+
+        // no cell output here
+    }
+
+    return 0;
+}
+
+static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)
+{
+    int size = bottom_blob.w;
+    int T = bottom_blob.h;
+
+    int num_output = top_blob.w;
+
+    // 4 x num_output
+    Mat gates(num_output, 4, 4u, opt.workspace_allocator);
+    if (gates.empty())
+        return -100;
+
+    // unroll
+    for (int t = 0; t < T; t++)
+    {
+        // clip hidden by continuation indicator
+        // h_cont_{t-1} = cont_t * h_{t-1}
+        // h_cont_{t-1} = h_{t-1} if cont_t == 1
+        //                0       otherwise
+        // calculate hidden
+        // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c
+
+        int ti = reverse ? T - 1 - t : t;
+        int remain_output = (num_output >> 1) << 1;
+        for (int q = 0; q + 1 < num_output; q += 2)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const float* weight_xc_I_0 = weight_xc.row(num_output * 0 + q);
+            const float* weight_xc_F_0 = weight_xc.row(num_output * 1 + q);
+            const float* weight_xc_O_0 = weight_xc.row(num_output * 2 + q);
+            const float* weight_xc_G_0 = weight_xc.row(num_output * 3 + q);
+            const float* weight_xc_I_1 = weight_xc.row(num_output * 0 + (q + 1));
+            const float* weight_xc_F_1 = weight_xc.row(num_output * 1 + (q + 1));
+            const float* weight_xc_O_1 = weight_xc.row(num_output * 2 + (q + 1));
+            const float* weight_xc_G_1 = weight_xc.row(num_output * 3 + (q + 1));
+
+            const float* weight_hc_I_0 = weight_hc.row(num_output * 0 + q);
+            const float* weight_hc_F_0 = weight_hc.row(num_output * 1 + q);
+            const float* weight_hc_O_0 = weight_hc.row(num_output * 2 + q);
+            const float* weight_hc_G_0 = weight_hc.row(num_output * 3 + q);
+            const float* weight_hc_I_1 = weight_hc.row(num_output * 0 + (q + 1));
+            const float* weight_hc_F_1 = weight_hc.row(num_output * 1 + (q + 1));
+            const float* weight_hc_O_1 = weight_hc.row(num_output * 2 + (q + 1));
+            const float* weight_hc_G_1 = weight_hc.row(num_output * 3 + (q + 1));
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            __m256 _sumI_0 = _mm256_setzero_ps();
+            __m256 _sumF_0 = _mm256_setzero_ps();
+            __m256 _sumO_0 = _mm256_setzero_ps();
+            __m256 _sumG_0 = _mm256_setzero_ps();
+            __m256 _sumI_1 = _mm256_setzero_ps();
+            __m256 _sumF_1 = _mm256_setzero_ps();
+            __m256 _sumO_1 = _mm256_setzero_ps();
+            __m256 _sumG_1 = _mm256_setzero_ps();
+            int nn_num_size = size >> 3;
+            int remain_size = size & 7;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                __m256 xi = _mm256_loadu_ps(x);
+                _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_0), xi, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_0), xi, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_0), xi, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_0), xi, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_1), xi, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_1), xi, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_1), xi, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_1), xi, _sumG_1);
+                x += 8;
+                weight_xc_I_0 += 8;
+                weight_xc_F_0 += 8;
+                weight_xc_O_0 += 8;
+                weight_xc_G_0 += 8;
+                weight_xc_I_1 += 8;
+                weight_xc_F_1 += 8;
+                weight_xc_O_1 += 8;
+                weight_xc_G_1 += 8;
+            }
+            int nn_num_output = num_output >> 3;
+            int remain_num_output = num_output & 7;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);
+
+                _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_0), h_cont, _sumI_0);
+                _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_0), h_cont, _sumF_0);
+                _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_0), h_cont, _sumO_0);
+                _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_0), h_cont, _sumG_0);
+                _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_1), h_cont, _sumI_1);
+                _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_1), h_cont, _sumF_1);
+                _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_1), h_cont, _sumO_1);
+                _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_1), h_cont, _sumG_1);
+                hidden_ptr_r += 8;
+                weight_hc_I_0 += 8;
+                weight_hc_F_0 += 8;
+                weight_hc_O_0 += 8;
+                weight_hc_G_0 += 8;
+                weight_hc_I_1 += 8;
+                weight_hc_F_1 += 8;
+                weight_hc_O_1 += 8;
+                weight_hc_G_1 += 8;
+            }
+            float sums[8];
+            _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1));
+            sums[0] += bias_c_I[q];
+            sums[1] += bias_c_F[q];
+            sums[2] += bias_c_O[q];
+            sums[3] += bias_c_G[q];
+            sums[4] += bias_c_I[q + 1];
+            sums[5] += bias_c_F[q + 1];
+            sums[6] += bias_c_O[q + 1];
+            sums[7] += bias_c_G[q + 1];
+
+            for (; remain_size > 0; remain_size--)
+            {
+                float xi = *x;
+                sums[0] += *weight_xc_I_0 * xi;
+                sums[1] += *weight_xc_F_0 * xi;
+                sums[2] += *weight_xc_O_0 * xi;
+                sums[3] += *weight_xc_G_0 * xi;
+                sums[4] += *weight_xc_I_1 * xi;
+                sums[5] += *weight_xc_F_1 * xi;
+                sums[6] += *weight_xc_O_1 * xi;
+                sums[7] += *weight_xc_G_1 * xi;
+                x++;
+                weight_xc_I_0++;
+                weight_xc_F_0++;
+                weight_xc_O_0++;
+                weight_xc_G_0++;
+                weight_xc_I_1++;
+                weight_xc_F_1++;
+                weight_xc_O_1++;
+                weight_xc_G_1++;
+            }
+
+            for (; remain_num_output > 0; remain_num_output--)
+            {
+                float h_cont = *hidden_ptr_r;
+                sums[0] += *weight_hc_I_0 * h_cont;
+                sums[1] += *weight_hc_F_0 * h_cont;
+                sums[2] += *weight_hc_O_0 * h_cont;
+                sums[3] += *weight_hc_G_0 * h_cont;
+                sums[4] += *weight_hc_I_1 * h_cont;
+                sums[5] += *weight_hc_F_1 * h_cont;
+                sums[6] += *weight_hc_O_1 * h_cont;
+                sums[7] += *weight_hc_G_1 * h_cont;
+                hidden_ptr_r++;
+                weight_hc_I_0++;
+                weight_hc_F_0++;
+                weight_hc_O_0++;
+                weight_hc_G_0++;
+                weight_hc_I_1++;
+                weight_hc_F_1++;
+                weight_hc_O_1++;
+                weight_hc_G_1++;
+            }
+            gates_data_I[q] = sums[0];
+            gates_data_F[q] = sums[1];
+            gates_data_O[q] = sums[2];
+            gates_data_G[q] = sums[3];
+            gates_data_I[q + 1] = sums[4];
+            gates_data_F[q + 1] = sums[5];
+            gates_data_O[q + 1] = sums[6];
+            gates_data_G[q + 1] = sums[7];
+        }
+
+        for (int q = remain_output; q < num_output; q++)
+        {
+            const float* x = bottom_blob.row(ti);
+            const float* hidden_ptr_r = hidden_state;
+            const float* bias_c_I = bias_c.row(0);
+            const float* bias_c_F = bias_c.row(1);
+            const float* bias_c_O = bias_c.row(2);
+            const float* bias_c_G = bias_c.row(3);
+
+            float* gates_data_I = gates.row(0);
+            float* gates_data_F = gates.row(1);
+            float* gates_data_O = gates.row(2);
+            float* gates_data_G = gates.row(3);
+            // gate I F O G
+            const float* weight_xc_I = weight_xc.row(num_output * 0 + q);
+            const float* weight_xc_F = weight_xc.row(num_output * 1 + q);
+            const float* weight_xc_O = weight_xc.row(num_output * 2 + q);
+            const float* weight_xc_G = weight_xc.row(num_output * 3 + q);
+
+            const float* weight_hc_I = weight_hc.row(num_output * 0 + q);
+            const float* weight_hc_F = weight_hc.row(num_output * 1 + q);
+            const float* weight_hc_O = weight_hc.row(num_output * 2 + q);
+            const float* weight_hc_G = weight_hc.row(num_output * 3 + q);
+
+            // float I = bias_c_I[q];
+            // float F = bias_c_F[q];
+            // float O = bias_c_O[q];
+            // float G = bias_c_G[q];
+            __m256 _sumI = _mm256_setzero_ps();
+            __m256 _sumF = _mm256_setzero_ps();
+            __m256 _sumO = _mm256_setzero_ps();
+            __m256 _sumG = _mm256_setzero_ps();
+            int nn_num_size = size >> 3;
+            int remain_size = size & 7;
+            for (; nn_num_size > 0; nn_num_size--)
+            {
+                __m256 xi = _mm256_loadu_ps(x);
+                _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I), xi, _sumI);
+                _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F), xi, _sumF);
+                _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O), xi, _sumO);
+                _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G), xi, _sumG);
+                x += 8;
+                weight_xc_I += 8;
+                weight_xc_F += 8;
+                weight_xc_O += 8;
+                weight_xc_G += 8;
+            }
+            int nn_num_output = num_output >> 3;
+            int remain_num_output = num_output & 7;
+            for (; nn_num_output > 0; nn_num_output--)
+            {
+                __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);
+
+                _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I), h_cont, _sumI);
+                _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F), h_cont, _sumF);
+                _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O), h_cont, _sumO);
+                _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G), h_cont, _sumG);
+                hidden_ptr_r += 8;
+                weight_hc_I += 8;
+                weight_hc_F += 8;
+                weight_hc_O += 8;
+                weight_hc_G += 8;
+            }
+            float sums[4];
+            _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG));
+            sums[0] += bias_c_I[q];
+            sums[1] += bias_c_F[q];
+            sums[2] += bias_c_O[q];
+            sums[3] += bias_c_G[q];
+
+            for (; remain_size > 0; remain_size--)
+            {
+                float xi = *x;
+                sums[0] += *weight_xc_I * xi;
+                sums[1] += *weight_xc_F * xi;
+                sums[2] += *weight_xc_O * xi;
+                sums[3] += *weight_xc_G * xi;
+                x++;
+                weight_xc_I++;
+                weight_xc_F++;
+                weight_xc_O++;
+                weight_xc_G++;
+            }
+
+            for (; remain_num_output > 0; remain_num_output--)
+            {
+                float h_cont = *hidden_ptr_r;
+                sums[0] += *weight_hc_I * h_cont;
+                sums[1] += *weight_hc_F * h_cont;
+                sums[2] += *weight_hc_O * h_cont;
+                sums[3] += *weight_hc_G * h_cont;
+                hidden_ptr_r++;
+                weight_hc_I++;
+                weight_hc_F++;
+                weight_hc_O++;
+                weight_hc_G++;
+            }
+            gates_data_I[q] = sums[0];
+            gates_data_F[q] = sums[1];
+            gates_data_O[q] = sums[2];
+            gates_data_G[q] = sums[3];
+        }
+
+        // lstm unit
+        // sigmoid(I)
+        // sigmoid(F)
+        // sigmoid(O)
+        // tanh(G)
+        // c_t := f_t .* c_{t-1} + i_t .* g_t
+        // h_t := o_t .* tanh[c_t]
+        float* output_data = top_blob.row(ti);
+        float* cell_ptr = cell_state;
+        float* hidden_ptr = hidden_state;
+        const float* gates_data_I = gates.row(0);
+        const float* gates_data_F = gates.row(1);
+        const float* gates_data_O = gates.row(2);
+        const float* gates_data_G = gates.row(3);
+        int nn_activation = num_output >> 3;
+        int remain_activations = num_output & 7;
+        for (; nn_activation > 0; nn_activation--)
+        {
+            __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I));
+            __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F));
+            __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O));
+            __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G));
+            __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G));
+            __m256 H = _mm256_mul_ps(O, tanh_avx(cell2));
+            _mm256_storeu_ps(cell_ptr, cell2);
+            _mm256_storeu_ps(hidden_ptr, H);
+            _mm256_storeu_ps(output_data, H);
+            cell_ptr += 8;
+            output_data += 8;
+            hidden_ptr += 8;
+            gates_data_I += 8;
+            gates_data_F += 8;
+            gates_data_O += 8;
+            gates_data_G += 8;
+        }
+        for (; remain_activations > 0; remain_activations--)
+        {
+            float I = *gates_data_I;
+            float F = *gates_data_F;
+            float O = *gates_data_O;
+            float G = *gates_data_G;
+
+            I = 1.f / (1.f + exp(-I));
+            F = 1.f / (1.f + exp(-F));
+            O = 1.f / (1.f + exp(-O));
+            G = tanh(G);
+            float cell2 = F * *cell_ptr + I * G;
+            float H = O * tanh(cell2);
+            *cell_ptr = cell2;
+            *hidden_ptr = H;
+            *output_data = H;
+            cell_ptr++;
+            output_data++;
+            hidden_ptr++;
+            gates_data_I++;
+            gates_data_F++;
+            gates_data_O++;
+            gates_data_G++;
+        }
+
+        // no cell output here
+    }
+
+    return 0;
+}
+#endif
+
+int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+#if __AVX__
+    int T = bottom_blob.h;
+    int num_directions = direction == 2 ? 2 : 1;
+
+    // initial hidden state
+    Mat hidden(num_output, 4u, opt.workspace_allocator);
+    if (hidden.empty())
+        return -100;
+    hidden.fill(0.f);
+    // internal cell state
+    Mat cell(num_output, 4u, opt.workspace_allocator);
+    if (cell.empty())
+        return -100;
+    cell.fill(0.f);
+
+    top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // Uni directional
+    if (direction == 0 || direction == 1)
+    {
+        if (opt.use_fp16_storage)
+        {
+            // Uni directional
+            int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);
+            if (ret != 0)
+                return ret;
+        }
+        else
+        {
+            // Uni directional
+            int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
+            if (ret != 0)
+                return ret;
+        }
+    }
+
+    if (direction == 2)
+    {
+        Mat top_blob_forward(num_output, T, 4u, opt.workspace_allocator);
+        if (top_blob_forward.empty())
+            return -100;
+
+        Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator);
+        if (top_blob_reverse.empty())
+            return -100;
+
+        if (opt.use_fp16_storage)
+        {
+            // Uni directional
+            int ret0 = lstm_fp16(bottom_blob, top_blob_forward, 0, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);
+            if (ret0 != 0)
+                return ret0;
+        }
+        else
+        {
+            // Uni directional
+            int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);
+            if (ret0 != 0)
+                return ret0;
+        }
+
+        hidden.fill(0.0f);
+        cell.fill(0.0f);
+        if (opt.use_fp16_storage)
+        {
+            // Uni directional
+            int ret1 = lstm_fp16(bottom_blob, top_blob_reverse, 1, weight_xc_data_fp16.channel(1), bias_c_data.channel(1), weight_hc_data_fp16.channel(1), hidden, cell, opt);
+            if (ret1 != 0)
+                return ret1;
+        }
+        else
+        {
+            // Uni directional
+            int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt);
+            if (ret1 != 0)
+                return ret1;
+        }
+
+        // concat w
+        for (int i = 0; i < T; i++)
+        {
+            const float* pf = top_blob_forward.row(i);
+            const float* pr = top_blob_reverse.row(i);
+            float* ptr = top_blob.row(i);
+
+            memcpy(ptr, pf, num_output * sizeof(float));
+            memcpy(ptr + num_output, pr, num_output * sizeof(float));
+        }
+    }
+
+    return 0;
+#else
+    return LSTM::forward(bottom_blob, top_blob, opt);
+#endif
+}
+
+int LSTM_x86::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
+{
+#if __AVX__
+    if (bottom_blobs.size() != 3 || top_blobs.size() != 3)
+    {
+        return forward(bottom_blobs[0], top_blobs[0], opt);
+    }
+    const Mat& bottom_blob = bottom_blobs[0];
+
+    int T = bottom_blob.h;
+    Mat& top_blob = top_blobs[0];
+    Mat& hidden_state = top_blobs[1];
+    Mat& cell_state = top_blobs[2];
+
+    //Copy previous states
+    hidden_state = bottom_blobs[1].clone(opt.blob_allocator);
+    cell_state = bottom_blobs[2].clone(opt.blob_allocator);
+
+    top_blob.create(num_output, T, 4u, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    if (opt.use_fp16_storage)
+    {
+        // Uni directional
+        int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden_state, cell_state, opt);
+        if (ret != 0)
+            return ret;
+    }
+    else
+    {
+        // Uni directional
+        int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt);
+        if (ret != 0)
+            return ret;
+    }
+    return 0;
+#else
+    return LSTM::forward(bottom_blobs, top_blobs, opt);
+#endif
+}
+
+} // namespace ncnn

src/layer/x86/lstm_x86.h

@@ -0,0 +1,40 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_LSTM_X86_H
+#define LAYER_LSTM_X86_H
+
+#include "lstm.h"
+
+namespace ncnn {
+
+class LSTM_x86 : virtual public LSTM
+{
+public:
+    LSTM_x86();
+
+    virtual int create_pipeline(const Option& opt);
+
+    virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+
+    virtual int forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const;
+
+public:
+    Mat weight_hc_data_fp16;
+    Mat weight_xc_data_fp16;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_LSTM_X86_H

src/layer/x86/scale_x86.cpp

@@ -0,0 +1,261 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#if __AVX__
+#include <immintrin.h>
+#endif // __AVX__
+
+#include "scale_x86.h"
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(Scale_x86)
+
+Scale_x86::Scale_x86()
+{
+#if __AVX__
+    support_packing = true;
+#endif // __AVX__
+}
+
+int Scale_x86::forward_inplace(std::vector<Mat>& bottom_top_blobs, const Option& opt) const
+{
+    Mat& bottom_top_blob = bottom_top_blobs[0];
+    const Mat& scale_blob = bottom_top_blobs[1];
+
+    int dims = bottom_top_blob.dims;
+    int elempack = bottom_top_blob.elempack;
+
+#if __AVX__
+    if (elempack == 8)
+    {
+        if (dims == 1)
+        {
+            int w = bottom_top_blob.w;
+
+            const float* scale = scale_blob;
+            if (bias_term)
+            {
+                const float* bias = bias_data;
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int i = 0; i < w; i++)
+                {
+                    float* ptr = (float*)bottom_top_blob + i * 8;
+
+                    __m256 _p = _mm256_loadu_ps(ptr);
+                    __m256 _s = _mm256_loadu_ps(scale + i * 8);
+                    __m256 _bias = _mm256_loadu_ps(bias + i * 8);
+                    _p = _mm256_fmadd_ps(_p, _s, _bias);
+                    _mm256_storeu_ps(ptr, _p);
+                }
+            }
+            else
+            {
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int i = 0; i < w; i++)
+                {
+                    float* ptr = (float*)bottom_top_blob + i * 8;
+
+                    __m256 _p = _mm256_loadu_ps(ptr);
+                    __m256 _s = _mm256_loadu_ps(scale + i * 8);
+                    _p = _mm256_mul_ps(_p, _s);
+                    _mm256_storeu_ps(ptr, _p);
+                }
+            }
+        }
+
+        if (dims == 2)
+        {
+            int w = bottom_top_blob.w;
+            int h = bottom_top_blob.h;
+
+            if (bias_term)
+            {
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int i = 0; i < h; i++)
+                {
+                    float* ptr = bottom_top_blob.row(i);
+                    __m256 _s = _mm256_loadu_ps((const float*)scale_blob + i * 8);
+                    __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8);
+
+                    for (int j = 0; j < w; j++)
+                    {
+                        __m256 _p = _mm256_loadu_ps(ptr);
+                        _p = _mm256_fmadd_ps(_p, _s, _bias);
+                        _mm256_storeu_ps(ptr, _p);
+
+                        ptr += 8;
+                    }
+                }
+            }
+            else
+            {
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int i = 0; i < h; i++)
+                {
+                    float* ptr = bottom_top_blob.row(i);
+                    __m256 _s = _mm256_loadu_ps((const float*)scale_blob + i * 8);
+
+                    for (int j = 0; j < w; j++)
+                    {
+                        __m256 _p = _mm256_loadu_ps(ptr);
+                        _p = _mm256_mul_ps(_p, _s);
+                        _mm256_storeu_ps(ptr, _p);
+
+                        ptr += 8;
+                    }
+                }
+            }
+        }
+
+        if (dims == 3)
+        {
+            int w = bottom_top_blob.w;
+            int h = bottom_top_blob.h;
+            int channels = bottom_top_blob.c;
+            int size = w * h;
+
+            if (bias_term)
+            {
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int q = 0; q < channels; q++)
+                {
+                    float* ptr = bottom_top_blob.channel(q);
+                    __m256 _s = _mm256_loadu_ps((const float*)scale_blob + q * 8);
+                    __m256 _bias = _mm256_loadu_ps((const float*)bias_data + q * 8);
+
+                    for (int i = 0; i < size; i++)
+                    {
+                        __m256 _p = _mm256_loadu_ps(ptr);
+                        _p = _mm256_fmadd_ps(_p, _s, _bias);
+                        _mm256_storeu_ps(ptr, _p);
+
+                        ptr += 8;
+                    }
+                }
+            }
+            else
+            {
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int q = 0; q < channels; q++)
+                {
+                    float* ptr = bottom_top_blob.channel(q);
+                    __m256 _s = _mm256_loadu_ps((const float*)scale_blob + q * 8);
+
+                    for (int i = 0; i < size; i++)
+                    {
+                        __m256 _p = _mm256_loadu_ps(ptr);
+                        _p = _mm256_mul_ps(_p, _s);
+                        _mm256_storeu_ps(ptr, _p);
+
+                        ptr += 8;
+                    }
+                }
+            }
+        }
+
+        return 0;
+    }
+#endif // __AVX__
+
+    if (dims != 3)
+        return Scale::forward_inplace(bottom_top_blobs, opt);
+
+    int w = bottom_top_blob.w;
+    int h = bottom_top_blob.h;
+    int channels = bottom_top_blob.c;
+    int size = w * h;
+
+    if (bias_term)
+    {
+        const float* scale_ptr = scale_blob;
+        const float* bias_ptr = bias_data;
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int q = 0; q < channels; q++)
+        {
+            float* ptr = bottom_top_blob.channel(q);
+
+            float s = scale_ptr[q];
+            float bias = bias_ptr[q];
+
+#if __AVX__
+            int nn = size >> 3;
+            int remain = size & 7;
+#else
+            int remain = size;
+#endif // __AVX__
+
+#if __AVX__
+            __m256 _s = _mm256_set1_ps(s);
+            __m256 _bias = _mm256_set1_ps(bias);
+            for (; nn > 0; nn--)
+            {
+                __m256 _p = _mm256_loadu_ps(ptr);
+                _p = _mm256_fmadd_ps(_p, _s, _bias);
+                _mm256_storeu_ps(ptr, _p);
+
+                ptr += 8;
+            }
+#endif // __AVX__
+
+            for (; remain > 0; remain--)
+            {
+                *ptr = *ptr * s + bias;
+
+                ptr++;
+            }
+        }
+    }
+    else
+    {
+        const float* scale_ptr = scale_blob;
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int q = 0; q < channels; q++)
+        {
+            float* ptr = bottom_top_blob.channel(q);
+
+            float s = scale_ptr[q];
+
+#if __AVX__
+            int nn = size >> 3;
+            int remain = size & 7;
+#else
+            int remain = size;
+#endif // __AVX__
+
+#if __AVX__
+            __m256 _s = _mm256_set1_ps(s);
+            for (; nn > 0; nn--)
+            {
+                __m256 _p = _mm256_loadu_ps(ptr);
+                _p = _mm256_mul_ps(_p, _s);
+                _mm256_storeu_ps(ptr, _p);
+
+                ptr += 8;
+            }
+#endif // __AVX__
+
+            for (; remain > 0; remain--)
+            {
+                *ptr *= s;
+
+                ptr++;
+            }
+        }
+    }
+
+    return 0;
+}
+
+} // namespace ncnn

src/layer/x86/scale_x86.h

@@ -0,0 +1,32 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_SCALE_X86_H
+#define LAYER_SCALE_X86_H
+
+#include "scale.h"
+
+namespace ncnn {
+
+class Scale_x86 : virtual public Scale
+{
+public:
+    Scale_x86();
+
+    virtual int forward_inplace(std::vector<Mat>& bottom_top_blobs, const Option& opt) const;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_SCALE_X86_H

src/layer/x86/sigmoid_x86.cpp

@@ -67,7 +67,7 @@ int Sigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const
         int remain = size & 7;
 #else
         int remain = size;
-#endif // __ARM_NEON
+#endif // __AVX__
 
 #if __AVX__
         for (; nn > 0; nn--)
@@ -76,7 +76,7 @@ int Sigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const
             _mm256_storeu_ps(ptr, sigmoid_avx(_p));
             ptr += 8;
         }
-#endif // __ARM_NEON
+#endif // __AVX__
         for (; remain > 0; remain--)
         {
             *ptr = 1.f / (1.f + exp(-*ptr));

src/layer/x86/slice_x86.cpp

@@ -0,0 +1,438 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#include <algorithm>
+
+#if __AVX__
+#include <immintrin.h>
+#endif // __AVX__
+
+#include "slice_x86.h"
+
+#include "layer_type.h"
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(Slice_x86)
+
+Slice_x86::Slice_x86()
+{
+#if __AVX__
+    support_packing = true;
+
+    packing_pack1 = 0;
+#endif // __AVX__
+}
+
+int Slice_x86::create_pipeline(const Option& opt)
+{
+#if __AVX__
+    if (opt.use_packing_layout)
+    {
+        packing_pack1 = ncnn::create_layer(ncnn::LayerType::Packing);
+
+        ncnn::ParamDict pd;
+        pd.set(0, 1);
+
+        packing_pack1->load_param(pd);
+
+        packing_pack1->create_pipeline(opt);
+    }
+#endif // __AVX__
+
+    return 0;
+}
+
+int Slice_x86::destroy_pipeline(const Option& opt)
+{
+#if __AVX__
+    if (opt.use_packing_layout)
+    {
+        if (packing_pack1)
+        {
+            packing_pack1->destroy_pipeline(opt);
+            delete packing_pack1;
+            packing_pack1 = 0;
+        }
+    }
+#endif // __AVX__
+
+    return 0;
+}
+
+int Slice_x86::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
+{
+    const Mat& bottom_blob = bottom_blobs[0];
+    int dims = bottom_blob.dims;
+    size_t elemsize = bottom_blob.elemsize;
+    int elempack = bottom_blob.elempack;
+    const int* slices_ptr = slices;
+
+#if __AVX__
+    if (opt.use_packing_layout)
+    {
+        if (dims == 1) // axis == 0
+        {
+            // slice vector
+            int w = bottom_blob.w * elempack;
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (w - q) / (top_blobs.size() - i);
+                }
+
+                int out_elempack = slice % 8 == 0 ? 8 : 1;
+                size_t out_elemsize = elemsize / elempack * out_elempack;
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                const float* ptr = (const float*)bottom_blob + q;
+                float* outptr = top_blob;
+                memcpy(outptr, ptr, top_blob.w * top_blob.elemsize);
+
+                q += slice;
+            }
+
+            return 0;
+        }
+
+        if (dims == 2 && axis == 0)
+        {
+            // slice image height
+            int w = bottom_blob.w;
+            int h = bottom_blob.h * elempack;
+
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (h - q) / (top_blobs.size() - i);
+                }
+
+                int out_elempack = slice % 8 == 0 ? 8 : 1;
+                size_t out_elemsize = elemsize / elempack * out_elempack;
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(w, slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                q += slice;
+            }
+
+            size_t out_elemsize = top_blobs[0].elemsize;
+            int out_elempack = top_blobs[0].elempack;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                out_elemsize = std::min(out_elemsize, top_blobs[i].elemsize);
+                out_elempack = std::min(out_elempack, top_blobs[i].elempack);
+            }
+
+            Mat bottom_blob_unpacked = bottom_blob;
+            if (elempack == 8 && out_elempack == 1)
+            {
+                packing_pack1->forward(bottom_blob, bottom_blob_unpacked, opt);
+            }
+
+            const float* ptr = bottom_blob_unpacked;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                Mat& top_blob = top_blobs[i];
+
+                if (out_elempack == 1 && top_blob.elempack == 8)
+                {
+                    for (int j = 0; j < top_blob.h; j++)
+                    {
+                        const float* r0 = ptr;
+                        const float* r1 = ptr + w;
+                        const float* r2 = ptr + w * 2;
+                        const float* r3 = ptr + w * 3;
+                        const float* r4 = ptr + w * 4;
+                        const float* r5 = ptr + w * 5;
+                        const float* r6 = ptr + w * 6;
+                        const float* r7 = ptr + w * 7;
+
+                        float* outptr0 = top_blob.row(j);
+
+                        for (int j = 0; j < w; j++)
+                        {
+                            outptr0[0] = *r0++;
+                            outptr0[1] = *r1++;
+                            outptr0[2] = *r2++;
+                            outptr0[3] = *r3++;
+                            outptr0[4] = *r4++;
+                            outptr0[5] = *r5++;
+                            outptr0[6] = *r6++;
+                            outptr0[7] = *r7++;
+
+                            outptr0 += 8;
+                        }
+
+                        ptr += w * 8;
+                    }
+                }
+                else // if (out_elempack == 1 && top_blob.elempack == 1) if (out_elempack == 8 && top_blob.elempack == 8)
+                {
+                    int size = w * top_blob.h;
+
+                    float* outptr = top_blob;
+                    memcpy(outptr, ptr, size * top_blob.elemsize);
+
+                    ptr += size * top_blob.elempack;
+                }
+            }
+
+            return 0;
+        }
+
+        if (dims == 2 && axis == 1)
+        {
+            // slice image width
+            int w = bottom_blob.w;
+            int h = bottom_blob.h;
+
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (w - q) / (top_blobs.size() - i);
+                }
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(slice, h, elemsize, elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                q += slice;
+            }
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int j = 0; j < h; j++)
+            {
+                const float* ptr = bottom_blob.row(j);
+                for (size_t i = 0; i < top_blobs.size(); i++)
+                {
+                    Mat& top_blob = top_blobs[i];
+
+                    float* outptr = top_blob.row(j);
+                    memcpy(outptr, ptr, top_blob.w * elemsize);
+
+                    ptr += top_blob.w * elempack;
+                }
+            }
+
+            return 0;
+        }
+
+        if (dims == 3 && axis == 0)
+        {
+            // slice dim channel
+            int w = bottom_blob.w;
+            int h = bottom_blob.h;
+            int channels = bottom_blob.c * elempack;
+
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (channels - q) / (top_blobs.size() - i);
+                }
+
+                int out_elempack = slice % 8 == 0 ? 8 : 1;
+                size_t out_elemsize = elemsize / elempack * out_elempack;
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(w, h, slice / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                q += slice;
+            }
+
+            size_t out_elemsize = top_blobs[0].elemsize;
+            int out_elempack = top_blobs[0].elempack;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                out_elemsize = std::min(out_elemsize, top_blobs[i].elemsize);
+                out_elempack = std::min(out_elempack, top_blobs[i].elempack);
+            }
+
+            Mat bottom_blob_unpacked = bottom_blob;
+            if (elempack == 8 && out_elempack == 1)
+            {
+                packing_pack1->forward(bottom_blob, bottom_blob_unpacked, opt);
+            }
+
+            int p = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                Mat& top_blob = top_blobs[i];
+
+                if (out_elempack == 1 && top_blob.elempack == 8)
+                {
+                    int size = top_blob.w * top_blob.h;
+
+                    for (int q = 0; q < top_blob.c; q++)
+                    {
+                        const float* r0 = bottom_blob_unpacked.channel(p);
+                        const float* r1 = bottom_blob_unpacked.channel(p + 1);
+                        const float* r2 = bottom_blob_unpacked.channel(p + 2);
+                        const float* r3 = bottom_blob_unpacked.channel(p + 3);
+                        const float* r4 = bottom_blob_unpacked.channel(p + 4);
+                        const float* r5 = bottom_blob_unpacked.channel(p + 5);
+                        const float* r6 = bottom_blob_unpacked.channel(p + 6);
+                        const float* r7 = bottom_blob_unpacked.channel(p + 7);
+
+                        float* outptr0 = top_blob.channel(q);
+
+                        for (int j = 0; j < size; j++)
+                        {
+                            outptr0[0] = *r0++;
+                            outptr0[1] = *r1++;
+                            outptr0[2] = *r2++;
+                            outptr0[3] = *r3++;
+                            outptr0[4] = *r4++;
+                            outptr0[5] = *r5++;
+                            outptr0[6] = *r6++;
+                            outptr0[7] = *r7++;
+
+                            outptr0 += 8;
+                        }
+
+                        p += 8;
+                    }
+                }
+                else // if (out_elempack == 1 && top_blob.elempack == 1) if (out_elempack == 8 && top_blob.elempack == 8)
+                {
+                    int size = top_blob.total();
+
+                    const float* ptr = bottom_blob_unpacked.channel(p);
+                    float* outptr = top_blob;
+                    memcpy(outptr, ptr, size * top_blob.elemsize);
+
+                    p += top_blob.c;
+                }
+            }
+
+            return 0;
+        }
+
+        if (dims == 3 && axis == 1)
+        {
+            // slice dim height
+            int w = bottom_blob.w;
+            int h = bottom_blob.h;
+            int channels = bottom_blob.c;
+
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (h - q) / (top_blobs.size() - i);
+                }
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(w, slice, channels, elemsize, elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                q += slice;
+            }
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int p = 0; p < channels; p++)
+            {
+                const float* ptr = bottom_blob.channel(p);
+
+                for (size_t i = 0; i < top_blobs.size(); i++)
+                {
+                    Mat& top_blob = top_blobs[i];
+
+                    int size = top_blob.w * top_blob.h;
+
+                    float* outptr = top_blob.channel(p);
+                    memcpy(outptr, ptr, size * elemsize);
+
+                    ptr += size * elempack;
+                }
+            }
+
+            return 0;
+        }
+
+        if (dims == 3 && axis == 2)
+        {
+            // slice dim width
+            int w = bottom_blob.w;
+            int h = bottom_blob.h;
+            int channels = bottom_blob.c;
+
+            int q = 0;
+            for (size_t i = 0; i < top_blobs.size(); i++)
+            {
+                int slice = slices_ptr[i];
+                if (slice == -233)
+                {
+                    slice = (w - q) / (top_blobs.size() - i);
+                }
+
+                Mat& top_blob = top_blobs[i];
+                top_blob.create(slice, h, channels, elemsize, elempack, opt.blob_allocator);
+                if (top_blob.empty())
+                    return -100;
+
+                q += slice;
+            }
+
+            #pragma omp parallel for num_threads(opt.num_threads)
+            for (int p = 0; p < channels; p++)
+            {
+                const float* ptr = bottom_blob.channel(p);
+
+                for (int j = 0; j < h; j++)
+                {
+                    for (size_t i = 0; i < top_blobs.size(); i++)
+                    {
+                        Mat& top_blob = top_blobs[i];
+
+                        float* outptr = top_blob.channel(p).row(j);
+                        memcpy(outptr, ptr, top_blob.w * elemsize);
+
+                        ptr += top_blob.w * elempack;
+                    }
+                }
+            }
+
+            return 0;
+        }
+
+    }  // opt.use_packing_layout
+#endif // __AVX__
+
+    return Slice::forward(bottom_blobs, top_blobs, opt);
+}
+} // namespace ncnn

src/layer/x86/slice_x86.h

@@ -0,0 +1,38 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_SLICE_x86_H
+#define LAYER_SLICE_x86_H
+
+#include "slice.h"
+
+namespace ncnn {
+
+class Slice_x86 : virtual public Slice
+{
+public:
+    Slice_x86();
+
+    virtual int create_pipeline(const Option& opt);
+    virtual int destroy_pipeline(const Option& opt);
+
+    virtual int forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const;
+
+public:
+    ncnn::Layer* packing_pack1;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_SLICE_x86_H

src/layer/x86/tanh_x86.cpp

@@ -0,0 +1,93 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#if __AVX__
+#include "avx_activation.h"
+#endif // __AVX__
+
+#include "tanh_x86.h"
+
+#include <math.h>
+
+namespace ncnn {
+
+DEFINE_LAYER_CREATOR(TanH_x86)
+
+TanH_x86::TanH_x86()
+{
+#if __AVX__
+    support_packing = true;
+#endif // __AVX__
+}
+
+int TanH_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) const
+{
+    int w = bottom_top_blob.w;
+    int h = bottom_top_blob.h;
+    int channels = bottom_top_blob.c;
+    int size = w * h;
+    int elempack = bottom_top_blob.elempack;
+
+#if __AVX__
+    if (elempack == 8)
+    {
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int q = 0; q < channels; q++)
+        {
+            float* ptr = bottom_top_blob.channel(q);
+
+            for (int i = 0; i < size; i++)
+            {
+                __m256 _p = _mm256_loadu_ps(ptr);
+                _p = tanh_avx(_p);
+                _mm256_storeu_ps(ptr, _p);
+                ptr += 8;
+            }
+        }
+
+        return 0;
+    }
+#endif // __AVX__
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int q = 0; q < channels; q++)
+    {
+        float* ptr = bottom_top_blob.channel(q);
+
+#if __AVX__
+        int nn = size >> 3;
+        int remain = size - (nn << 3);
+#else
+        int remain = size;
+#endif // __AVX__
+
+#if __AVX__
+        for (; nn > 0; nn--)
+        {
+            __m256 _p = _mm256_loadu_ps(ptr);
+            _p = tanh_avx(_p);
+            _mm256_storeu_ps(ptr, _p);
+            ptr += 8;
+        }
+#endif // __AVX__
+        for (; remain > 0; remain--)
+        {
+            *ptr = tanh(*ptr);
+            ptr++;
+        }
+    }
+
+    return 0;
+}
+
+} // namespace ncnn

src/layer/x86/tanh_x86.h

@@ -0,0 +1,32 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef LAYER_TANH_X86_H
+#define LAYER_TANH_X86_H
+
+#include "tanh.h"
+
+namespace ncnn {
+
+class TanH_x86 : virtual public TanH
+{
+public:
+    TanH_x86();
+
+    virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const;
+};
+
+} // namespace ncnn
+
+#endif // LAYER_TANH_X86_H

tests/CMakeLists.txt

@@ -68,3 +68,4 @@ ncnn_add_layer_test(TanH)
 ncnn_add_layer_test(UnaryOp)
 ncnn_add_layer_test(Mish)
 ncnn_add_layer_test(Swish)
+ncnn_add_layer_test(LSTM)

tests/test_lstm.cpp

@@ -0,0 +1,217 @@
+// Tencent is pleased to support the open source community by making ncnn available.
+//
+// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
+//
+// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// https://opensource.org/licenses/BSD-3-Clause
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#include "layer/lstm.h"
+#include "testutil.h"
+
+static int test_lstm(const ncnn::Mat& a, int outch, int direction)
+{
+    int input_size = a.w * a.h * a.c;
+    int num_directions = direction == 2 ? 2 : 1;
+
+    ncnn::ParamDict pd;
+    pd.set(0, outch); // num_output
+    pd.set(1, outch * input_size * 4 * num_directions);
+    pd.set(2, direction); // bias_term
+
+    std::vector<ncnn::Mat> weights(3);
+    weights[0] = RandomMat(outch * input_size * 4 * num_directions);
+    weights[1] = RandomMat(outch * 4 * num_directions);
+    weights[2] = RandomMat(outch * outch * 4 * num_directions);
+    ncnn::Option opt;
+    opt.num_threads = 1;
+    opt.use_int8_inference = false;
+
+    int ret = test_layer<ncnn::LSTM>("LSTM", pd, weights, opt, a);
+    if (ret != 0)
+    {
+        fprintf(stderr, "test_lstm failed a.dims=%d a=(%d %d %d) outch=%d, direction = %d \n", a.dims, a.w, a.h, a.c, outch, direction);
+    }
+
+    return ret;
+}
+
+int test_lstm_layer(const ncnn::Mat& a, int outch, int direction, float epsilon = 0.01)
+{
+    int input_size = a.w * a.h * a.c;
+    ncnn::ParamDict pd;
+    pd.set(0, outch); // num_output
+    pd.set(1, outch * input_size * 4);
+    pd.set(2, direction); // bias_term
+    int num_directions = direction == 2 ? 2 : 1;
+
+    std::vector<ncnn::Mat> weights(3);
+    weights[0] = RandomMat(outch * input_size * 4 * num_directions);
+    weights[1] = RandomMat(outch * 4 * num_directions);
+    weights[2] = RandomMat(outch * outch * 4 * num_directions);
+
+    ncnn::Option opt;
+    opt.num_threads = 1;
+    opt.use_int8_inference = false;
+
+    ncnn::LSTM* op = (ncnn::LSTM*)ncnn::create_layer(ncnn::layer_to_index("LSTM"));
+
+    if (!op->support_vulkan) opt.use_vulkan_compute = false;
+    if (!op->support_packing) opt.use_packing_layout = false;
+    if (!op->support_bf16_storage) opt.use_bf16_storage = false;
+    if (!op->support_image_storage) opt.use_image_storage = false;
+
+    op->load_param(pd);
+
+    ncnn::ModelBinFromMatArray mb(weights.data());
+
+    op->load_model(mb);
+
+    op->create_pipeline(opt);
+
+    ncnn::Mat b;
+    op->LSTM::forward(a, b, opt);
+
+    std::vector<ncnn::Mat> _c1(3);
+    std::vector<ncnn::Mat> _c2(3);
+    std::vector<ncnn::Mat> a1(3);
+    std::vector<ncnn::Mat> a2(3);
+    if (direction == 0)
+    {
+        a1[0] = a.row_range(0, a.h / 2).clone();
+        a2[0] = a.row_range(a.h / 2, a.h - a.h / 2).clone();
+    }
+    else
+    {
+        a2[0] = a.row_range(0, a.h / 2).clone();
+        a1[0] = a.row_range(a.h / 2, a.h - a.h / 2).clone();
+    }
+
+    // initial hidden state
+    ncnn::Mat hidden(outch);
+    if (hidden.empty())
+        return -100;
+    hidden.fill(0.f);
+
+    ncnn::Mat cell(outch);
+    if (cell.empty())
+        return -100;
+    cell.fill(0.f);
+
+    a1[1] = hidden;
+    a1[2] = cell;
+    op->forward(a1, _c1, opt);
+    a2[1] = _c1[1];
+    a2[2] = _c1[2];
+    op->forward(a2, _c2, opt);
+
+    ncnn::Mat c1 = _c1[0];
+    ncnn::Mat c2 = _c2[0];
+
+    if (direction == 1)
+    {
+        c2 = _c1[0];
+        c1 = _c2[0];
+    }
+
+    // total height
+    ncnn::Mat c;
+    c.create(b.w, b.h, b.elemsize, opt.blob_allocator);
+    if (c.empty())
+        return -100;
+
+    unsigned char* outptr = c;
+    int c1_size = c1.w * c1.h;
+    const unsigned char* c1ptr = c1;
+    memcpy(outptr, c1ptr, c1_size * c1.elemsize);
+    outptr += c1_size * c1.elemsize;
+    int c2_size = c2.w * c2.h;
+    const unsigned char* c2ptr = c2;
+    memcpy(outptr, c2ptr, c2_size * c2.elemsize);
+
+    op->destroy_pipeline(opt);
+
+    delete op;
+
+    if (CompareMat(b, c, epsilon) != 0)
+    {
+        fprintf(stderr, "test_lstm two step failed a.dims=%d a=(%d %d %d) outch=%d, direction = %d \n", a.dims, a.w, a.h, a.c, outch, direction);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int test_lstm_0()
+{
+    return 0
+           || test_lstm(RandomMat(4, 1), 2, 2)
+           || test_lstm(RandomMat(8, 2), 2, 2)
+           || test_lstm(RandomMat(16, 8), 7, 2)
+           || test_lstm(RandomMat(17, 8), 8, 2)
+           || test_lstm(RandomMat(19, 15), 8, 2)
+           || test_lstm(RandomMat(5, 16), 16, 2)
+           || test_lstm(RandomMat(3, 16), 8, 2)
+           || test_lstm(RandomMat(8, 16), 16, 2)
+           || test_lstm(RandomMat(2, 5), 17, 2);
+}
+
+static int test_lstm_1()
+{
+    return 0
+           || test_lstm_layer(RandomMat(4, 4), 1, 1)
+           || test_lstm_layer(RandomMat(8, 2), 2, 1)
+           || test_lstm_layer(RandomMat(16, 8), 7, 1)
+           || test_lstm_layer(RandomMat(17, 8), 8, 1)
+           || test_lstm_layer(RandomMat(19, 15), 8, 1)
+           || test_lstm_layer(RandomMat(5, 16), 16, 1)
+           || test_lstm_layer(RandomMat(3, 16), 8, 1)
+           || test_lstm_layer(RandomMat(2, 5), 99, 1)
+           || test_lstm_layer(RandomMat(4, 2), 1, 0)
+           || test_lstm_layer(RandomMat(8, 2), 2, 0)
+           || test_lstm_layer(RandomMat(16, 8), 7, 0)
+           || test_lstm_layer(RandomMat(17, 8), 8, 0)
+           || test_lstm_layer(RandomMat(19, 15), 8, 0)
+           || test_lstm_layer(RandomMat(5, 16), 16, 0)
+           || test_lstm_layer(RandomMat(3, 16), 8, 0)
+           || test_lstm_layer(RandomMat(2, 5), 17, 0);
+}
+
+static int test_lstm_2()
+{
+    return 0
+           || test_lstm(RandomMat(4, 1), 1, 0)
+           || test_lstm(RandomMat(8, 2), 2, 0)
+           || test_lstm(RandomMat(16, 8), 7, 0)
+           || test_lstm(RandomMat(17, 8), 8, 0)
+           || test_lstm(RandomMat(19, 15), 8, 0)
+           || test_lstm(RandomMat(5, 16), 16, 0)
+           || test_lstm(RandomMat(3, 16), 8, 0)
+           || test_lstm(RandomMat(8, 16), 16, 0)
+           || test_lstm(RandomMat(2, 5), 17, 0);
+}
+static int test_lstm_3()
+{
+    return 0
+           || test_lstm(RandomMat(4, 1), 1, 1)
+           || test_lstm(RandomMat(8, 2), 2, 1)
+           || test_lstm(RandomMat(16, 8), 7, 1)
+           || test_lstm(RandomMat(17, 8), 8, 1)
+           || test_lstm(RandomMat(19, 15), 8, 1)
+           || test_lstm(RandomMat(5, 16), 16, 1)
+           || test_lstm(RandomMat(3, 16), 8, 1)
+           || test_lstm(RandomMat(8, 16), 16, 1)
+           || test_lstm(RandomMat(2, 5), 17, 1);
+}
+
+int main()
+{
+    SRAND(7767517);
+    return 0 || test_lstm_0() || test_lstm_1() || test_lstm_2() || test_lstm_3();
+}

tests/testutil.h

@@ -515,7 +515,6 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn:
         cmd.submit_and_wait();
     }
 #endif // NCNN_VULKAN
-
     ncnn::Mat b;
     if (op->support_inplace)
     {
@@ -663,7 +662,7 @@ int test_layer(int typeindex, const ncnn::ParamDict& pd, const std::vector<ncnn:
 template<typename T>
 int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const std::vector<ncnn::Mat>& a, int top_blob_count = 1, float epsilon = 0.001, void (*func)(T*) = 0)
 {
-    ncnn::Option opts[3];
+    ncnn::Option opts[4];
     opts[0] = _opt;
     opts[0].use_packing_layout = false;
     opts[0].use_fp16_packed = false;
@@ -683,8 +682,14 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec
     opts[2].use_bf16_storage = true;
     opts[2].use_shader_pack8 = true;
     opts[2].use_image_storage = true;
-
-    for (int i = 0; i < 3; i++)
+    opts[3] = _opt;
+    opts[3].use_packing_layout = true;
+    opts[3].use_fp16_packed = true;
+    opts[3].use_fp16_storage = true;
+    opts[3].use_bf16_storage = false;
+    opts[3].use_shader_pack8 = true;
+    opts[3].use_image_storage = true;
+    for (int i = 0; i < 4; i++)
     {
         const ncnn::Option& opt = opts[i];
 
@@ -750,7 +755,7 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec
 template<typename T>
 int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vector<ncnn::Mat>& weights, const ncnn::Option& _opt, const ncnn::Mat& a, float epsilon = 0.001, void (*func)(T*) = 0)
 {
-    ncnn::Option opts[3];
+    ncnn::Option opts[4];
     opts[0] = _opt;
     opts[0].use_packing_layout = false;
     opts[0].use_fp16_packed = false;
@@ -770,11 +775,16 @@ int test_layer(const char* layer_type, const ncnn::ParamDict& pd, const std::vec
     opts[2].use_bf16_storage = true;
     opts[2].use_shader_pack8 = true;
     opts[2].use_image_storage = true;
-
-    for (int i = 0; i < 3; i++)
+    opts[3] = _opt;
+    opts[3].use_packing_layout = true;
+    opts[3].use_fp16_packed = true;
+    opts[3].use_fp16_storage = true;
+    opts[3].use_bf16_storage = false;
+    opts[3].use_shader_pack8 = true;
+    opts[3].use_image_storage = true;
+    for (int i = 0; i < 4; i++)
     {
         const ncnn::Option& opt = opts[i];
-
         // fp16 representation
         ncnn::Mat a_fp16;
         std::vector<ncnn::Mat> weights_fp16;

toolchains/arm-linux-gnueabihf.toolchain.cmake

@@ -8,8 +8,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 
-set(CMAKE_C_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4")
-set(CMAKE_CXX_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4")
+set(CMAKE_C_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4 -mfp16-format=ieee")
+set(CMAKE_CXX_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon-vfpv4 -mfp16-format=ieee")
 
 # cache flags
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags")