nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 4d2d625432
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '4d2d625432'
${ noResults }
ncnn/src/layer/x86/lstm_x86.cpp

940 lines
40 kB
Raw Blame History 

  1. // Tencent is pleased to support the open source community by making ncnn available.

  2. //

  3. // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.

  4. //

  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except

  6. // in compliance with the License. You may obtain a copy of the License at

  7. //

  8. // https://opensource.org/licenses/BSD-3-Clause

  9. //

  10. // Unless required by applicable law or agreed to in writing, software distributed

  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR

  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the

  13. // specific language governing permissions and limitations under the License.

  14. #ifdef __AVX__

  15. #include "avx_activation.h"

  16. #include "avx_usability.h"

  17. #endif // NCNN_AVX2

  18. 

  19. #include "lstm_x86.h"

  20. 

  21. #include <math.h>

  22. #include "layer_type.h"

  23. 

  24. namespace ncnn {

  25. 

  26. LSTM_x86::LSTM_x86()

  27. {

  28. #ifdef __AVX__

  29.     support_weight_fp16_storage = true;

  30. #endif

  31.     one_blob_only = false;

  32.     support_inplace = false;

  33. }

  34. 

  35. int LSTM_x86::create_pipeline(const Option& opt)

  36. {

  37. #if __AVX__

  38.     if (opt.use_weight_fp16_storage)

  39.     {

  40.         ncnn::cast_float32_to_float16(weight_xc_data, weight_xc_data_fp16, opt);

  41.         ncnn::cast_float32_to_float16(weight_hc_data, weight_hc_data_fp16, opt);

  42.     }

  43. #endif // __AVX__

  44. 

  45.     return 0;

  46. }

  47. #ifdef __AVX__

  48. 

  49. static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)

  50. {

  51.     int size = bottom_blob.w;

  52.     int T = bottom_blob.h;

  53. 

  54.     int num_output = top_blob.w;

  55.     // fprintf(stderr, "bottom_blob = %d x %d x %d num_output = %d \n", bottom_blob.w,bottom_blob.h,bottom_blob.c,num_output);

  56.     // 4 x num_output

  57.     Mat gates(num_output, 4, 4u, opt.workspace_allocator);

  58.     if (gates.empty())

  59.         return -100;

  60.     // unroll

  61.     for (int t = 0; t < T; t++)

  62.     {

  63.         // clip hidden by continuation indicator

  64.         // h_cont_{t-1} = cont_t * h_{t-1}

  65.         // h_cont_{t-1} = h_{t-1} if cont_t == 1

  66.         //                0       otherwise

  67.         // calculate hidden

  68.         // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c

  69.         int ti = reverse ? T - 1 - t : t;

  70.         int remain_output = (num_output >> 1) << 1;

  71.         for (int q = 0; q + 1 < num_output; q += 2)

  72.         {

  73.             const float* x = bottom_blob.row(ti);

  74.             const float* hidden_ptr_r = hidden_state;

  75.             const float* bias_c_I = bias_c.row(0);

  76.             const float* bias_c_F = bias_c.row(1);

  77.             const float* bias_c_O = bias_c.row(2);

  78.             const float* bias_c_G = bias_c.row(3);

  79. 

  80.             float* gates_data_I = gates.row(0);

  81.             float* gates_data_F = gates.row(1);

  82.             float* gates_data_O = gates.row(2);

  83.             float* gates_data_G = gates.row(3);

  84.             // gate I F O G

  85.             const unsigned short* weight_xc_I_0 = (const unsigned short*)weight_xc.row(num_output * 0 + q);

  86.             const unsigned short* weight_xc_F_0 = (const unsigned short*)weight_xc.row(num_output * 1 + q);

  87.             const unsigned short* weight_xc_O_0 = (const unsigned short*)weight_xc.row(num_output * 2 + q);

  88.             const unsigned short* weight_xc_G_0 = (const unsigned short*)weight_xc.row(num_output * 3 + q);

  89.             const unsigned short* weight_xc_I_1 = (const unsigned short*)weight_xc.row(num_output * 0 + (q + 1));

  90.             const unsigned short* weight_xc_F_1 = (const unsigned short*)weight_xc.row(num_output * 1 + (q + 1));

  91.             const unsigned short* weight_xc_O_1 = (const unsigned short*)weight_xc.row(num_output * 2 + (q + 1));

  92.             const unsigned short* weight_xc_G_1 = (const unsigned short*)weight_xc.row(num_output * 3 + (q + 1));

  93. 

  94.             const unsigned short* weight_hc_I_0 = (const unsigned short*)weight_hc.row(num_output * 0 + q);

  95.             const unsigned short* weight_hc_F_0 = (const unsigned short*)weight_hc.row(num_output * 1 + q);

  96.             const unsigned short* weight_hc_O_0 = (const unsigned short*)weight_hc.row(num_output * 2 + q);

  97.             const unsigned short* weight_hc_G_0 = (const unsigned short*)weight_hc.row(num_output * 3 + q);

  98.             const unsigned short* weight_hc_I_1 = (const unsigned short*)weight_hc.row(num_output * 0 + (q + 1));

  99.             const unsigned short* weight_hc_F_1 = (const unsigned short*)weight_hc.row(num_output * 1 + (q + 1));

  100.             const unsigned short* weight_hc_O_1 = (const unsigned short*)weight_hc.row(num_output * 2 + (q + 1));

  101.             const unsigned short* weight_hc_G_1 = (const unsigned short*)weight_hc.row(num_output * 3 + (q + 1));

  102. 

  103.             // float I = bias_c_I[q];

  104.             // float F = bias_c_F[q];

  105.             // float O = bias_c_O[q];

  106.             // float G = bias_c_G[q];

  107.             __m256 _sumI_0 = _mm256_setzero_ps();

  108.             __m256 _sumF_0 = _mm256_setzero_ps();

  109.             __m256 _sumO_0 = _mm256_setzero_ps();

  110.             __m256 _sumG_0 = _mm256_setzero_ps();

  111.             __m256 _sumI_1 = _mm256_setzero_ps();

  112.             __m256 _sumF_1 = _mm256_setzero_ps();

  113.             __m256 _sumO_1 = _mm256_setzero_ps();

  114.             __m256 _sumG_1 = _mm256_setzero_ps();

  115.             int nn_num_size = size >> 3;

  116.             int remain_size = size & 7;

  117.             for (; nn_num_size > 0; nn_num_size--)

  118.             {

  119.                 __m256 xi = _mm256_loadu_ps(x);

  120.                 _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_xc_I_0), xi, _sumI_0);

  121.                 _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_xc_F_0), xi, _sumF_0);

  122.                 _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_xc_O_0), xi, _sumO_0);

  123.                 _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_xc_G_0), xi, _sumG_0);

  124.                 _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_xc_I_1), xi, _sumI_1);

  125.                 _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_xc_F_1), xi, _sumF_1);

  126.                 _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_xc_O_1), xi, _sumO_1);

  127.                 _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_xc_G_1), xi, _sumG_1);

  128.                 x += 8;

  129.                 weight_xc_I_0 += 8;

  130.                 weight_xc_F_0 += 8;

  131.                 weight_xc_O_0 += 8;

  132.                 weight_xc_G_0 += 8;

  133.                 weight_xc_I_1 += 8;

  134.                 weight_xc_F_1 += 8;

  135.                 weight_xc_O_1 += 8;

  136.                 weight_xc_G_1 += 8;

  137.             }

  138.             int nn_num_output = num_output >> 3;

  139.             int remain_num_output = num_output & 7;

  140.             for (; nn_num_output > 0; nn_num_output--)

  141.             {

  142.                 __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);

  143. 

  144.                 _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_hc_I_0), h_cont, _sumI_0);

  145.                 _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_hc_F_0), h_cont, _sumF_0);

  146.                 _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_hc_O_0), h_cont, _sumO_0);

  147.                 _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_hc_G_0), h_cont, _sumG_0);

  148.                 _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_hc_I_1), h_cont, _sumI_1);

  149.                 _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_hc_F_1), h_cont, _sumF_1);

  150.                 _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_hc_O_1), h_cont, _sumO_1);

  151.                 _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_hc_G_1), h_cont, _sumG_1);

  152.                 hidden_ptr_r += 8;

  153.                 weight_hc_I_0 += 8;

  154.                 weight_hc_F_0 += 8;

  155.                 weight_hc_O_0 += 8;

  156.                 weight_hc_G_0 += 8;

  157.                 weight_hc_I_1 += 8;

  158.                 weight_hc_F_1 += 8;

  159.                 weight_hc_O_1 += 8;

  160.                 weight_hc_G_1 += 8;

  161.             }

  162.             if (remain_size != 0)

  163.             {

  164.                 unsigned short fp16_weights[8][8] = {{0}};

  165.                 float _xi_f[8] = {0};

  166.                 // No fast way to convert to fp32 one element at the time

  167.                 // so batch an 8 lane vector.

  168.                 for (int i = 0; i < remain_size; i++)

  169.                 {

  170.                     _xi_f[i] = *x;

  171.                     fp16_weights[0][i] = *weight_xc_I_0;

  172.                     fp16_weights[1][i] = *weight_xc_F_0;

  173.                     fp16_weights[2][i] = *weight_xc_O_0;

  174.                     fp16_weights[3][i] = *weight_xc_G_0;

  175.                     fp16_weights[4][i] = *weight_xc_I_1;

  176.                     fp16_weights[5][i] = *weight_xc_F_1;

  177.                     fp16_weights[6][i] = *weight_xc_O_1;

  178.                     fp16_weights[7][i] = *weight_xc_G_1;

  179.                     x++;

  180.                     weight_xc_I_0++;

  181.                     weight_xc_F_0++;

  182.                     weight_xc_O_0++;

  183.                     weight_xc_G_0++;

  184.                     weight_xc_I_1++;

  185.                     weight_xc_F_1++;

  186.                     weight_xc_O_1++;

  187.                     weight_xc_G_1++;

  188.                 }

  189.                 __m256 xi = _mm256_loadu_ps(_xi_f);

  190.                 _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI_0);

  191.                 _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF_0);

  192.                 _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO_0);

  193.                 _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG_0);

  194.                 _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), xi, _sumI_1);

  195.                 _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), xi, _sumF_1);

  196.                 _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), xi, _sumO_1);

  197.                 _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), xi, _sumG_1);

  198.             }

  199.             if (remain_num_output != 0)

  200.             {

  201.                 unsigned short fp16_weights[8][8] = {{0}};

  202.                 float _hcont_f[8] = {0};

  203.                 // No fast way to convert to fp32 one element at the time

  204.                 // so batch an 8 lane vector.

  205.                 for (int i = 0; i < remain_num_output; i++)

  206.                 {

  207.                     _hcont_f[i] = *hidden_ptr_r;

  208.                     fp16_weights[0][i] = *weight_hc_I_0;

  209.                     fp16_weights[1][i] = *weight_hc_F_0;

  210.                     fp16_weights[2][i] = *weight_hc_O_0;

  211.                     fp16_weights[3][i] = *weight_hc_G_0;

  212.                     fp16_weights[4][i] = *weight_hc_I_1;

  213.                     fp16_weights[5][i] = *weight_hc_F_1;

  214.                     fp16_weights[6][i] = *weight_hc_O_1;

  215.                     fp16_weights[7][i] = *weight_hc_G_1;

  216.                     hidden_ptr_r++;

  217.                     weight_hc_I_0++;

  218.                     weight_hc_F_0++;

  219.                     weight_hc_O_0++;

  220.                     weight_hc_G_0++;

  221.                     weight_hc_I_1++;

  222.                     weight_hc_F_1++;

  223.                     weight_hc_O_1++;

  224.                     weight_hc_G_1++;

  225.                 }

  226.                 __m256 h_cont = _mm256_loadu_ps(_hcont_f);

  227.                 _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI_0);

  228.                 _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF_0);

  229.                 _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO_0);

  230.                 _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG_0);

  231.                 _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), h_cont, _sumI_1);

  232.                 _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), h_cont, _sumF_1);

  233.                 _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), h_cont, _sumO_1);

  234.                 _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), h_cont, _sumG_1);

  235.             }

  236.             float sums[8];

  237.             _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1));

  238.             sums[0] += bias_c_I[q];

  239.             sums[1] += bias_c_F[q];

  240.             sums[2] += bias_c_O[q];

  241.             sums[3] += bias_c_G[q];

  242.             sums[4] += bias_c_I[q + 1];

  243.             sums[5] += bias_c_F[q + 1];

  244.             sums[6] += bias_c_O[q + 1];

  245.             sums[7] += bias_c_G[q + 1];

  246.             gates_data_I[q] = sums[0];

  247.             gates_data_F[q] = sums[1];

  248.             gates_data_O[q] = sums[2];

  249.             gates_data_G[q] = sums[3];

  250.             gates_data_I[q + 1] = sums[4];

  251.             gates_data_F[q + 1] = sums[5];

  252.             gates_data_O[q + 1] = sums[6];

  253.             gates_data_G[q + 1] = sums[7];

  254.         }

  255. 

  256.         for (int q = remain_output; q < num_output; q++)

  257.         {

  258.             const float* x = bottom_blob.row(ti);

  259.             const float* hidden_ptr_r = hidden_state;

  260.             const float* bias_c_I = bias_c.row(0);

  261.             const float* bias_c_F = bias_c.row(1);

  262.             const float* bias_c_O = bias_c.row(2);

  263.             const float* bias_c_G = bias_c.row(3);

  264. 

  265.             float* gates_data_I = gates.row(0);

  266.             float* gates_data_F = gates.row(1);

  267.             float* gates_data_O = gates.row(2);

  268.             float* gates_data_G = gates.row(3);

  269.             // gate I F O G

  270.             const unsigned short* weight_xc_I = (const unsigned short*)weight_xc.row(num_output * 0 + q);

  271.             const unsigned short* weight_xc_F = (const unsigned short*)weight_xc.row(num_output * 1 + q);

  272.             const unsigned short* weight_xc_O = (const unsigned short*)weight_xc.row(num_output * 2 + q);

  273.             const unsigned short* weight_xc_G = (const unsigned short*)weight_xc.row(num_output * 3 + q);

  274. 

  275.             const unsigned short* weight_hc_I = (const unsigned short*)weight_hc.row(num_output * 0 + q);

  276.             const unsigned short* weight_hc_F = (const unsigned short*)weight_hc.row(num_output * 1 + q);

  277.             const unsigned short* weight_hc_O = (const unsigned short*)weight_hc.row(num_output * 2 + q);

  278.             const unsigned short* weight_hc_G = (const unsigned short*)weight_hc.row(num_output * 3 + q);

  279. 

  280.             // float I = bias_c_I[q];

  281.             // float F = bias_c_F[q];

  282.             // float O = bias_c_O[q];

  283.             // float G = bias_c_G[q];

  284.             __m256 _sumI = _mm256_setzero_ps();

  285.             __m256 _sumF = _mm256_setzero_ps();

  286.             __m256 _sumO = _mm256_setzero_ps();

  287.             __m256 _sumG = _mm256_setzero_ps();

  288.             int nn_num_size = size >> 3;

  289.             int remain_size = size & 7;

  290.             for (; nn_num_size > 0; nn_num_size--)

  291.             {

  292.                 __m256 xi = _mm256_loadu_ps(x);

  293.                 _sumI = _mm256_fmadd_ps(loadfp16(weight_xc_I), xi, _sumI);

  294.                 _sumF = _mm256_fmadd_ps(loadfp16(weight_xc_F), xi, _sumF);

  295.                 _sumO = _mm256_fmadd_ps(loadfp16(weight_xc_O), xi, _sumO);

  296.                 _sumG = _mm256_fmadd_ps(loadfp16(weight_xc_G), xi, _sumG);

  297.                 x += 8;

  298.                 weight_xc_I += 8;

  299.                 weight_xc_F += 8;

  300.                 weight_xc_O += 8;

  301.                 weight_xc_G += 8;

  302.             }

  303.             int nn_num_output = num_output >> 3;

  304.             int remain_num_output = num_output & 7;

  305.             for (; nn_num_output > 0; nn_num_output--)

  306.             {

  307.                 __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);

  308. 

  309.                 _sumI = _mm256_fmadd_ps(loadfp16(weight_hc_I), h_cont, _sumI);

  310.                 _sumF = _mm256_fmadd_ps(loadfp16(weight_hc_F), h_cont, _sumF);

  311.                 _sumO = _mm256_fmadd_ps(loadfp16(weight_hc_O), h_cont, _sumO);

  312.                 _sumG = _mm256_fmadd_ps(loadfp16(weight_hc_G), h_cont, _sumG);

  313.                 hidden_ptr_r += 8;

  314.                 weight_hc_I += 8;

  315.                 weight_hc_F += 8;

  316.                 weight_hc_O += 8;

  317.                 weight_hc_G += 8;

  318.             }

  319.             if (remain_size != 0)

  320.             {

  321.                 unsigned short fp16_weights[4][8] = {{0}};

  322.                 float _xi_f[8] = {0};

  323.                 // No fast way to convert to fp32 one element at the time

  324.                 // so batch an 8 lane vector.

  325.                 for (int i = 0; i < remain_size; i++)

  326.                 {

  327.                     _xi_f[i] = *x;

  328.                     fp16_weights[0][i] = *weight_xc_I;

  329.                     fp16_weights[1][i] = *weight_xc_F;

  330.                     fp16_weights[2][i] = *weight_xc_O;

  331.                     fp16_weights[3][i] = *weight_xc_G;

  332.                     x++;

  333.                     weight_xc_I++;

  334.                     weight_xc_F++;

  335.                     weight_xc_O++;

  336.                     weight_xc_G++;

  337.                 }

  338.                 __m256 xi = _mm256_loadu_ps(_xi_f);

  339.                 _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI);

  340.                 _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF);

  341.                 _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO);

  342.                 _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG);

  343.             }

  344.             if (remain_num_output != 0)

  345.             {

  346.                 unsigned short fp16_weights[4][8] = {{0}};

  347.                 float _hcont_f[8] = {0};

  348.                 // No fast way to convert to fp32 one element at the time

  349.                 // so batch an 8 lane vector.

  350.                 for (int i = 0; i < remain_num_output; i++)

  351.                 {

  352.                     _hcont_f[i] = *hidden_ptr_r;

  353.                     fp16_weights[0][i] = *weight_hc_I;

  354.                     fp16_weights[1][i] = *weight_hc_F;

  355.                     fp16_weights[2][i] = *weight_hc_O;

  356.                     fp16_weights[3][i] = *weight_hc_G;

  357.                     hidden_ptr_r++;

  358.                     weight_hc_I++;

  359.                     weight_hc_F++;

  360.                     weight_hc_O++;

  361.                     weight_hc_G++;

  362.                 }

  363.                 __m256 h_cont = _mm256_loadu_ps(_hcont_f);

  364.                 _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI);

  365.                 _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF);

  366.                 _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO);

  367.                 _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG);

  368.             }

  369. 

  370.             float sums[4];

  371.             _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG));

  372.             sums[0] += bias_c_I[q];

  373.             sums[1] += bias_c_F[q];

  374.             sums[2] += bias_c_O[q];

  375.             sums[3] += bias_c_G[q];

  376.             gates_data_I[q] = sums[0];

  377.             gates_data_F[q] = sums[1];

  378.             gates_data_O[q] = sums[2];

  379.             gates_data_G[q] = sums[3];

  380.         }

  381. 

  382.         // lstm unit

  383.         // sigmoid(I)

  384.         // sigmoid(F)

  385.         // sigmoid(O)

  386.         // tanh(G)

  387.         // c_t := f_t .* c_{t-1} + i_t .* g_t

  388.         // h_t := o_t .* tanh[c_t]

  389.         float* output_data = top_blob.row(ti);

  390.         float* cell_ptr = cell_state;

  391.         float* hidden_ptr = hidden_state;

  392.         const float* gates_data_I = gates.row(0);

  393.         const float* gates_data_F = gates.row(1);

  394.         const float* gates_data_O = gates.row(2);

  395.         const float* gates_data_G = gates.row(3);

  396.         int nn_activation = num_output >> 3;

  397.         int remain_activations = num_output & 7;

  398.         for (; nn_activation > 0; nn_activation--)

  399.         {

  400.             __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I));

  401.             __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F));

  402.             __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O));

  403.             __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G));

  404.             __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G));

  405.             __m256 H = _mm256_mul_ps(O, tanh_avx(cell2));

  406.             _mm256_storeu_ps(cell_ptr, cell2);

  407.             _mm256_storeu_ps(hidden_ptr, H);

  408.             _mm256_storeu_ps(output_data, H);

  409.             cell_ptr += 8;

  410.             output_data += 8;

  411.             hidden_ptr += 8;

  412.             gates_data_I += 8;

  413.             gates_data_F += 8;

  414.             gates_data_O += 8;

  415.             gates_data_G += 8;

  416.         }

  417.         for (; remain_activations > 0; remain_activations--)

  418.         {

  419.             float I = *gates_data_I;

  420.             float F = *gates_data_F;

  421.             float O = *gates_data_O;

  422.             float G = *gates_data_G;

  423. 

  424.             I = 1.f / (1.f + exp(-I));

  425.             F = 1.f / (1.f + exp(-F));

  426.             O = 1.f / (1.f + exp(-O));

  427.             G = tanh(G);

  428.             float cell2 = F * *cell_ptr + I * G;

  429.             float H = O * tanh(cell2);

  430.             *cell_ptr = cell2;

  431.             *hidden_ptr = H;

  432.             *output_data = H;

  433.             cell_ptr++;

  434.             output_data++;

  435.             hidden_ptr++;

  436.             gates_data_I++;

  437.             gates_data_F++;

  438.             gates_data_O++;

  439.             gates_data_G++;

  440.         }

  441. 

  442.         // no cell output here

  443.     }

  444. 

  445.     return 0;

  446. }

  447. 

  448. static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt)

  449. {

  450.     int size = bottom_blob.w;

  451.     int T = bottom_blob.h;

  452. 

  453.     int num_output = top_blob.w;

  454. 

  455.     // 4 x num_output

  456.     Mat gates(num_output, 4, 4u, opt.workspace_allocator);

  457.     if (gates.empty())

  458.         return -100;

  459. 

  460.     // unroll

  461.     for (int t = 0; t < T; t++)

  462.     {

  463.         // clip hidden by continuation indicator

  464.         // h_cont_{t-1} = cont_t * h_{t-1}

  465.         // h_cont_{t-1} = h_{t-1} if cont_t == 1

  466.         //                0       otherwise

  467.         // calculate hidden

  468.         // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c

  469. 

  470.         int ti = reverse ? T - 1 - t : t;

  471.         int remain_output = (num_output >> 1) << 1;

  472.         for (int q = 0; q + 1 < num_output; q += 2)

  473.         {

  474.             const float* x = bottom_blob.row(ti);

  475.             const float* hidden_ptr_r = hidden_state;

  476.             const float* bias_c_I = bias_c.row(0);

  477.             const float* bias_c_F = bias_c.row(1);

  478.             const float* bias_c_O = bias_c.row(2);

  479.             const float* bias_c_G = bias_c.row(3);

  480. 

  481.             float* gates_data_I = gates.row(0);

  482.             float* gates_data_F = gates.row(1);

  483.             float* gates_data_O = gates.row(2);

  484.             float* gates_data_G = gates.row(3);

  485.             // gate I F O G

  486.             const float* weight_xc_I_0 = weight_xc.row(num_output * 0 + q);

  487.             const float* weight_xc_F_0 = weight_xc.row(num_output * 1 + q);

  488.             const float* weight_xc_O_0 = weight_xc.row(num_output * 2 + q);

  489.             const float* weight_xc_G_0 = weight_xc.row(num_output * 3 + q);

  490.             const float* weight_xc_I_1 = weight_xc.row(num_output * 0 + (q + 1));

  491.             const float* weight_xc_F_1 = weight_xc.row(num_output * 1 + (q + 1));

  492.             const float* weight_xc_O_1 = weight_xc.row(num_output * 2 + (q + 1));

  493.             const float* weight_xc_G_1 = weight_xc.row(num_output * 3 + (q + 1));

  494. 

  495.             const float* weight_hc_I_0 = weight_hc.row(num_output * 0 + q);

  496.             const float* weight_hc_F_0 = weight_hc.row(num_output * 1 + q);

  497.             const float* weight_hc_O_0 = weight_hc.row(num_output * 2 + q);

  498.             const float* weight_hc_G_0 = weight_hc.row(num_output * 3 + q);

  499.             const float* weight_hc_I_1 = weight_hc.row(num_output * 0 + (q + 1));

  500.             const float* weight_hc_F_1 = weight_hc.row(num_output * 1 + (q + 1));

  501.             const float* weight_hc_O_1 = weight_hc.row(num_output * 2 + (q + 1));

  502.             const float* weight_hc_G_1 = weight_hc.row(num_output * 3 + (q + 1));

  503. 

  504.             // float I = bias_c_I[q];

  505.             // float F = bias_c_F[q];

  506.             // float O = bias_c_O[q];

  507.             // float G = bias_c_G[q];

  508.             __m256 _sumI_0 = _mm256_setzero_ps();

  509.             __m256 _sumF_0 = _mm256_setzero_ps();

  510.             __m256 _sumO_0 = _mm256_setzero_ps();

  511.             __m256 _sumG_0 = _mm256_setzero_ps();

  512.             __m256 _sumI_1 = _mm256_setzero_ps();

  513.             __m256 _sumF_1 = _mm256_setzero_ps();

  514.             __m256 _sumO_1 = _mm256_setzero_ps();

  515.             __m256 _sumG_1 = _mm256_setzero_ps();

  516.             int nn_num_size = size >> 3;

  517.             int remain_size = size & 7;

  518.             for (; nn_num_size > 0; nn_num_size--)

  519.             {

  520.                 __m256 xi = _mm256_loadu_ps(x);

  521.                 _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_0), xi, _sumI_0);

  522.                 _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_0), xi, _sumF_0);

  523.                 _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_0), xi, _sumO_0);

  524.                 _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_0), xi, _sumG_0);

  525.                 _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_1), xi, _sumI_1);

  526.                 _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_1), xi, _sumF_1);

  527.                 _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_1), xi, _sumO_1);

  528.                 _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_1), xi, _sumG_1);

  529.                 x += 8;

  530.                 weight_xc_I_0 += 8;

  531.                 weight_xc_F_0 += 8;

  532.                 weight_xc_O_0 += 8;

  533.                 weight_xc_G_0 += 8;

  534.                 weight_xc_I_1 += 8;

  535.                 weight_xc_F_1 += 8;

  536.                 weight_xc_O_1 += 8;

  537.                 weight_xc_G_1 += 8;

  538.             }

  539.             int nn_num_output = num_output >> 3;

  540.             int remain_num_output = num_output & 7;

  541.             for (; nn_num_output > 0; nn_num_output--)

  542.             {

  543.                 __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);

  544. 

  545.                 _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_0), h_cont, _sumI_0);

  546.                 _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_0), h_cont, _sumF_0);

  547.                 _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_0), h_cont, _sumO_0);

  548.                 _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_0), h_cont, _sumG_0);

  549.                 _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_1), h_cont, _sumI_1);

  550.                 _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_1), h_cont, _sumF_1);

  551.                 _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_1), h_cont, _sumO_1);

  552.                 _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_1), h_cont, _sumG_1);

  553.                 hidden_ptr_r += 8;

  554.                 weight_hc_I_0 += 8;

  555.                 weight_hc_F_0 += 8;

  556.                 weight_hc_O_0 += 8;

  557.                 weight_hc_G_0 += 8;

  558.                 weight_hc_I_1 += 8;

  559.                 weight_hc_F_1 += 8;

  560.                 weight_hc_O_1 += 8;

  561.                 weight_hc_G_1 += 8;

  562.             }

  563.             float sums[8];

  564.             _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1));

  565.             sums[0] += bias_c_I[q];

  566.             sums[1] += bias_c_F[q];

  567.             sums[2] += bias_c_O[q];

  568.             sums[3] += bias_c_G[q];

  569.             sums[4] += bias_c_I[q + 1];

  570.             sums[5] += bias_c_F[q + 1];

  571.             sums[6] += bias_c_O[q + 1];

  572.             sums[7] += bias_c_G[q + 1];

  573. 

  574.             for (; remain_size > 0; remain_size--)

  575.             {

  576.                 float xi = *x;

  577.                 sums[0] += *weight_xc_I_0 * xi;

  578.                 sums[1] += *weight_xc_F_0 * xi;

  579.                 sums[2] += *weight_xc_O_0 * xi;

  580.                 sums[3] += *weight_xc_G_0 * xi;

  581.                 sums[4] += *weight_xc_I_1 * xi;

  582.                 sums[5] += *weight_xc_F_1 * xi;

  583.                 sums[6] += *weight_xc_O_1 * xi;

  584.                 sums[7] += *weight_xc_G_1 * xi;

  585.                 x++;

  586.                 weight_xc_I_0++;

  587.                 weight_xc_F_0++;

  588.                 weight_xc_O_0++;

  589.                 weight_xc_G_0++;

  590.                 weight_xc_I_1++;

  591.                 weight_xc_F_1++;

  592.                 weight_xc_O_1++;

  593.                 weight_xc_G_1++;

  594.             }

  595. 

  596.             for (; remain_num_output > 0; remain_num_output--)

  597.             {

  598.                 float h_cont = *hidden_ptr_r;

  599.                 sums[0] += *weight_hc_I_0 * h_cont;

  600.                 sums[1] += *weight_hc_F_0 * h_cont;

  601.                 sums[2] += *weight_hc_O_0 * h_cont;

  602.                 sums[3] += *weight_hc_G_0 * h_cont;

  603.                 sums[4] += *weight_hc_I_1 * h_cont;

  604.                 sums[5] += *weight_hc_F_1 * h_cont;

  605.                 sums[6] += *weight_hc_O_1 * h_cont;

  606.                 sums[7] += *weight_hc_G_1 * h_cont;

  607.                 hidden_ptr_r++;

  608.                 weight_hc_I_0++;

  609.                 weight_hc_F_0++;

  610.                 weight_hc_O_0++;

  611.                 weight_hc_G_0++;

  612.                 weight_hc_I_1++;

  613.                 weight_hc_F_1++;

  614.                 weight_hc_O_1++;

  615.                 weight_hc_G_1++;

  616.             }

  617.             gates_data_I[q] = sums[0];

  618.             gates_data_F[q] = sums[1];

  619.             gates_data_O[q] = sums[2];

  620.             gates_data_G[q] = sums[3];

  621.             gates_data_I[q + 1] = sums[4];

  622.             gates_data_F[q + 1] = sums[5];

  623.             gates_data_O[q + 1] = sums[6];

  624.             gates_data_G[q + 1] = sums[7];

  625.         }

  626. 

  627.         for (int q = remain_output; q < num_output; q++)

  628.         {

  629.             const float* x = bottom_blob.row(ti);

  630.             const float* hidden_ptr_r = hidden_state;

  631.             const float* bias_c_I = bias_c.row(0);

  632.             const float* bias_c_F = bias_c.row(1);

  633.             const float* bias_c_O = bias_c.row(2);

  634.             const float* bias_c_G = bias_c.row(3);

  635. 

  636.             float* gates_data_I = gates.row(0);

  637.             float* gates_data_F = gates.row(1);

  638.             float* gates_data_O = gates.row(2);

  639.             float* gates_data_G = gates.row(3);

  640.             // gate I F O G

  641.             const float* weight_xc_I = weight_xc.row(num_output * 0 + q);

  642.             const float* weight_xc_F = weight_xc.row(num_output * 1 + q);

  643.             const float* weight_xc_O = weight_xc.row(num_output * 2 + q);

  644.             const float* weight_xc_G = weight_xc.row(num_output * 3 + q);

  645. 

  646.             const float* weight_hc_I = weight_hc.row(num_output * 0 + q);

  647.             const float* weight_hc_F = weight_hc.row(num_output * 1 + q);

  648.             const float* weight_hc_O = weight_hc.row(num_output * 2 + q);

  649.             const float* weight_hc_G = weight_hc.row(num_output * 3 + q);

  650. 

  651.             // float I = bias_c_I[q];

  652.             // float F = bias_c_F[q];

  653.             // float O = bias_c_O[q];

  654.             // float G = bias_c_G[q];

  655.             __m256 _sumI = _mm256_setzero_ps();

  656.             __m256 _sumF = _mm256_setzero_ps();

  657.             __m256 _sumO = _mm256_setzero_ps();

  658.             __m256 _sumG = _mm256_setzero_ps();

  659.             int nn_num_size = size >> 3;

  660.             int remain_size = size & 7;

  661.             for (; nn_num_size > 0; nn_num_size--)

  662.             {

  663.                 __m256 xi = _mm256_loadu_ps(x);

  664.                 _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I), xi, _sumI);

  665.                 _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F), xi, _sumF);

  666.                 _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O), xi, _sumO);

  667.                 _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G), xi, _sumG);

  668.                 x += 8;

  669.                 weight_xc_I += 8;

  670.                 weight_xc_F += 8;

  671.                 weight_xc_O += 8;

  672.                 weight_xc_G += 8;

  673.             }

  674.             int nn_num_output = num_output >> 3;

  675.             int remain_num_output = num_output & 7;

  676.             for (; nn_num_output > 0; nn_num_output--)

  677.             {

  678.                 __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r);

  679. 

  680.                 _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I), h_cont, _sumI);

  681.                 _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F), h_cont, _sumF);

  682.                 _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O), h_cont, _sumO);

  683.                 _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G), h_cont, _sumG);

  684.                 hidden_ptr_r += 8;

  685.                 weight_hc_I += 8;

  686.                 weight_hc_F += 8;

  687.                 weight_hc_O += 8;

  688.                 weight_hc_G += 8;

  689.             }

  690.             float sums[4];

  691.             _mm_storeu_ps(sums, HorizontalSums(_sumI, _sumF, _sumO, _sumG));

  692.             sums[0] += bias_c_I[q];

  693.             sums[1] += bias_c_F[q];

  694.             sums[2] += bias_c_O[q];

  695.             sums[3] += bias_c_G[q];

  696. 

  697.             for (; remain_size > 0; remain_size--)

  698.             {

  699.                 float xi = *x;

  700.                 sums[0] += *weight_xc_I * xi;

  701.                 sums[1] += *weight_xc_F * xi;

  702.                 sums[2] += *weight_xc_O * xi;

  703.                 sums[3] += *weight_xc_G * xi;

  704.                 x++;

  705.                 weight_xc_I++;

  706.                 weight_xc_F++;

  707.                 weight_xc_O++;

  708.                 weight_xc_G++;

  709.             }

  710. 

  711.             for (; remain_num_output > 0; remain_num_output--)

  712.             {

  713.                 float h_cont = *hidden_ptr_r;

  714.                 sums[0] += *weight_hc_I * h_cont;

  715.                 sums[1] += *weight_hc_F * h_cont;

  716.                 sums[2] += *weight_hc_O * h_cont;

  717.                 sums[3] += *weight_hc_G * h_cont;

  718.                 hidden_ptr_r++;

  719.                 weight_hc_I++;

  720.                 weight_hc_F++;

  721.                 weight_hc_O++;

  722.                 weight_hc_G++;

  723.             }

  724.             gates_data_I[q] = sums[0];

  725.             gates_data_F[q] = sums[1];

  726.             gates_data_O[q] = sums[2];

  727.             gates_data_G[q] = sums[3];

  728.         }

  729. 

  730.         // lstm unit

  731.         // sigmoid(I)

  732.         // sigmoid(F)

  733.         // sigmoid(O)

  734.         // tanh(G)

  735.         // c_t := f_t .* c_{t-1} + i_t .* g_t

  736.         // h_t := o_t .* tanh[c_t]

  737.         float* output_data = top_blob.row(ti);

  738.         float* cell_ptr = cell_state;

  739.         float* hidden_ptr = hidden_state;

  740.         const float* gates_data_I = gates.row(0);

  741.         const float* gates_data_F = gates.row(1);

  742.         const float* gates_data_O = gates.row(2);

  743.         const float* gates_data_G = gates.row(3);

  744.         int nn_activation = num_output >> 3;

  745.         int remain_activations = num_output & 7;

  746.         for (; nn_activation > 0; nn_activation--)

  747.         {

  748.             __m256 I = sigmoid_avx(_mm256_loadu_ps(gates_data_I));

  749.             __m256 F = sigmoid_avx(_mm256_loadu_ps(gates_data_F));

  750.             __m256 O = sigmoid_avx(_mm256_loadu_ps(gates_data_O));

  751.             __m256 G = tanh_avx(_mm256_loadu_ps(gates_data_G));

  752.             __m256 cell2 = _mm256_add_ps(_mm256_mul_ps(F, _mm256_loadu_ps(cell_ptr)), _mm256_mul_ps(I, G));

  753.             __m256 H = _mm256_mul_ps(O, tanh_avx(cell2));

  754.             _mm256_storeu_ps(cell_ptr, cell2);

  755.             _mm256_storeu_ps(hidden_ptr, H);

  756.             _mm256_storeu_ps(output_data, H);

  757.             cell_ptr += 8;

  758.             output_data += 8;

  759.             hidden_ptr += 8;

  760.             gates_data_I += 8;

  761.             gates_data_F += 8;

  762.             gates_data_O += 8;

  763.             gates_data_G += 8;

  764.         }

  765.         for (; remain_activations > 0; remain_activations--)

  766.         {

  767.             float I = *gates_data_I;

  768.             float F = *gates_data_F;

  769.             float O = *gates_data_O;

  770.             float G = *gates_data_G;

  771. 

  772.             I = 1.f / (1.f + exp(-I));

  773.             F = 1.f / (1.f + exp(-F));

  774.             O = 1.f / (1.f + exp(-O));

  775.             G = tanh(G);

  776.             float cell2 = F * *cell_ptr + I * G;

  777.             float H = O * tanh(cell2);

  778.             *cell_ptr = cell2;

  779.             *hidden_ptr = H;

  780.             *output_data = H;

  781.             cell_ptr++;

  782.             output_data++;

  783.             hidden_ptr++;

  784.             gates_data_I++;

  785.             gates_data_F++;

  786.             gates_data_O++;

  787.             gates_data_G++;

  788.         }

  789. 

  790.         // no cell output here

  791.     }

  792. 

  793.     return 0;

  794. }

  795. #endif

  796. 

  797. int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const

  798. {

  799. #if __AVX__

  800.     int T = bottom_blob.h;

  801.     int num_directions = direction == 2 ? 2 : 1;

  802. 

  803.     // initial hidden state

  804.     Mat hidden(num_output, 4u, opt.workspace_allocator);

  805.     if (hidden.empty())

  806.         return -100;

  807.     hidden.fill(0.f);

  808.     // internal cell state

  809.     Mat cell(num_output, 4u, opt.workspace_allocator);

  810.     if (cell.empty())

  811.         return -100;

  812.     cell.fill(0.f);

  813. 

  814.     top_blob.create(num_output * num_directions, T, 4u, opt.blob_allocator);

  815.     if (top_blob.empty())

  816.         return -100;

  817. 

  818.     // Uni directional

  819.     if (direction == 0 || direction == 1)

  820.     {

  821.         if (opt.use_weight_fp16_storage)

  822.         {

  823.             // Uni directional

  824.             int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);

  825.             if (ret != 0)

  826.                 return ret;

  827.         }

  828.         else

  829.         {

  830.             // Uni directional

  831.             int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);

  832.             if (ret != 0)

  833.                 return ret;

  834.         }

  835.     }

  836. 

  837.     if (direction == 2)

  838.     {

  839.         Mat top_blob_forward(num_output, T, 4u, opt.workspace_allocator);

  840.         if (top_blob_forward.empty())

  841.             return -100;

  842. 

  843.         Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator);

  844.         if (top_blob_reverse.empty())

  845.             return -100;

  846. 

  847.         if (opt.use_weight_fp16_storage)

  848.         {

  849.             // Uni directional

  850.             int ret0 = lstm_fp16(bottom_blob, top_blob_forward, 0, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden, cell, opt);

  851.             if (ret0 != 0)

  852.                 return ret0;

  853.         }

  854.         else

  855.         {

  856.             // Uni directional

  857.             int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt);

  858.             if (ret0 != 0)

  859.                 return ret0;

  860.         }

  861. 

  862.         hidden.fill(0.0f);

  863.         cell.fill(0.0f);

  864.         if (opt.use_weight_fp16_storage)

  865.         {

  866.             // Uni directional

  867.             int ret1 = lstm_fp16(bottom_blob, top_blob_reverse, 1, weight_xc_data_fp16.channel(1), bias_c_data.channel(1), weight_hc_data_fp16.channel(1), hidden, cell, opt);

  868.             if (ret1 != 0)

  869.                 return ret1;

  870.         }

  871.         else

  872.         {

  873.             // Uni directional

  874.             int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt);

  875.             if (ret1 != 0)

  876.                 return ret1;

  877.         }

  878. 

  879.         // concat w

  880.         for (int i = 0; i < T; i++)

  881.         {

  882.             const float* pf = top_blob_forward.row(i);

  883.             const float* pr = top_blob_reverse.row(i);

  884.             float* ptr = top_blob.row(i);

  885. 

  886.             memcpy(ptr, pf, num_output * sizeof(float));

  887.             memcpy(ptr + num_output, pr, num_output * sizeof(float));

  888.         }

  889.     }

  890. 

  891.     return 0;

  892. #else

  893.     return LSTM::forward(bottom_blob, top_blob, opt);

  894. #endif

  895. }

  896. 

  897. int LSTM_x86::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const

  898. {

  899. #if __AVX__

  900.     if (bottom_blobs.size() != 3 || top_blobs.size() != 3)

  901.     {

  902.         return forward(bottom_blobs[0], top_blobs[0], opt);

  903.     }

  904.     const Mat& bottom_blob = bottom_blobs[0];

  905. 

  906.     int T = bottom_blob.h;

  907.     Mat& top_blob = top_blobs[0];

  908.     Mat& hidden_state = top_blobs[1];

  909.     Mat& cell_state = top_blobs[2];

  910. 

  911.     //Copy previous states

  912.     hidden_state = bottom_blobs[1].clone(opt.blob_allocator);

  913.     cell_state = bottom_blobs[2].clone(opt.blob_allocator);

  914. 

  915.     top_blob.create(num_output, T, 4u, opt.blob_allocator);

  916.     if (top_blob.empty())

  917.         return -100;

  918. 

  919.     if (opt.use_weight_fp16_storage)

  920.     {

  921.         // Uni directional

  922.         int ret = lstm_fp16(bottom_blob, top_blob, direction, weight_xc_data_fp16.channel(0), bias_c_data.channel(0), weight_hc_data_fp16.channel(0), hidden_state, cell_state, opt);

  923.         if (ret != 0)

  924.             return ret;

  925.     }

  926.     else

  927.     {

  928.         // Uni directional

  929.         int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt);

  930.         if (ret != 0)

  931.             return ret;

  932.     }

  933.     return 0;

  934. #else

  935.     return LSTM::forward(bottom_blobs, top_blobs, opt);

  936. #endif

  937. }

  938. 

  939. } // namespace ncnn