OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/riscv64/strmm_kernel_8x8_zvl128b.c

992 lines
30 kB
Raw Permalink Blame History 

  1. /*

  2. 

  3. AUTOGENERATED KERNEL

  4. Script: ./kernel/riscv64/generate_kernel.py

  5. Settings:

  6.  LMUL=2

  7.  M=8

  8.  M_tail_scalar_from=2

  9.  N=8

  10.  __riscv_='__riscv_'

  11.  complex=False

  12.  conjugate=False

  13.  cpu='zvl128b'

  14.  force_acc_double=False

  15.  index_type='BLASLONG'

  16.  op='trmm'

  17.  param_precision='float'

  18.  reg_width_bits=128

  19.  tail_policy=''

  20.  trace=False

  21. 

  22. Derived:

  23.  ELEN_ACC=32

  24.  ELEN_PARAM=32

  25.  LMUL_ACC=2

  26.  VFMACC='__riscv_vfmacc_vf_f32m2'

  27.  VFMUL='__riscv_vfmul_vf_f32m2'

  28.  VLEV='__riscv_vle32_v_f32m2'

  29.  VLSEV='__riscv_vlse32_v_f32m2'

  30.  VMACC_TO_ACC='__riscv_vfmacc_vf_f32m2'

  31.  VMUL_TO_ACC='__riscv_vfmul_vf_f32m2'

  32.  VSETVL='__riscv_vsetvl_e32m2'

  33.  VSEV='__riscv_vse32_v_f32m2'

  34.  VSSEV='__riscv_vsse32_v_f32m2'

  35.  acc_vector_t='vfloat32m2_t'

  36.  output='strmm_kernel_8x8_zvl128b.c'

  37.  param_scalar_t='float'

  38.  param_vector_t='vfloat32m2_t'

  39. 

  40. */

  41. 

  42. #include "common.h"

  43. 

  44. #if defined(LEFT) != defined(TRANSA)

  45. #define BACKWARDS

  46. #endif

  47. 

  48. int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT *A, FLOAT *B, FLOAT *C, BLASLONG ldc, BLASLONG offset)

  49. 

  50. {

  51.     BLASLONG gvl = 0;

  52.     BLASLONG m_top = 0;

  53.     BLASLONG n_top = 0;

  54. 

  55.     // -- MAIN PASS

  56. 

  57.     for (BLASLONG j = 0; j < N / 8; j += 1) {

  58.         m_top = 0;

  59.         BLASLONG gvl = __riscv_vsetvl_e32m2(8);

  60. 

  61.         for (BLASLONG i = 0; i < M / 8; i += 1) {

  62.             BLASLONG ai = m_top * K;

  63.             BLASLONG bi = n_top * K;

  64.             BLASLONG pass_K = K;

  65. #ifdef LEFT

  66.             BLASLONG off = offset + m_top;

  67. #else

  68.             BLASLONG off = -offset + n_top;

  69. #endif

  70. #ifdef BACKWARDS

  71.             ai += off * 8;

  72.             bi += off * 8;

  73.             pass_K -= off;

  74. #else

  75. #ifdef LEFT

  76.             pass_K = off + 8;

  77. #else

  78.             pass_K = off + 8;

  79. #endif

  80. #endif

  81.             float B0 = B[bi + 0];

  82.             float B1 = B[bi + 1];

  83.             float B2 = B[bi + 2];

  84.             float B3 = B[bi + 3];

  85.             float B4 = B[bi + 4];

  86.             float B5 = B[bi + 5];

  87.             float B6 = B[bi + 6];

  88.             float B7 = B[bi + 7];

  89.             bi += 8;

  90. 

  91.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  92.             ai += 8;

  93. 

  94.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  95.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  96.             vfloat32m2_t result2 = __riscv_vfmul_vf_f32m2(A0, B2, gvl);

  97.             vfloat32m2_t result3 = __riscv_vfmul_vf_f32m2(A0, B3, gvl);

  98.             vfloat32m2_t result4 = __riscv_vfmul_vf_f32m2(A0, B4, gvl);

  99.             vfloat32m2_t result5 = __riscv_vfmul_vf_f32m2(A0, B5, gvl);

  100.             vfloat32m2_t result6 = __riscv_vfmul_vf_f32m2(A0, B6, gvl);

  101.             vfloat32m2_t result7 = __riscv_vfmul_vf_f32m2(A0, B7, gvl);

  102. 

  103.             for (BLASLONG k = 1; k < pass_K; k++) {

  104.                 B0 = B[bi + 0];

  105.                 B1 = B[bi + 1];

  106.                 B2 = B[bi + 2];

  107.                 B3 = B[bi + 3];

  108.                 B4 = B[bi + 4];

  109.                 B5 = B[bi + 5];

  110.                 B6 = B[bi + 6];

  111.                 B7 = B[bi + 7];

  112.                 bi += 8;

  113. 

  114.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  115.                 ai += 8;

  116. 

  117.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  118.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  119.                 result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);

  120.                 result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);

  121.                 result4 = __riscv_vfmacc_vf_f32m2(result4, B4, A0, gvl);

  122.                 result5 = __riscv_vfmacc_vf_f32m2(result5, B5, A0, gvl);

  123.                 result6 = __riscv_vfmacc_vf_f32m2(result6, B6, A0, gvl);

  124.                 result7 = __riscv_vfmacc_vf_f32m2(result7, B7, A0, gvl);

  125.             }

  126. 

  127.             BLASLONG ci = n_top * ldc + m_top;

  128. 

  129.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  130.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  131.             vfloat32m2_t c2 = __riscv_vfmul_vf_f32m2(result2, alpha, gvl);

  132.             vfloat32m2_t c3 = __riscv_vfmul_vf_f32m2(result3, alpha, gvl);

  133.             vfloat32m2_t c4 = __riscv_vfmul_vf_f32m2(result4, alpha, gvl);

  134.             vfloat32m2_t c5 = __riscv_vfmul_vf_f32m2(result5, alpha, gvl);

  135.             vfloat32m2_t c6 = __riscv_vfmul_vf_f32m2(result6, alpha, gvl);

  136.             vfloat32m2_t c7 = __riscv_vfmul_vf_f32m2(result7, alpha, gvl);

  137.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  138.             ci += ldc - gvl * 0;

  139.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  140.             ci += ldc - gvl * 0;

  141.             __riscv_vse32_v_f32m2(&C[ci], c2, gvl);

  142.             ci += ldc - gvl * 0;

  143.             __riscv_vse32_v_f32m2(&C[ci], c3, gvl);

  144.             ci += ldc - gvl * 0;

  145.             __riscv_vse32_v_f32m2(&C[ci], c4, gvl);

  146.             ci += ldc - gvl * 0;

  147.             __riscv_vse32_v_f32m2(&C[ci], c5, gvl);

  148.             ci += ldc - gvl * 0;

  149.             __riscv_vse32_v_f32m2(&C[ci], c6, gvl);

  150.             ci += ldc - gvl * 0;

  151.             __riscv_vse32_v_f32m2(&C[ci], c7, gvl);

  152.             m_top += 8;

  153.         }

  154. 

  155.         // -- tails for main pass

  156. 

  157.         if (M & 4) {

  158.             gvl = __riscv_vsetvl_e32m2(4);

  159. 

  160.             BLASLONG ai = m_top * K;

  161.             BLASLONG bi = n_top * K;

  162.             BLASLONG pass_K = K;

  163. #ifdef LEFT

  164.             BLASLONG off = offset + m_top;

  165. #else

  166.             BLASLONG off = -offset + n_top;

  167. #endif

  168. #ifdef BACKWARDS

  169.             ai += off * 4;

  170.             bi += off * 8;

  171.             pass_K -= off;

  172. #else

  173. #ifdef LEFT

  174.             pass_K = off + 4;

  175. #else

  176.             pass_K = off + 8;

  177. #endif

  178. #endif

  179.             float B0 = B[bi + 0];

  180.             float B1 = B[bi + 1];

  181.             float B2 = B[bi + 2];

  182.             float B3 = B[bi + 3];

  183.             float B4 = B[bi + 4];

  184.             float B5 = B[bi + 5];

  185.             float B6 = B[bi + 6];

  186.             float B7 = B[bi + 7];

  187.             bi += 8;

  188. 

  189.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  190.             ai += 4;

  191. 

  192.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  193.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  194.             vfloat32m2_t result2 = __riscv_vfmul_vf_f32m2(A0, B2, gvl);

  195.             vfloat32m2_t result3 = __riscv_vfmul_vf_f32m2(A0, B3, gvl);

  196.             vfloat32m2_t result4 = __riscv_vfmul_vf_f32m2(A0, B4, gvl);

  197.             vfloat32m2_t result5 = __riscv_vfmul_vf_f32m2(A0, B5, gvl);

  198.             vfloat32m2_t result6 = __riscv_vfmul_vf_f32m2(A0, B6, gvl);

  199.             vfloat32m2_t result7 = __riscv_vfmul_vf_f32m2(A0, B7, gvl);

  200. 

  201.             for (BLASLONG k = 1; k < pass_K; k++) {

  202.                 B0 = B[bi + 0];

  203.                 B1 = B[bi + 1];

  204.                 B2 = B[bi + 2];

  205.                 B3 = B[bi + 3];

  206.                 B4 = B[bi + 4];

  207.                 B5 = B[bi + 5];

  208.                 B6 = B[bi + 6];

  209.                 B7 = B[bi + 7];

  210.                 bi += 8;

  211. 

  212.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  213.                 ai += 4;

  214. 

  215.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  216.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  217.                 result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);

  218.                 result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);

  219.                 result4 = __riscv_vfmacc_vf_f32m2(result4, B4, A0, gvl);

  220.                 result5 = __riscv_vfmacc_vf_f32m2(result5, B5, A0, gvl);

  221.                 result6 = __riscv_vfmacc_vf_f32m2(result6, B6, A0, gvl);

  222.                 result7 = __riscv_vfmacc_vf_f32m2(result7, B7, A0, gvl);

  223.             }

  224. 

  225.             BLASLONG ci = n_top * ldc + m_top;

  226. 

  227.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  228.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  229.             vfloat32m2_t c2 = __riscv_vfmul_vf_f32m2(result2, alpha, gvl);

  230.             vfloat32m2_t c3 = __riscv_vfmul_vf_f32m2(result3, alpha, gvl);

  231.             vfloat32m2_t c4 = __riscv_vfmul_vf_f32m2(result4, alpha, gvl);

  232.             vfloat32m2_t c5 = __riscv_vfmul_vf_f32m2(result5, alpha, gvl);

  233.             vfloat32m2_t c6 = __riscv_vfmul_vf_f32m2(result6, alpha, gvl);

  234.             vfloat32m2_t c7 = __riscv_vfmul_vf_f32m2(result7, alpha, gvl);

  235.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  236.             ci += ldc - gvl * 0;

  237.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  238.             ci += ldc - gvl * 0;

  239.             __riscv_vse32_v_f32m2(&C[ci], c2, gvl);

  240.             ci += ldc - gvl * 0;

  241.             __riscv_vse32_v_f32m2(&C[ci], c3, gvl);

  242.             ci += ldc - gvl * 0;

  243.             __riscv_vse32_v_f32m2(&C[ci], c4, gvl);

  244.             ci += ldc - gvl * 0;

  245.             __riscv_vse32_v_f32m2(&C[ci], c5, gvl);

  246.             ci += ldc - gvl * 0;

  247.             __riscv_vse32_v_f32m2(&C[ci], c6, gvl);

  248.             ci += ldc - gvl * 0;

  249.             __riscv_vse32_v_f32m2(&C[ci], c7, gvl);

  250.             m_top += 4;

  251.         }

  252. 

  253.         if (M & 2) {

  254.             float result0 = 0;

  255.             float result1 = 0;

  256.             float result2 = 0;

  257.             float result3 = 0;

  258.             float result4 = 0;

  259.             float result5 = 0;

  260.             float result6 = 0;

  261.             float result7 = 0;

  262.             float result8 = 0;

  263.             float result9 = 0;

  264.             float result10 = 0;

  265.             float result11 = 0;

  266.             float result12 = 0;

  267.             float result13 = 0;

  268.             float result14 = 0;

  269.             float result15 = 0;

  270.             BLASLONG ai = m_top * K;

  271.             BLASLONG bi = n_top * K;

  272.             BLASLONG pass_K = K;

  273. #ifdef LEFT

  274.             BLASLONG off = offset + m_top;

  275. #else

  276.             BLASLONG off = -offset + n_top;

  277. #endif

  278. #ifdef BACKWARDS

  279.             ai += off * 2;

  280.             bi += off * 8;

  281.             pass_K -= off;

  282. #else

  283. #ifdef LEFT

  284.             pass_K = off + 2;

  285. #else

  286.             pass_K = off + 8;

  287. #endif

  288. #endif

  289. 

  290.             for (BLASLONG k = 0; k < pass_K; k++) {

  291.                 result0 += A[ai + 0] * B[bi + 0];

  292.                 result1 += A[ai + 1] * B[bi + 0];

  293.                 result2 += A[ai + 0] * B[bi + 1];

  294.                 result3 += A[ai + 1] * B[bi + 1];

  295.                 result4 += A[ai + 0] * B[bi + 2];

  296.                 result5 += A[ai + 1] * B[bi + 2];

  297.                 result6 += A[ai + 0] * B[bi + 3];

  298.                 result7 += A[ai + 1] * B[bi + 3];

  299.                 result8 += A[ai + 0] * B[bi + 4];

  300.                 result9 += A[ai + 1] * B[bi + 4];

  301.                 result10 += A[ai + 0] * B[bi + 5];

  302.                 result11 += A[ai + 1] * B[bi + 5];

  303.                 result12 += A[ai + 0] * B[bi + 6];

  304.                 result13 += A[ai + 1] * B[bi + 6];

  305.                 result14 += A[ai + 0] * B[bi + 7];

  306.                 result15 += A[ai + 1] * B[bi + 7];

  307.                 ai += 2;

  308.                 bi += 8;

  309.             }

  310. 

  311.             BLASLONG ci = n_top * ldc + m_top;

  312.             C[ci + 0 * ldc + 0] = alpha * result0;

  313.             C[ci + 0 * ldc + 1] = alpha * result1;

  314.             C[ci + 1 * ldc + 0] = alpha * result2;

  315.             C[ci + 1 * ldc + 1] = alpha * result3;

  316.             C[ci + 2 * ldc + 0] = alpha * result4;

  317.             C[ci + 2 * ldc + 1] = alpha * result5;

  318.             C[ci + 3 * ldc + 0] = alpha * result6;

  319.             C[ci + 3 * ldc + 1] = alpha * result7;

  320.             C[ci + 4 * ldc + 0] = alpha * result8;

  321.             C[ci + 4 * ldc + 1] = alpha * result9;

  322.             C[ci + 5 * ldc + 0] = alpha * result10;

  323.             C[ci + 5 * ldc + 1] = alpha * result11;

  324.             C[ci + 6 * ldc + 0] = alpha * result12;

  325.             C[ci + 6 * ldc + 1] = alpha * result13;

  326.             C[ci + 7 * ldc + 0] = alpha * result14;

  327.             C[ci + 7 * ldc + 1] = alpha * result15;

  328.             m_top += 2;

  329.         }

  330. 

  331.         if (M & 1) {

  332.             float result0 = 0;

  333.             float result1 = 0;

  334.             float result2 = 0;

  335.             float result3 = 0;

  336.             float result4 = 0;

  337.             float result5 = 0;

  338.             float result6 = 0;

  339.             float result7 = 0;

  340.             BLASLONG ai = m_top * K;

  341.             BLASLONG bi = n_top * K;

  342.             BLASLONG pass_K = K;

  343. #ifdef LEFT

  344.             BLASLONG off = offset + m_top;

  345. #else

  346.             BLASLONG off = -offset + n_top;

  347. #endif

  348. #ifdef BACKWARDS

  349.             ai += off * 1;

  350.             bi += off * 8;

  351.             pass_K -= off;

  352. #else

  353. #ifdef LEFT

  354.             pass_K = off + 1;

  355. #else

  356.             pass_K = off + 8;

  357. #endif

  358. #endif

  359. 

  360.             for (BLASLONG k = 0; k < pass_K; k++) {

  361.                 result0 += A[ai + 0] * B[bi + 0];

  362.                 result1 += A[ai + 0] * B[bi + 1];

  363.                 result2 += A[ai + 0] * B[bi + 2];

  364.                 result3 += A[ai + 0] * B[bi + 3];

  365.                 result4 += A[ai + 0] * B[bi + 4];

  366.                 result5 += A[ai + 0] * B[bi + 5];

  367.                 result6 += A[ai + 0] * B[bi + 6];

  368.                 result7 += A[ai + 0] * B[bi + 7];

  369.                 ai += 1;

  370.                 bi += 8;

  371.             }

  372. 

  373.             BLASLONG ci = n_top * ldc + m_top;

  374.             C[ci + 0 * ldc + 0] = alpha * result0;

  375.             C[ci + 1 * ldc + 0] = alpha * result1;

  376.             C[ci + 2 * ldc + 0] = alpha * result2;

  377.             C[ci + 3 * ldc + 0] = alpha * result3;

  378.             C[ci + 4 * ldc + 0] = alpha * result4;

  379.             C[ci + 5 * ldc + 0] = alpha * result5;

  380.             C[ci + 6 * ldc + 0] = alpha * result6;

  381.             C[ci + 7 * ldc + 0] = alpha * result7;

  382.             m_top += 1;

  383.         }

  384. 

  385.         n_top += 8;

  386.     }

  387. 

  388.     // -- tails for N=4

  389. 

  390.     if (N & 4) {

  391.         gvl = __riscv_vsetvl_e32m2(8);

  392.         m_top = 0;

  393. 

  394.         for (BLASLONG i = 0; i < M / 8; i += 1) {

  395.             BLASLONG ai = m_top * K;

  396.             BLASLONG bi = n_top * K;

  397.             BLASLONG pass_K = K;

  398. #ifdef LEFT

  399.             BLASLONG off = offset + m_top;

  400. #else

  401.             BLASLONG off = -offset + n_top;

  402. #endif

  403. #ifdef BACKWARDS

  404.             ai += off * 8;

  405.             bi += off * 4;

  406.             pass_K -= off;

  407. #else

  408. #ifdef LEFT

  409.             pass_K = off + 8;

  410. #else

  411.             pass_K = off + 4;

  412. #endif

  413. #endif

  414.             float B0 = B[bi + 0];

  415.             float B1 = B[bi + 1];

  416.             float B2 = B[bi + 2];

  417.             float B3 = B[bi + 3];

  418.             bi += 4;

  419. 

  420.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  421.             ai += 8;

  422. 

  423.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  424.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  425.             vfloat32m2_t result2 = __riscv_vfmul_vf_f32m2(A0, B2, gvl);

  426.             vfloat32m2_t result3 = __riscv_vfmul_vf_f32m2(A0, B3, gvl);

  427. 

  428.             for (BLASLONG k = 1; k < pass_K; k++) {

  429.                 B0 = B[bi + 0];

  430.                 B1 = B[bi + 1];

  431.                 B2 = B[bi + 2];

  432.                 B3 = B[bi + 3];

  433.                 bi += 4;

  434. 

  435.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  436.                 ai += 8;

  437. 

  438.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  439.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  440.                 result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);

  441.                 result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);

  442.             }

  443. 

  444.             BLASLONG ci = n_top * ldc + m_top;

  445. 

  446.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  447.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  448.             vfloat32m2_t c2 = __riscv_vfmul_vf_f32m2(result2, alpha, gvl);

  449.             vfloat32m2_t c3 = __riscv_vfmul_vf_f32m2(result3, alpha, gvl);

  450.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  451.             ci += ldc - gvl * 0;

  452.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  453.             ci += ldc - gvl * 0;

  454.             __riscv_vse32_v_f32m2(&C[ci], c2, gvl);

  455.             ci += ldc - gvl * 0;

  456.             __riscv_vse32_v_f32m2(&C[ci], c3, gvl);

  457.             m_top += 8;

  458.         }

  459. 

  460.         if (M & 4) {

  461.             gvl = __riscv_vsetvl_e32m2(4);

  462. 

  463.             BLASLONG ai = m_top * K;

  464.             BLASLONG bi = n_top * K;

  465.             BLASLONG pass_K = K;

  466. #ifdef LEFT

  467.             BLASLONG off = offset + m_top;

  468. #else

  469.             BLASLONG off = -offset + n_top;

  470. #endif

  471. #ifdef BACKWARDS

  472.             ai += off * 4;

  473.             bi += off * 4;

  474.             pass_K -= off;

  475. #else

  476. #ifdef LEFT

  477.             pass_K = off + 4;

  478. #else

  479.             pass_K = off + 4;

  480. #endif

  481. #endif

  482.             float B0 = B[bi + 0];

  483.             float B1 = B[bi + 1];

  484.             float B2 = B[bi + 2];

  485.             float B3 = B[bi + 3];

  486.             bi += 4;

  487. 

  488.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  489.             ai += 4;

  490. 

  491.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  492.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  493.             vfloat32m2_t result2 = __riscv_vfmul_vf_f32m2(A0, B2, gvl);

  494.             vfloat32m2_t result3 = __riscv_vfmul_vf_f32m2(A0, B3, gvl);

  495. 

  496.             for (BLASLONG k = 1; k < pass_K; k++) {

  497.                 B0 = B[bi + 0];

  498.                 B1 = B[bi + 1];

  499.                 B2 = B[bi + 2];

  500.                 B3 = B[bi + 3];

  501.                 bi += 4;

  502. 

  503.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  504.                 ai += 4;

  505. 

  506.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  507.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  508.                 result2 = __riscv_vfmacc_vf_f32m2(result2, B2, A0, gvl);

  509.                 result3 = __riscv_vfmacc_vf_f32m2(result3, B3, A0, gvl);

  510.             }

  511. 

  512.             BLASLONG ci = n_top * ldc + m_top;

  513. 

  514.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  515.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  516.             vfloat32m2_t c2 = __riscv_vfmul_vf_f32m2(result2, alpha, gvl);

  517.             vfloat32m2_t c3 = __riscv_vfmul_vf_f32m2(result3, alpha, gvl);

  518.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  519.             ci += ldc - gvl * 0;

  520.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  521.             ci += ldc - gvl * 0;

  522.             __riscv_vse32_v_f32m2(&C[ci], c2, gvl);

  523.             ci += ldc - gvl * 0;

  524.             __riscv_vse32_v_f32m2(&C[ci], c3, gvl);

  525.             m_top += 4;

  526.         }

  527. 

  528.         if (M & 2) {

  529.             float result0 = 0;

  530.             float result1 = 0;

  531.             float result2 = 0;

  532.             float result3 = 0;

  533.             float result4 = 0;

  534.             float result5 = 0;

  535.             float result6 = 0;

  536.             float result7 = 0;

  537.             BLASLONG ai = m_top * K;

  538.             BLASLONG bi = n_top * K;

  539.             BLASLONG pass_K = K;

  540. #ifdef LEFT

  541.             BLASLONG off = offset + m_top;

  542. #else

  543.             BLASLONG off = -offset + n_top;

  544. #endif

  545. #ifdef BACKWARDS

  546.             ai += off * 2;

  547.             bi += off * 4;

  548.             pass_K -= off;

  549. #else

  550. #ifdef LEFT

  551.             pass_K = off + 2;

  552. #else

  553.             pass_K = off + 4;

  554. #endif

  555. #endif

  556. 

  557.             for (BLASLONG k = 0; k < pass_K; k++) {

  558.                 result0 += A[ai + 0] * B[bi + 0];

  559.                 result1 += A[ai + 1] * B[bi + 0];

  560.                 result2 += A[ai + 0] * B[bi + 1];

  561.                 result3 += A[ai + 1] * B[bi + 1];

  562.                 result4 += A[ai + 0] * B[bi + 2];

  563.                 result5 += A[ai + 1] * B[bi + 2];

  564.                 result6 += A[ai + 0] * B[bi + 3];

  565.                 result7 += A[ai + 1] * B[bi + 3];

  566.                 ai += 2;

  567.                 bi += 4;

  568.             }

  569. 

  570.             BLASLONG ci = n_top * ldc + m_top;

  571.             C[ci + 0 * ldc + 0] = alpha * result0;

  572.             C[ci + 0 * ldc + 1] = alpha * result1;

  573.             C[ci + 1 * ldc + 0] = alpha * result2;

  574.             C[ci + 1 * ldc + 1] = alpha * result3;

  575.             C[ci + 2 * ldc + 0] = alpha * result4;

  576.             C[ci + 2 * ldc + 1] = alpha * result5;

  577.             C[ci + 3 * ldc + 0] = alpha * result6;

  578.             C[ci + 3 * ldc + 1] = alpha * result7;

  579.             m_top += 2;

  580.         }

  581. 

  582.         if (M & 1) {

  583.             float result0 = 0;

  584.             float result1 = 0;

  585.             float result2 = 0;

  586.             float result3 = 0;

  587.             BLASLONG ai = m_top * K;

  588.             BLASLONG bi = n_top * K;

  589.             BLASLONG pass_K = K;

  590. #ifdef LEFT

  591.             BLASLONG off = offset + m_top;

  592. #else

  593.             BLASLONG off = -offset + n_top;

  594. #endif

  595. #ifdef BACKWARDS

  596.             ai += off * 1;

  597.             bi += off * 4;

  598.             pass_K -= off;

  599. #else

  600. #ifdef LEFT

  601.             pass_K = off + 1;

  602. #else

  603.             pass_K = off + 4;

  604. #endif

  605. #endif

  606. 

  607.             for (BLASLONG k = 0; k < pass_K; k++) {

  608.                 result0 += A[ai + 0] * B[bi + 0];

  609.                 result1 += A[ai + 0] * B[bi + 1];

  610.                 result2 += A[ai + 0] * B[bi + 2];

  611.                 result3 += A[ai + 0] * B[bi + 3];

  612.                 ai += 1;

  613.                 bi += 4;

  614.             }

  615. 

  616.             BLASLONG ci = n_top * ldc + m_top;

  617.             C[ci + 0 * ldc + 0] = alpha * result0;

  618.             C[ci + 1 * ldc + 0] = alpha * result1;

  619.             C[ci + 2 * ldc + 0] = alpha * result2;

  620.             C[ci + 3 * ldc + 0] = alpha * result3;

  621.             m_top += 1;

  622.         }

  623. 

  624.         n_top += 4;

  625.     }

  626. 

  627.     // -- tails for N=2

  628. 

  629.     if (N & 2) {

  630.         gvl = __riscv_vsetvl_e32m2(8);

  631.         m_top = 0;

  632. 

  633.         for (BLASLONG i = 0; i < M / 8; i += 1) {

  634.             BLASLONG ai = m_top * K;

  635.             BLASLONG bi = n_top * K;

  636.             BLASLONG pass_K = K;

  637. #ifdef LEFT

  638.             BLASLONG off = offset + m_top;

  639. #else

  640.             BLASLONG off = -offset + n_top;

  641. #endif

  642. #ifdef BACKWARDS

  643.             ai += off * 8;

  644.             bi += off * 2;

  645.             pass_K -= off;

  646. #else

  647. #ifdef LEFT

  648.             pass_K = off + 8;

  649. #else

  650.             pass_K = off + 2;

  651. #endif

  652. #endif

  653.             float B0 = B[bi + 0];

  654.             float B1 = B[bi + 1];

  655.             bi += 2;

  656. 

  657.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  658.             ai += 8;

  659. 

  660.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  661.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  662. 

  663.             for (BLASLONG k = 1; k < pass_K; k++) {

  664.                 B0 = B[bi + 0];

  665.                 B1 = B[bi + 1];

  666.                 bi += 2;

  667. 

  668.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  669.                 ai += 8;

  670. 

  671.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  672.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  673.             }

  674. 

  675.             BLASLONG ci = n_top * ldc + m_top;

  676. 

  677.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  678.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  679.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  680.             ci += ldc - gvl * 0;

  681.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  682.             m_top += 8;

  683.         }

  684. 

  685.         if (M & 4) {

  686.             gvl = __riscv_vsetvl_e32m2(4);

  687. 

  688.             BLASLONG ai = m_top * K;

  689.             BLASLONG bi = n_top * K;

  690.             BLASLONG pass_K = K;

  691. #ifdef LEFT

  692.             BLASLONG off = offset + m_top;

  693. #else

  694.             BLASLONG off = -offset + n_top;

  695. #endif

  696. #ifdef BACKWARDS

  697.             ai += off * 4;

  698.             bi += off * 2;

  699.             pass_K -= off;

  700. #else

  701. #ifdef LEFT

  702.             pass_K = off + 4;

  703. #else

  704.             pass_K = off + 2;

  705. #endif

  706. #endif

  707.             float B0 = B[bi + 0];

  708.             float B1 = B[bi + 1];

  709.             bi += 2;

  710. 

  711.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  712.             ai += 4;

  713. 

  714.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  715.             vfloat32m2_t result1 = __riscv_vfmul_vf_f32m2(A0, B1, gvl);

  716. 

  717.             for (BLASLONG k = 1; k < pass_K; k++) {

  718.                 B0 = B[bi + 0];

  719.                 B1 = B[bi + 1];

  720.                 bi += 2;

  721. 

  722.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  723.                 ai += 4;

  724. 

  725.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  726.                 result1 = __riscv_vfmacc_vf_f32m2(result1, B1, A0, gvl);

  727.             }

  728. 

  729.             BLASLONG ci = n_top * ldc + m_top;

  730. 

  731.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  732.             vfloat32m2_t c1 = __riscv_vfmul_vf_f32m2(result1, alpha, gvl);

  733.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  734.             ci += ldc - gvl * 0;

  735.             __riscv_vse32_v_f32m2(&C[ci], c1, gvl);

  736.             m_top += 4;

  737.         }

  738. 

  739.         if (M & 2) {

  740.             float result0 = 0;

  741.             float result1 = 0;

  742.             float result2 = 0;

  743.             float result3 = 0;

  744.             BLASLONG ai = m_top * K;

  745.             BLASLONG bi = n_top * K;

  746.             BLASLONG pass_K = K;

  747. #ifdef LEFT

  748.             BLASLONG off = offset + m_top;

  749. #else

  750.             BLASLONG off = -offset + n_top;

  751. #endif

  752. #ifdef BACKWARDS

  753.             ai += off * 2;

  754.             bi += off * 2;

  755.             pass_K -= off;

  756. #else

  757. #ifdef LEFT

  758.             pass_K = off + 2;

  759. #else

  760.             pass_K = off + 2;

  761. #endif

  762. #endif

  763. 

  764.             for (BLASLONG k = 0; k < pass_K; k++) {

  765.                 result0 += A[ai + 0] * B[bi + 0];

  766.                 result1 += A[ai + 1] * B[bi + 0];

  767.                 result2 += A[ai + 0] * B[bi + 1];

  768.                 result3 += A[ai + 1] * B[bi + 1];

  769.                 ai += 2;

  770.                 bi += 2;

  771.             }

  772. 

  773.             BLASLONG ci = n_top * ldc + m_top;

  774.             C[ci + 0 * ldc + 0] = alpha * result0;

  775.             C[ci + 0 * ldc + 1] = alpha * result1;

  776.             C[ci + 1 * ldc + 0] = alpha * result2;

  777.             C[ci + 1 * ldc + 1] = alpha * result3;

  778.             m_top += 2;

  779.         }

  780. 

  781.         if (M & 1) {

  782.             float result0 = 0;

  783.             float result1 = 0;

  784.             BLASLONG ai = m_top * K;

  785.             BLASLONG bi = n_top * K;

  786.             BLASLONG pass_K = K;

  787. #ifdef LEFT

  788.             BLASLONG off = offset + m_top;

  789. #else

  790.             BLASLONG off = -offset + n_top;

  791. #endif

  792. #ifdef BACKWARDS

  793.             ai += off * 1;

  794.             bi += off * 2;

  795.             pass_K -= off;

  796. #else

  797. #ifdef LEFT

  798.             pass_K = off + 1;

  799. #else

  800.             pass_K = off + 2;

  801. #endif

  802. #endif

  803. 

  804.             for (BLASLONG k = 0; k < pass_K; k++) {

  805.                 result0 += A[ai + 0] * B[bi + 0];

  806.                 result1 += A[ai + 0] * B[bi + 1];

  807.                 ai += 1;

  808.                 bi += 2;

  809.             }

  810. 

  811.             BLASLONG ci = n_top * ldc + m_top;

  812.             C[ci + 0 * ldc + 0] = alpha * result0;

  813.             C[ci + 1 * ldc + 0] = alpha * result1;

  814.             m_top += 1;

  815.         }

  816. 

  817.         n_top += 2;

  818.     }

  819. 

  820.     // -- tails for N=1

  821. 

  822.     if (N & 1) {

  823.         gvl = __riscv_vsetvl_e32m2(8);

  824.         m_top = 0;

  825. 

  826.         for (BLASLONG i = 0; i < M / 8; i += 1) {

  827.             BLASLONG ai = m_top * K;

  828.             BLASLONG bi = n_top * K;

  829.             BLASLONG pass_K = K;

  830. #ifdef LEFT

  831.             BLASLONG off = offset + m_top;

  832. #else

  833.             BLASLONG off = -offset + n_top;

  834. #endif

  835. #ifdef BACKWARDS

  836.             ai += off * 8;

  837.             bi += off * 1;

  838.             pass_K -= off;

  839. #else

  840. #ifdef LEFT

  841.             pass_K = off + 8;

  842. #else

  843.             pass_K = off + 1;

  844. #endif

  845. #endif

  846.             float B0 = B[bi + 0];

  847.             bi += 1;

  848. 

  849.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  850.             ai += 8;

  851. 

  852.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  853. 

  854.             for (BLASLONG k = 1; k < pass_K; k++) {

  855.                 B0 = B[bi + 0];

  856.                 bi += 1;

  857. 

  858.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  859.                 ai += 8;

  860. 

  861.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  862.             }

  863. 

  864.             BLASLONG ci = n_top * ldc + m_top;

  865. 

  866.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  867.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  868.             m_top += 8;

  869.         }

  870. 

  871.         if (M & 4) {

  872.             gvl = __riscv_vsetvl_e32m2(4);

  873. 

  874.             BLASLONG ai = m_top * K;

  875.             BLASLONG bi = n_top * K;

  876.             BLASLONG pass_K = K;

  877. #ifdef LEFT

  878.             BLASLONG off = offset + m_top;

  879. #else

  880.             BLASLONG off = -offset + n_top;

  881. #endif

  882. #ifdef BACKWARDS

  883.             ai += off * 4;

  884.             bi += off * 1;

  885.             pass_K -= off;

  886. #else

  887. #ifdef LEFT

  888.             pass_K = off + 4;

  889. #else

  890.             pass_K = off + 1;

  891. #endif

  892. #endif

  893.             float B0 = B[bi + 0];

  894.             bi += 1;

  895. 

  896.             vfloat32m2_t A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  897.             ai += 4;

  898. 

  899.             vfloat32m2_t result0 = __riscv_vfmul_vf_f32m2(A0, B0, gvl);

  900. 

  901.             for (BLASLONG k = 1; k < pass_K; k++) {

  902.                 B0 = B[bi + 0];

  903.                 bi += 1;

  904. 

  905.                 A0 = __riscv_vle32_v_f32m2(&A[ai + 0 * gvl], gvl);

  906.                 ai += 4;

  907. 

  908.                 result0 = __riscv_vfmacc_vf_f32m2(result0, B0, A0, gvl);

  909.             }

  910. 

  911.             BLASLONG ci = n_top * ldc + m_top;

  912. 

  913.             vfloat32m2_t c0 = __riscv_vfmul_vf_f32m2(result0, alpha, gvl);

  914.             __riscv_vse32_v_f32m2(&C[ci], c0, gvl);

  915.             m_top += 4;

  916.         }

  917. 

  918.         if (M & 2) {

  919.             float result0 = 0;

  920.             float result1 = 0;

  921.             BLASLONG ai = m_top * K;

  922.             BLASLONG bi = n_top * K;

  923.             BLASLONG pass_K = K;

  924. #ifdef LEFT

  925.             BLASLONG off = offset + m_top;

  926. #else

  927.             BLASLONG off = -offset + n_top;

  928. #endif

  929. #ifdef BACKWARDS

  930.             ai += off * 2;

  931.             bi += off * 1;

  932.             pass_K -= off;

  933. #else

  934. #ifdef LEFT

  935.             pass_K = off + 2;

  936. #else

  937.             pass_K = off + 1;

  938. #endif

  939. #endif

  940. 

  941.             for (BLASLONG k = 0; k < pass_K; k++) {

  942.                 result0 += A[ai + 0] * B[bi + 0];

  943.                 result1 += A[ai + 1] * B[bi + 0];

  944.                 ai += 2;

  945.                 bi += 1;

  946.             }

  947. 

  948.             BLASLONG ci = n_top * ldc + m_top;

  949.             C[ci + 0 * ldc + 0] = alpha * result0;

  950.             C[ci + 0 * ldc + 1] = alpha * result1;

  951.             m_top += 2;

  952.         }

  953. 

  954.         if (M & 1) {

  955.             float result0 = 0;

  956.             BLASLONG ai = m_top * K;

  957.             BLASLONG bi = n_top * K;

  958.             BLASLONG pass_K = K;

  959. #ifdef LEFT

  960.             BLASLONG off = offset + m_top;

  961. #else

  962.             BLASLONG off = -offset + n_top;

  963. #endif

  964. #ifdef BACKWARDS

  965.             ai += off * 1;

  966.             bi += off * 1;

  967.             pass_K -= off;

  968. #else

  969. #ifdef LEFT

  970.             pass_K = off + 1;

  971. #else

  972.             pass_K = off + 1;

  973. #endif

  974. #endif

  975. 

  976.             for (BLASLONG k = 0; k < pass_K; k++) {

  977.                 result0 += A[ai + 0] * B[bi + 0];

  978.                 ai += 1;

  979.                 bi += 1;

  980.             }

  981. 

  982.             BLASLONG ci = n_top * ldc + m_top;

  983.             C[ci + 0 * ldc + 0] = alpha * result0;

  984.             m_top += 1;

  985.         }

  986. 

  987.         n_top += 1;

  988.     }

  989. 

  990.     return 0;

  991. }