OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 3a63bbabd1
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3a63bbabd1'
${ noResults }
OpenBLAS/kernel/riscv64/ztrmmkernel_2x2_rvv.c

597 lines
23 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2022, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. 

  30. #if !defined(DOUBLE)

  31. #define VSETVL(n)               __riscv_vsetvl_e32m2(n)

  32. #define VSETVL_MAX              __riscv_vsetvlmax_e32m2()

  33. #define VSETVL_MAX_M1           __riscv_vsetvlmax_e32m1()

  34. #define FLOAT_V_T               vfloat32m2_t

  35. #define FLOAT_V_T_M1            vfloat32m1_t

  36. #define VLEV_FLOAT              __riscv_vle32_v_f32m2

  37. #define VLSEG4_FLOAT            __riscv_vlseg4e32_v_f32m2

  38. #define VLSEG2_FLOAT            __riscv_vlseg2e32_v_f32m2

  39. #define VFMVVF_FLOAT            __riscv_vfmv_v_f_f32m2

  40. #define VFMACCVF_FLOAT          __riscv_vfmacc_vf_f32m2

  41. #define VFMACCVV_FLOAT          __riscv_vfmacc_vv_f32m2

  42. #define VFNMSACVV_FLOAT         __riscv_vfnmsac_vv_f32m2

  43. #define VFREDSUMVS_FLOAT        __riscv_vfredusum_vs_f32m2_f32m1

  44. #define VFMVVF_FLOAT_M1         __riscv_vfmv_v_f_f32m1

  45. #define VFMVFS_FLOAT_M1         __riscv_vfmv_f_s_f32m1_f32

  46. #else

  47. #define VSETVL(n)               __riscv_vsetvl_e64m2(n)

  48. #define VSETVL_MAX              __riscv_vsetvlmax_e64m2()

  49. #define VSETVL_MAX_M1           __riscv_vsetvlmax_e64m1()

  50. #define FLOAT_V_T               vfloat64m2_t

  51. #define FLOAT_V_T_M1            vfloat64m1_t

  52. #define VLEV_FLOAT              __riscv_vle64_v_f64m2

  53. #define VLSEG4_FLOAT            __riscv_vlseg4e64_v_f64m2

  54. #define VLSEG2_FLOAT            __riscv_vlseg2e64_v_f64m2

  55. #define VFMVVF_FLOAT            __riscv_vfmv_v_f_f64m2

  56. #define VFMACCVF_FLOAT          __riscv_vfmacc_vf_f64m2

  57. #define VFMACCVV_FLOAT          __riscv_vfmacc_vv_f64m2

  58. #define VFNMSACVV_FLOAT         __riscv_vfnmsac_vv_f64m2

  59. #define VFREDSUMVS_FLOAT        __riscv_vfredusum_vs_f64m2_f64m1

  60. #define VFMVVF_FLOAT_M1         __riscv_vfmv_v_f_f64m1

  61. #define VFMVFS_FLOAT_M1         __riscv_vfmv_f_s_f64m1_f64

  62. #endif

  63. 

  64. // Optimizes the implementation in ../generic/ztrmmkernel_2x2.c

  65. 

  66. 

  67. /********************************

  68.   ADD1 a*c

  69.   ADD2 b*c

  70.   ADD3 a*d

  71.   ADD4 b*d

  72.  *********************************/

  73. int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alphar,FLOAT alphai,FLOAT* ba,FLOAT* bb,

  74.         FLOAT* C,BLASLONG ldc, BLASLONG offset)

  75. {

  76.     BLASLONG i,j,k;

  77.     FLOAT *C0,*C1,*ptrba,*ptrbb;

  78.     FLOAT res0,res1;

  79.     BLASLONG off, temp;

  80.     

  81.     FLOAT_V_T va0, va1, va2, va3, vb0, vb1, vb2, vb3;

  82.     FLOAT_V_T vres0, vres1, vres2, vres3, vres4, vres5, vres6, vres7;

  83.     FLOAT_V_T_M1 v_m1_res0, v_m1_res1;

  84.     FLOAT_V_T_M1 v_z0 = VFMVVF_FLOAT_M1(0, VSETVL_MAX_M1);

  85. 

  86.     size_t vl;

  87.     size_t vlmax = VSETVL_MAX;

  88. 

  89. #if defined(TRMMKERNEL) && !defined(LEFT)

  90.     off = -offset;

  91. #else

  92.     off = 0;

  93. #endif

  94. 

  95.     for (j = bn/2; j > 0; j--)

  96.     {

  97. #if defined(TRMMKERNEL) &&  defined(LEFT)

  98.         off = offset;

  99. #endif

  100.         C0 = C;

  101.         C1 = C0+2*ldc;

  102.         ptrba = ba;

  103. 

  104.         for (i = bm/2; i > 0; i--)

  105.         {

  106. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  107.             ptrbb = bb;

  108. #else

  109.             ptrba += off*2*2;

  110.             ptrbb = bb+off*2*2;

  111. #endif

  112. 

  113.             vres0 = VFMVVF_FLOAT(0.0, vlmax);

  114.             vres1 = VFMVVF_FLOAT(0.0, vlmax);

  115.             vres2 = VFMVVF_FLOAT(0.0, vlmax);

  116.             vres3 = VFMVVF_FLOAT(0.0, vlmax);

  117.             vres4 = VFMVVF_FLOAT(0.0, vlmax);

  118.             vres5 = VFMVVF_FLOAT(0.0, vlmax);

  119.             vres6 = VFMVVF_FLOAT(0.0, vlmax);

  120.             vres7 = VFMVVF_FLOAT(0.0, vlmax);

  121. 

  122. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  123.             temp = bk - off;

  124. #elif defined(LEFT)

  125.             temp = off + 2;

  126. #else

  127.             temp = off + 2;

  128. #endif

  129. 

  130.             for (k = temp; k > 0; k -= vl)

  131.             {

  132.                 vl = VSETVL(k);

  133.                 VLSEG4_FLOAT(&va0, &va1, &va2, &va3, ptrba, vl);

  134.                 VLSEG4_FLOAT(&vb0, &vb1, &vb2, &vb3, ptrbb, vl);

  135. 

  136. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  137.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  138.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  139.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  140.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  141. 

  142.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  143.                 vres3 = VFMACCVV_FLOAT(vres3, va3, vb0, vl);

  144.                 vres2 = VFNMSACVV_FLOAT(vres2, va3, vb1, vl);

  145.                 vres3 = VFMACCVV_FLOAT(vres3, va2, vb1, vl);

  146. 

  147.                 vres4 = VFMACCVV_FLOAT(vres4, va0, vb2, vl);

  148.                 vres5 = VFMACCVV_FLOAT(vres5, va1, vb2, vl);

  149.                 vres4 = VFNMSACVV_FLOAT(vres4, va1, vb3, vl);

  150.                 vres5 = VFMACCVV_FLOAT(vres5, va0, vb3, vl);

  151. 

  152.                 vres6 = VFMACCVV_FLOAT(vres6, va2, vb2, vl);

  153.                 vres7 = VFMACCVV_FLOAT(vres7, va3, vb2, vl);

  154.                 vres6 = VFNMSACVV_FLOAT(vres6, va3, vb3, vl);

  155.                 vres7 = VFMACCVV_FLOAT(vres7, va2, vb3, vl);

  156. #endif

  157. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  158.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  159.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  160.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  161.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  162. 

  163.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  164.                 vres3 = VFMACCVV_FLOAT(vres3, va3, vb0, vl);

  165.                 vres2 = VFMACCVV_FLOAT(vres2, va3, vb1, vl);

  166.                 vres3 = VFNMSACVV_FLOAT(vres3, va2, vb1, vl);

  167. 

  168.                 vres4 = VFMACCVV_FLOAT(vres4, va0, vb2, vl);

  169.                 vres5 = VFMACCVV_FLOAT(vres5, va1, vb2, vl);

  170.                 vres4 = VFMACCVV_FLOAT(vres4, va1, vb3, vl);

  171.                 vres5 = VFNMSACVV_FLOAT(vres5, va0, vb3, vl);

  172. 

  173.                 vres6 = VFMACCVV_FLOAT(vres6, va2, vb2, vl);

  174.                 vres7 = VFMACCVV_FLOAT(vres7, va3, vb2, vl);

  175.                 vres6 = VFMACCVV_FLOAT(vres6, va3, vb3, vl);

  176.                 vres7 = VFNMSACVV_FLOAT(vres7, va2, vb3, vl);

  177. #endif

  178. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  179.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  180.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  181.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  182.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  183. 

  184.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  185.                 vres3 = VFNMSACVV_FLOAT(vres3, va3, vb0, vl);

  186.                 vres2 = VFMACCVV_FLOAT(vres2, va3, vb1, vl);

  187.                 vres3 = VFMACCVV_FLOAT(vres3, va2, vb1, vl);

  188. 

  189.                 vres4 = VFMACCVV_FLOAT(vres4, va0, vb2, vl);

  190.                 vres5 = VFNMSACVV_FLOAT(vres5, va1, vb2, vl);

  191.                 vres4 = VFMACCVV_FLOAT(vres4, va1, vb3, vl);

  192.                 vres5 = VFMACCVV_FLOAT(vres5, va0, vb3, vl);

  193. 

  194.                 vres6 = VFMACCVV_FLOAT(vres6, va2, vb2, vl);

  195.                 vres7 = VFNMSACVV_FLOAT(vres7, va3, vb2, vl);

  196.                 vres6 = VFMACCVV_FLOAT(vres6, va3, vb3, vl);

  197.                 vres7 = VFMACCVV_FLOAT(vres7, va2, vb3, vl);

  198. #endif

  199. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  200.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  201.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  202.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  203.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  204. 

  205.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  206.                 vres3 = VFNMSACVV_FLOAT(vres3, va3, vb0, vl);

  207.                 vres2 = VFMACCVV_FLOAT(vres2, va3, vb1, vl);

  208.                 vres3 = VFNMSACVV_FLOAT(vres3, va2, vb1, vl);

  209. 

  210.                 vres4 = VFMACCVV_FLOAT(vres4, va0, vb2, vl);

  211.                 vres5 = VFNMSACVV_FLOAT(vres5, va1, vb2, vl);

  212.                 vres4 = VFMACCVV_FLOAT(vres4, va1, vb3, vl);

  213.                 vres5 = VFNMSACVV_FLOAT(vres5, va0, vb3, vl);

  214. 

  215.                 vres6 = VFMACCVV_FLOAT(vres6, va2, vb2, vl);

  216.                 vres7 = VFNMSACVV_FLOAT(vres7, va3, vb2, vl);

  217.                 vres6 = VFMACCVV_FLOAT(vres6, va3, vb3, vl);

  218.                 vres7 = VFNMSACVV_FLOAT(vres7, va2, vb3, vl);

  219. 

  220. #endif

  221.                 ptrba += vl * 4;

  222.                 ptrbb += vl * 4;

  223.             }

  224.             

  225.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres0, v_z0, vlmax);

  226.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres1, v_z0, vlmax);

  227.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  228.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  229.             C0[0] = res0 * alphar - res1 * alphai;

  230.             C0[1] = res1 * alphar + res0 * alphai;

  231. 

  232.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres2, v_z0, vlmax);

  233.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres3, v_z0, vlmax);

  234.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  235.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  236.             C0[2] = res0 * alphar - res1 * alphai;

  237.             C0[3] = res1 * alphar + res0 * alphai;

  238. 

  239.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres4, v_z0, vlmax);

  240.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres5, v_z0, vlmax);

  241.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  242.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  243.             C1[0] = res0 * alphar - res1 * alphai;

  244.             C1[1] = res1 * alphar + res0 * alphai;

  245. 

  246.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres6, v_z0, vlmax);

  247.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres7, v_z0, vlmax);

  248.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  249.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  250.             C1[2] = res0 * alphar - res1 * alphai;

  251.             C1[3] = res1 * alphar + res0 * alphai;

  252. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  253.             (!defined(LEFT) && !defined(TRANSA))

  254.             temp = bk - off;

  255. #ifdef LEFT

  256.             temp -= 2;

  257. #else

  258.             temp -= 2;

  259. #endif

  260. 

  261.             ptrba += temp*2*2;

  262.             ptrbb += temp*2*2;

  263. 

  264. #endif

  265. 

  266. #ifdef LEFT

  267.             off += 2;

  268. #endif

  269. 

  270.             C0 = C0+4;

  271.             C1 = C1+4;

  272.         }

  273. 

  274.         if (bm & 1)

  275.         {

  276. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  277.             ptrbb = bb;

  278. #else

  279.             ptrba += off*2;

  280.             ptrbb = bb + off*2*2;

  281. #endif

  282.             vres0 = VFMVVF_FLOAT(0.0, vlmax);

  283.             vres1 = VFMVVF_FLOAT(0.0, vlmax);

  284.             vres2 = VFMVVF_FLOAT(0.0, vlmax);

  285.             vres3 = VFMVVF_FLOAT(0.0, vlmax);

  286. 

  287. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  288.             temp = bk - off;

  289. #elif defined(LEFT)

  290.             temp = off+1;

  291. #else

  292.             temp = off+2;

  293. #endif

  294.             for (k = temp; k > 0; k -= vl)

  295.             {

  296.                 vl = VSETVL(k);

  297.                 VLSEG2_FLOAT(&va0, &va1, ptrba, vl);

  298.                 VLSEG4_FLOAT(&vb0, &vb1, &vb2, &vb3, ptrbb, vl);

  299. 

  300. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  301.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  302.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  303.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  304.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  305. 

  306.                 vres2 = VFMACCVV_FLOAT(vres2, va0, vb2, vl);

  307.                 vres3 = VFMACCVV_FLOAT(vres3, va1, vb2, vl);

  308.                 vres2 = VFNMSACVV_FLOAT(vres2, va1, vb3, vl);

  309.                 vres3 = VFMACCVV_FLOAT(vres3, va0, vb3, vl);

  310. #endif

  311. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  312. 

  313.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  314.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  315.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  316.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  317. 

  318.                 vres2 = VFMACCVV_FLOAT(vres2, va0, vb2, vl);

  319.                 vres3 = VFMACCVV_FLOAT(vres3, va1, vb2, vl);

  320.                 vres2 = VFMACCVV_FLOAT(vres2, va1, vb3, vl);

  321.                 vres3 = VFNMSACVV_FLOAT(vres3, va0, vb3, vl);

  322. 

  323. #endif

  324. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  325. 

  326.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  327.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  328.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  329.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  330. 

  331.                 vres2 = VFMACCVV_FLOAT(vres2, va0, vb2, vl);

  332.                 vres3 = VFNMSACVV_FLOAT(vres3, va1, vb2, vl);

  333.                 vres2 = VFMACCVV_FLOAT(vres2, va1, vb3, vl);

  334.                 vres3 = VFMACCVV_FLOAT(vres3, va0, vb3, vl);

  335. 

  336. #endif

  337. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  338. 

  339.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  340.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  341.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  342.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  343. 

  344.                 vres2 = VFMACCVV_FLOAT(vres2, va0, vb2, vl);

  345.                 vres3 = VFNMSACVV_FLOAT(vres3, va1, vb2, vl);

  346.                 vres2 = VFNMSACVV_FLOAT(vres2, va1, vb3, vl);

  347.                 vres3 = VFNMSACVV_FLOAT(vres3, va0, vb3, vl);

  348. 

  349. #endif

  350.                 ptrba += vl * 2;

  351.                 ptrbb += vl * 4;

  352.             }

  353.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres0, v_z0, vlmax);

  354.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres1, v_z0, vlmax);

  355.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  356.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  357.             C0[0] = res0 * alphar - res1 * alphai;

  358.             C0[1] = res1 * alphar + res0 * alphai;

  359. 

  360.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres2, v_z0, vlmax);

  361.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres3, v_z0, vlmax);

  362.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  363.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  364.             C1[0] = res0 * alphar - res1 * alphai;

  365.             C1[1] = res1 * alphar + res0 * alphai;

  366. 

  367. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  368.             (!defined(LEFT) && !defined(TRANSA))

  369.             temp = bk - off;

  370. #ifdef LEFT

  371.             temp -= 1;

  372. #else

  373.             temp -= 2;

  374. #endif

  375.             ptrba += temp*2;

  376.             ptrbb += temp*2*2;

  377. #endif

  378. #ifdef LEFT

  379.             off += 1;

  380. #endif

  381.             C0 = C0+2;

  382.             C1 = C1+2;

  383.         }

  384. #if defined(TRMMKERNEL) && !defined(LEFT)

  385.         off += 2;

  386. #endif

  387.         k = (bk<<2);

  388.         bb = bb+k;

  389.         i = (ldc<<2);

  390.         C = C+i;

  391.     }

  392. 

  393.     if (bn & 1)

  394.     {

  395.         C0 = C;

  396. #if defined(TRMMKERNEL) &&  defined(LEFT)

  397.         off = offset;

  398. #endif

  399.         ptrba = ba;

  400. 

  401.         for (i = bm/2; i > 0; i--)

  402.         {

  403. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  404.             ptrbb = bb;

  405. #else

  406.             ptrba += off*2*2;

  407.             ptrbb = bb+off*2;

  408. #endif

  409.             vres0 = VFMVVF_FLOAT(0.0, vlmax);

  410.             vres1 = VFMVVF_FLOAT(0.0, vlmax);

  411.             vres2 = VFMVVF_FLOAT(0.0, vlmax);

  412.             vres3 = VFMVVF_FLOAT(0.0, vlmax);

  413. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  414.             temp = bk - off;

  415. #elif defined(LEFT)

  416.             temp = off + 2;

  417. #else

  418.             temp = off + 1;

  419. #endif

  420. 

  421.             for (k = temp; k > 0; k -= vl)

  422.             {

  423.                 vl = VSETVL(k);

  424.                 VLSEG4_FLOAT(&va0, &va1, &va2, &va3, ptrba, vl);

  425.                 VLSEG2_FLOAT(&vb0, &vb1, ptrbb, vl);

  426. 

  427. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  428.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  429.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  430.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  431.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  432. 

  433.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  434.                 vres3 = VFMACCVV_FLOAT(vres3, va3, vb0, vl);

  435.                 vres2 = VFNMSACVV_FLOAT(vres2, va3, vb1, vl);

  436.                 vres3 = VFMACCVV_FLOAT(vres3, va2, vb1, vl);

  437. #endif

  438. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  439.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  440.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  441.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  442.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  443. 

  444.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  445.                 vres3 = VFMACCVV_FLOAT(vres3, va3, vb0, vl);

  446.                 vres2 = VFMACCVV_FLOAT(vres2, va3, vb1, vl);

  447.                 vres3 = VFNMSACVV_FLOAT(vres3, va2, vb1, vl);

  448. #endif

  449. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  450.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  451.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  452.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  453.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  454. 

  455.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  456.                 vres3 = VFNMSACVV_FLOAT(vres3, va3, vb0, vl);

  457.                 vres2 = VFMACCVV_FLOAT(vres2, va3, vb1, vl);

  458.                 vres3 = VFMACCVV_FLOAT(vres3, va2, vb1, vl);

  459. 

  460. #endif

  461. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  462.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  463.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  464.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  465.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  466. 

  467.                 vres2 = VFMACCVV_FLOAT(vres2, va2, vb0, vl);

  468.                 vres3 = VFNMSACVV_FLOAT(vres3, va3, vb0, vl);

  469.                 vres2 = VFNMSACVV_FLOAT(vres2, va3, vb1, vl);

  470.                 vres3 = VFNMSACVV_FLOAT(vres3, va2, vb1, vl);

  471. 

  472. #endif

  473.                 ptrba += vl * 4;

  474.                 ptrbb += vl * 2;

  475.             }

  476.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres0, v_z0, vlmax);

  477.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres1, v_z0, vlmax);

  478.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  479.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  480.             C0[0] = res0 * alphar - res1 * alphai;

  481.             C0[1] = res1 * alphar + res0 * alphai;

  482. 

  483.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres2, v_z0, vlmax);

  484.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres3, v_z0, vlmax);

  485.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  486.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  487.             C0[2] = res0 * alphar - res1 * alphai;

  488.             C0[3] = res1 * alphar + res0 * alphai;

  489. 

  490. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  491.             (!defined(LEFT) && !defined(TRANSA))

  492.             temp = bk-off;

  493. #ifdef LEFT

  494.             temp -= 2;

  495. #else

  496.             temp -= 1;

  497. #endif

  498.             ptrba += temp*2*2;

  499.             ptrbb += temp*2;

  500. #endif

  501. #ifdef LEFT

  502.             off += 2;

  503. #endif

  504.             C0 = C0+4;

  505.         }

  506. 

  507.         if (bm & 1)

  508.         {

  509. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  510.             ptrbb = bb;

  511. #else

  512.             ptrba += off*2;

  513.             ptrbb = bb + off*2;

  514. #endif

  515.             vres0 = VFMVVF_FLOAT(0.0, vlmax);

  516.             vres1 = VFMVVF_FLOAT(0.0, vlmax);

  517. 

  518. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  519.             temp = bk-off;

  520. #elif defined(LEFT)

  521.             temp = off + 1;

  522. #else

  523.             temp = off + 1;

  524. #endif

  525. 

  526.             for (k = temp; k > 0; k -= vl)

  527.             {

  528.                 vl = VSETVL(k);

  529.                 VLSEG2_FLOAT(&va0, &va1, ptrba, vl);

  530.                 VLSEG2_FLOAT(&vb0, &vb1, ptrbb, vl);

  531. 

  532. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  533.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  534.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  535.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  536.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  537. 

  538. #endif

  539. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  540.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  541.                 vres1 = VFMACCVV_FLOAT(vres1, va1, vb0, vl);

  542.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  543.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  544. 

  545. #endif

  546. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  547.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  548.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  549.                 vres0 = VFMACCVV_FLOAT(vres0, va1, vb1, vl);

  550.                 vres1 = VFMACCVV_FLOAT(vres1, va0, vb1, vl);

  551. 

  552. #endif

  553. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  554.                 vres0 = VFMACCVV_FLOAT(vres0, va0, vb0, vl);

  555.                 vres1 = VFNMSACVV_FLOAT(vres1, va1, vb0, vl);

  556.                 vres0 = VFNMSACVV_FLOAT(vres0, va1, vb1, vl);

  557.                 vres1 = VFNMSACVV_FLOAT(vres1, va0, vb1, vl);

  558. 

  559. #endif

  560.                 ptrba += vl * 2;

  561.                 ptrbb += vl * 2;

  562.                 

  563.             }

  564.             

  565.             v_m1_res0 = VFREDSUMVS_FLOAT(v_m1_res0, vres0, v_z0, vlmax);

  566.             v_m1_res1 = VFREDSUMVS_FLOAT(v_m1_res1, vres1, v_z0, vlmax);

  567.             res0 = VFMVFS_FLOAT_M1(v_m1_res0);

  568.             res1 = VFMVFS_FLOAT_M1(v_m1_res1);

  569.             

  570.             C0[0] = res0 * alphar - res1 * alphai;

  571.             C0[1] = res1 * alphar + res0 * alphai;

  572. 

  573. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  574.             (!defined(LEFT) && !defined(TRANSA))

  575.             temp = bk - off;

  576. #ifdef LEFT

  577.             temp -= 1;

  578. #else

  579.             temp -= 1;

  580. #endif

  581.             ptrba += temp*2;

  582.             ptrbb += temp*2;

  583.             

  584. #endif

  585. #ifdef LEFT

  586.             off += 1;

  587. #endif

  588.             C0 = C0+2;

  589.         }

  590.         k = (bk<<1);

  591.         bb = bb+k;

  592.         i = (ldc<<1);

  593.         C = C+i;

  594.     }

  595.     return 0;

  596. }