OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f4b82d7bc4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f4b82d7bc4'
${ noResults }
OpenBLAS/kernel/zarch/dgemv_n_4.c

692 lines
23 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2017, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. 

  30. #define NBMAX 2048

  31. 

  32. static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)

  33. {

  34.     __asm__ volatile (

  35.         "vlrepg %%v0,0(%5)               \n\t"

  36.         "vlrepg %%v1,8(%5)               \n\t"

  37.         "vlrepg %%v2,16(%5)              \n\t"

  38.         "vlrepg %%v3,24(%5)              \n\t"

  39.         "vlrepg %%v4,%7                  \n\t"

  40.         "vfmdb  %%v0,%%v0,%%v4           \n\t"

  41.         "vfmdb  %%v1,%%v1,%%v4           \n\t"

  42.         "vfmdb  %%v2,%%v2,%%v4           \n\t"

  43.         "vfmdb  %%v3,%%v3,%%v4           \n\t"

  44.         "xgr   %%r1,%%r1                 \n\t"

  45. 

  46.         "lghi    %%r0,-16                \n\t"

  47.         "ngr     %%r0,%0                 \n\t"

  48.         "ltgr    %%r0,%%r0               \n\t"

  49.         "jz      1f                      \n\t"

  50. 

  51.         "srlg  %%r0,%%r0,4               \n\t"

  52.         "0:                              \n\t"

  53.         "pfd 1,1024(%%r1,%1)             \n\t"

  54.         "pfd 1,1024(%%r1,%2)             \n\t"

  55.         "pfd 1,1024(%%r1,%3)             \n\t"

  56.         "pfd 1,1024(%%r1,%4)             \n\t"

  57.         "pfd 2,1024(%%r1,%6)             \n\t"

  58. 

  59.         "vl  %%v16,0(%%r1,%1)            \n\t"

  60.         "vl  %%v17,0(%%r1,%2)            \n\t"

  61.         "vl  %%v18,0(%%r1,%3)            \n\t"

  62.         "vl  %%v19,0(%%r1,%4)            \n\t"

  63.         "vl  %%v20,16(%%r1,%1)           \n\t"

  64.         "vl  %%v21,16(%%r1,%2)           \n\t"

  65.         "vl  %%v22,16(%%r1,%3)           \n\t"

  66.         "vl  %%v23,16(%%r1,%4)           \n\t"

  67.         "vl  %%v24,32(%%r1,%1)           \n\t"

  68.         "vl  %%v25,32(%%r1,%2)           \n\t"

  69.         "vl  %%v26,32(%%r1,%3)           \n\t"

  70.         "vl  %%v27,32(%%r1,%4)           \n\t"

  71.         "vl  %%v28,48(%%r1,%1)           \n\t"

  72.         "vl  %%v29,48(%%r1,%2)           \n\t"

  73.         "vl  %%v30,48(%%r1,%3)           \n\t"

  74.         "vl  %%v31,48(%%r1,%4)           \n\t"

  75. 

  76.         "vl  %%v4,0(%%r1,%6)             \n\t"

  77.         "vfmadb   %%v4,%%v16,%%v0,%%v4   \n\t"

  78.         "vfmadb   %%v4,%%v17,%%v1,%%v4   \n\t"

  79.         "vfmadb   %%v4,%%v18,%%v2,%%v4   \n\t"

  80.         "vfmadb   %%v4,%%v19,%%v3,%%v4   \n\t"

  81.         "vst %%v4,0(%%r1,%6)             \n\t"

  82. 

  83.         "vl  %%v4,16(%%r1,%6)            \n\t"

  84.         "vfmadb   %%v4,%%v20,%%v0,%%v4   \n\t"

  85.         "vfmadb   %%v4,%%v21,%%v1,%%v4   \n\t"

  86.         "vfmadb   %%v4,%%v22,%%v2,%%v4   \n\t"

  87.         "vfmadb   %%v4,%%v23,%%v3,%%v4   \n\t"

  88.         "vst %%v4,16(%%r1,%6)            \n\t"

  89. 

  90.         "vl  %%v4,32(%%r1,%6)            \n\t"

  91.         "vfmadb   %%v4,%%v24,%%v0,%%v4   \n\t"

  92.         "vfmadb   %%v4,%%v25,%%v1,%%v4   \n\t"

  93.         "vfmadb   %%v4,%%v26,%%v2,%%v4   \n\t"

  94.         "vfmadb   %%v4,%%v27,%%v3,%%v4   \n\t"

  95.         "vst %%v4,32(%%r1,%6)            \n\t"

  96. 

  97.         "vl  %%v4,48(%%r1,%6)            \n\t"

  98.         "vfmadb   %%v4,%%v28,%%v0,%%v4   \n\t"

  99.         "vfmadb   %%v4,%%v29,%%v1,%%v4   \n\t"

  100.         "vfmadb   %%v4,%%v30,%%v2,%%v4   \n\t"

  101.         "vfmadb   %%v4,%%v31,%%v3,%%v4   \n\t"

  102.         "vst %%v4,48(%%r1,%6)            \n\t"

  103. 

  104.         "vl  %%v16,64(%%r1,%1)           \n\t"

  105.         "vl  %%v17,64(%%r1,%2)           \n\t"

  106.         "vl  %%v18,64(%%r1,%3)           \n\t"

  107.         "vl  %%v19,64(%%r1,%4)           \n\t"

  108.         "vl  %%v20,80(%%r1,%1)           \n\t"

  109.         "vl  %%v21,80(%%r1,%2)           \n\t"

  110.         "vl  %%v22,80(%%r1,%3)           \n\t"

  111.         "vl  %%v23,80(%%r1,%4)           \n\t"

  112.         "vl  %%v24,96(%%r1,%1)           \n\t"

  113.         "vl  %%v25,96(%%r1,%2)           \n\t"

  114.         "vl  %%v26,96(%%r1,%3)           \n\t"

  115.         "vl  %%v27,96(%%r1,%4)           \n\t"

  116.         "vl  %%v28,112(%%r1,%1)          \n\t"

  117.         "vl  %%v29,112(%%r1,%2)          \n\t"

  118.         "vl  %%v30,112(%%r1,%3)          \n\t"

  119.         "vl  %%v31,112(%%r1,%4)          \n\t"

  120. 

  121.         "vl  %%v4,64(%%r1,%6)            \n\t"

  122.         "vfmadb   %%v4,%%v16,%%v0,%%v4   \n\t"

  123.         "vfmadb   %%v4,%%v17,%%v1,%%v4   \n\t"

  124.         "vfmadb   %%v4,%%v18,%%v2,%%v4   \n\t"

  125.         "vfmadb   %%v4,%%v19,%%v3,%%v4   \n\t"

  126.         "vst %%v4,64(%%r1,%6)            \n\t"

  127. 

  128.         "vl  %%v4,80(%%r1,%6)            \n\t"

  129.         "vfmadb   %%v4,%%v20,%%v0,%%v4   \n\t"

  130.         "vfmadb   %%v4,%%v21,%%v1,%%v4   \n\t"

  131.         "vfmadb   %%v4,%%v22,%%v2,%%v4   \n\t"

  132.         "vfmadb   %%v4,%%v23,%%v3,%%v4   \n\t"

  133.         "vst %%v4,80(%%r1,%6)            \n\t"

  134. 

  135.         "vl  %%v4,96(%%r1,%6)            \n\t"

  136.         "vfmadb   %%v4,%%v24,%%v0,%%v4   \n\t"

  137.         "vfmadb   %%v4,%%v25,%%v1,%%v4   \n\t"

  138.         "vfmadb   %%v4,%%v26,%%v2,%%v4   \n\t"

  139.         "vfmadb   %%v4,%%v27,%%v3,%%v4   \n\t"

  140.         "vst %%v4,96(%%r1,%6)            \n\t"

  141. 

  142.         "vl  %%v4,112(%%r1,%6)           \n\t"

  143.         "vfmadb   %%v4,%%v28,%%v0,%%v4   \n\t"

  144.         "vfmadb   %%v4,%%v29,%%v1,%%v4   \n\t"

  145.         "vfmadb   %%v4,%%v30,%%v2,%%v4   \n\t"

  146.         "vfmadb   %%v4,%%v31,%%v3,%%v4   \n\t"

  147.         "vst %%v4,112(%%r1,%6)           \n\t"

  148.         

  149.         "agfi   %%r1,128                 \n\t"

  150.         "brctg  %%r0,0b                  \n\t"

  151. 

  152.         "1:                              \n\t"

  153.         "lghi    %%r0,12                 \n\t"

  154.         "ngr     %%r0,%0                 \n\t"

  155.         "ltgr    %%r0,%%r0               \n\t"

  156.         "jz      3f                      \n\t"

  157. 

  158.         "srlg  %%r0,%%r0,2               \n\t"

  159.         "2:                              \n\t"

  160.         "vl  %%v16,0(%%r1,%1)            \n\t"

  161.         "vl  %%v17,0(%%r1,%2)            \n\t"

  162.         "vl  %%v18,0(%%r1,%3)            \n\t"

  163.         "vl  %%v19,0(%%r1,%4)            \n\t"

  164.         "vl  %%v20,16(%%r1,%1)           \n\t"

  165.         "vl  %%v21,16(%%r1,%2)           \n\t"

  166.         "vl  %%v22,16(%%r1,%3)           \n\t"

  167.         "vl  %%v23,16(%%r1,%4)           \n\t"

  168. 

  169.         "vl  %%v4,0(%%r1,%6)             \n\t"

  170.         "vfmadb   %%v4,%%v16,%%v0,%%v4   \n\t"

  171.         "vfmadb   %%v4,%%v17,%%v1,%%v4   \n\t"

  172.         "vfmadb   %%v4,%%v18,%%v2,%%v4   \n\t"

  173.         "vfmadb   %%v4,%%v19,%%v3,%%v4   \n\t"

  174.         "vst %%v4,0(%%r1,%6)             \n\t"

  175. 

  176.         "vl  %%v4,16(%%r1,%6)            \n\t"

  177.         "vfmadb   %%v4,%%v20,%%v0,%%v4   \n\t"

  178.         "vfmadb   %%v4,%%v21,%%v1,%%v4   \n\t"

  179.         "vfmadb   %%v4,%%v22,%%v2,%%v4   \n\t"

  180.         "vfmadb   %%v4,%%v23,%%v3,%%v4   \n\t"

  181.         "vst %%v4,16(%%r1,%6)            \n\t"

  182. 

  183.         "agfi   %%r1,32                  \n\t"

  184.         "brctg  %%r0,2b                  \n\t"

  185. 

  186.         "3:                              \n\t"

  187.         "nop                                 "

  188.         :

  189.         :"r"(n),"ZR"((const FLOAT (*)[n])ap[0]),"ZR"((const FLOAT (*)[n])ap[1]),"ZR"((const FLOAT (*)[n])ap[2]),"ZR"((const FLOAT (*)[n])ap[3]),"ZQ"((const FLOAT (*)[4])xo),"ZR"((FLOAT (*)[n])y),"m"(*alpha)

  190.         :"memory","cc","r0","r1","v0","v1","v2","v3","v4","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  191.     );

  192. }

  193. 

  194. static void dgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)

  195. {

  196.     __asm__ volatile (

  197.         "vlrepg %%v0,0(%3)               \n\t"

  198.         "vlrepg %%v1,8(%3)               \n\t"

  199.         "vlrepg %%v2,%5                  \n\t"

  200.         "vfmdb  %%v0,%%v0,%%v2           \n\t"

  201.         "vfmdb  %%v1,%%v1,%%v2           \n\t"

  202.         "xgr   %%r1,%%r1                 \n\t"

  203. 

  204.         "lghi    %%r0,-16                \n\t"

  205.         "ngr     %%r0,%0                 \n\t"

  206.         "ltgr    %%r0,%%r0               \n\t"

  207.         "jz      1f                      \n\t"

  208. 

  209.         "srlg  %%r0,%%r0,4               \n\t"

  210.         "0:                              \n\t"

  211.         "pfd 1,1024(%%r1,%1)             \n\t"

  212.         "pfd 1,1024(%%r1,%2)             \n\t"

  213.         "pfd 2,1024(%%r1,%4)             \n\t"

  214. 

  215.         "vl  %%v16,0(%%r1,%1)            \n\t"

  216.         "vl  %%v17,0(%%r1,%2)            \n\t"

  217.         "vl  %%v18,16(%%r1,%1)           \n\t"

  218.         "vl  %%v19,16(%%r1,%2)           \n\t"

  219.         "vl  %%v20,32(%%r1,%1)           \n\t"

  220.         "vl  %%v21,32(%%r1,%2)           \n\t"

  221.         "vl  %%v22,48(%%r1,%1)           \n\t"

  222.         "vl  %%v23,48(%%r1,%2)           \n\t"

  223.         "vl  %%v24,64(%%r1,%1)           \n\t"

  224.         "vl  %%v25,64(%%r1,%2)           \n\t"

  225.         "vl  %%v26,80(%%r1,%1)           \n\t"

  226.         "vl  %%v27,80(%%r1,%2)           \n\t"

  227.         "vl  %%v28,96(%%r1,%1)           \n\t"

  228.         "vl  %%v29,96(%%r1,%2)           \n\t"

  229.         "vl  %%v30,112(%%r1,%1)          \n\t"

  230.         "vl  %%v31,112(%%r1,%2)          \n\t"

  231. 

  232.         "vl  %%v2,0(%%r1,%4)             \n\t"

  233.         "vfmadb   %%v2,%%v16,%%v0,%%v2   \n\t"

  234.         "vfmadb   %%v2,%%v17,%%v1,%%v2   \n\t"

  235.         "vst %%v2,0(%%r1,%4)             \n\t"

  236. 

  237.         "vl  %%v2,16(%%r1,%4)            \n\t"

  238.         "vfmadb   %%v2,%%v18,%%v0,%%v2   \n\t"

  239.         "vfmadb   %%v2,%%v19,%%v1,%%v2   \n\t"

  240.         "vst %%v2,16(%%r1,%4)            \n\t"

  241. 

  242.         "vl  %%v2,32(%%r1,%4)            \n\t"

  243.         "vfmadb   %%v2,%%v20,%%v0,%%v2   \n\t"

  244.         "vfmadb   %%v2,%%v21,%%v1,%%v2   \n\t"

  245.         "vst %%v2,32(%%r1,%4)            \n\t"

  246. 

  247.         "vl  %%v2,48(%%r1,%4)            \n\t"

  248.         "vfmadb   %%v2,%%v22,%%v0,%%v2   \n\t"

  249.         "vfmadb   %%v2,%%v23,%%v1,%%v2   \n\t"

  250.         "vst %%v2,48(%%r1,%4)            \n\t"

  251. 

  252.         "vl  %%v2,64(%%r1,%4)            \n\t"

  253.         "vfmadb   %%v2,%%v24,%%v0,%%v2   \n\t"

  254.         "vfmadb   %%v2,%%v25,%%v1,%%v2   \n\t"

  255.         "vst %%v2,64(%%r1,%4)            \n\t"

  256. 

  257.         "vl  %%v2,80(%%r1,%4)            \n\t"

  258.         "vfmadb   %%v2,%%v26,%%v0,%%v2   \n\t"

  259.         "vfmadb   %%v2,%%v27,%%v1,%%v2   \n\t"

  260.         "vst %%v2,80(%%r1,%4)            \n\t"

  261. 

  262.         "vl  %%v2,96(%%r1,%4)            \n\t"

  263.         "vfmadb   %%v2,%%v28,%%v0,%%v2   \n\t"

  264.         "vfmadb   %%v2,%%v29,%%v1,%%v2   \n\t"

  265.         "vst %%v2,96(%%r1,%4)            \n\t"

  266. 

  267.         "vl  %%v2,112(%%r1,%4)           \n\t"

  268.         "vfmadb   %%v2,%%v30,%%v0,%%v2   \n\t"

  269.         "vfmadb   %%v2,%%v31,%%v1,%%v2   \n\t"

  270.         "vst %%v2,112(%%r1,%4)           \n\t"

  271.         

  272.         "agfi   %%r1,128                 \n\t"

  273.         "brctg  %%r0,0b                  \n\t"

  274. 

  275.         "1:                              \n\t"

  276.         "lghi    %%r0,12                 \n\t"

  277.         "ngr     %%r0,%0                 \n\t"

  278.         "ltgr    %%r0,%%r0               \n\t"

  279.         "jz      3f                      \n\t"

  280. 

  281.         "srlg  %%r0,%%r0,2               \n\t"

  282.         "2:                              \n\t"

  283.         "vl  %%v16,0(%%r1,%1)            \n\t"

  284.         "vl  %%v17,0(%%r1,%2)            \n\t"

  285.         "vl  %%v18,16(%%r1,%1)           \n\t"

  286.         "vl  %%v19,16(%%r1,%2)           \n\t"

  287. 

  288.         "vl  %%v2,0(%%r1,%4)             \n\t"

  289.         "vfmadb   %%v2,%%v16,%%v0,%%v2   \n\t"

  290.         "vfmadb   %%v2,%%v17,%%v1,%%v2   \n\t"

  291.         "vst %%v2,0(%%r1,%4)             \n\t"

  292. 

  293.         "vl  %%v2,16(%%r1,%4)            \n\t"

  294.         "vfmadb   %%v2,%%v18,%%v0,%%v2   \n\t"

  295.         "vfmadb   %%v2,%%v19,%%v1,%%v2   \n\t"

  296.         "vst %%v2,16(%%r1,%4)            \n\t"

  297. 

  298.         "agfi   %%r1,32                  \n\t"

  299.         "brctg  %%r0,2b                  \n\t"

  300. 

  301.         "3:                              \n\t"

  302.         "nop                                 "

  303.         :

  304.         :"r"(n),"ZR"((const FLOAT (*)[n])ap[0]),"ZR"((const FLOAT (*)[n])ap[1]),"ZQ"((const FLOAT (*)[2])xo),"ZR"((FLOAT (*)[n])y),"m"(*alpha)

  305.         :"memory","cc","r0","r1","v0","v1","v2","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  306.     );

  307. }

  308. 

  309. static void dgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *xo, FLOAT *y, FLOAT *alpha)

  310. {

  311.     __asm__ volatile (

  312.         "vlrepg %%v0,0(%2)               \n\t"

  313.         "vlrepg %%v1,%4                  \n\t"

  314.         "vfmdb  %%v0,%%v0,%%v1           \n\t"

  315.         "xgr   %%r1,%%r1                 \n\t"

  316. 

  317.         "lghi    %%r0,-16                \n\t"

  318.         "ngr     %%r0,%0                 \n\t"

  319.         "ltgr    %%r0,%%r0               \n\t"

  320.         "jz      1f                      \n\t"

  321. 

  322.         "srlg  %%r0,%%r0,4               \n\t"

  323.         "0:                              \n\t"

  324.         "pfd 1,1024(%%r1,%1)             \n\t"

  325.         "pfd 2,1024(%%r1,%3)             \n\t"

  326. 

  327.         "vl  %%v16,0(%%r1,%1)            \n\t"

  328.         "vl  %%v17,16(%%r1,%1)           \n\t"

  329.         "vl  %%v18,32(%%r1,%1)           \n\t"

  330.         "vl  %%v19,48(%%r1,%1)           \n\t"

  331.         "vl  %%v20,64(%%r1,%1)           \n\t"

  332.         "vl  %%v21,80(%%r1,%1)           \n\t"

  333.         "vl  %%v22,96(%%r1,%1)           \n\t"

  334.         "vl  %%v23,112(%%r1,%1)          \n\t"

  335. 

  336.         "vl  %%v1,0(%%r1,%3)             \n\t"

  337.         "vfmadb   %%v1,%%v16,%%v0,%%v1   \n\t"

  338.         "vst %%v1,0(%%r1,%3)             \n\t"

  339. 

  340.         "vl  %%v1,16(%%r1,%3)            \n\t"

  341.         "vfmadb   %%v1,%%v17,%%v0,%%v1   \n\t"

  342.         "vst %%v1,16(%%r1,%3)            \n\t"

  343. 

  344.         "vl  %%v1,32(%%r1,%3)            \n\t"

  345.         "vfmadb   %%v1,%%v18,%%v0,%%v1   \n\t"

  346.         "vst %%v1,32(%%r1,%3)            \n\t"

  347. 

  348.         "vl  %%v1,48(%%r1,%3)            \n\t"

  349.         "vfmadb   %%v1,%%v19,%%v0,%%v1   \n\t"

  350.         "vst %%v1,48(%%r1,%3)            \n\t"

  351. 

  352.         "vl  %%v1,64(%%r1,%3)            \n\t"

  353.         "vfmadb   %%v1,%%v20,%%v0,%%v1   \n\t"

  354.         "vst %%v1,64(%%r1,%3)            \n\t"

  355. 

  356.         "vl  %%v1,80(%%r1,%3)            \n\t"

  357.         "vfmadb   %%v1,%%v21,%%v0,%%v1   \n\t"

  358.         "vst %%v1,80(%%r1,%3)            \n\t"

  359. 

  360.         "vl  %%v1,96(%%r1,%3)            \n\t"

  361.         "vfmadb   %%v1,%%v22,%%v0,%%v1   \n\t"

  362.         "vst %%v1,96(%%r1,%3)            \n\t"

  363. 

  364.         "vl  %%v1,112(%%r1,%3)           \n\t"

  365.         "vfmadb   %%v1,%%v23,%%v0,%%v1   \n\t"

  366.         "vst %%v1,112(%%r1,%3)           \n\t"

  367.         

  368.         "agfi   %%r1,128                 \n\t"

  369.         "brctg  %%r0,0b                  \n\t"

  370. 

  371.         "1:                              \n\t"

  372.         "lghi    %%r0,12                 \n\t"

  373.         "ngr     %%r0,%0                 \n\t"

  374.         "ltgr    %%r0,%%r0               \n\t"

  375.         "jz      3f                      \n\t"

  376. 

  377.         "srlg  %%r0,%%r0,2               \n\t"

  378.         "2:                              \n\t"

  379.         "vl  %%v16,0(%%r1,%1)            \n\t"

  380.         "vl  %%v17,16(%%r1,%1)           \n\t"

  381. 

  382.         "vl  %%v1,0(%%r1,%3)             \n\t"

  383.         "vfmadb   %%v1,%%v16,%%v0,%%v1   \n\t"

  384.         "vst %%v1,0(%%r1,%3)             \n\t"

  385. 

  386.         "vl  %%v1,16(%%r1,%3)            \n\t"

  387.         "vfmadb   %%v1,%%v17,%%v0,%%v1   \n\t"

  388.         "vst %%v1,16(%%r1,%3)            \n\t"

  389. 

  390.         "agfi   %%r1,32                  \n\t"

  391.         "brctg  %%r0,2b                  \n\t"

  392. 

  393.         "3:                              \n\t"

  394.         "nop                                 "

  395.         :

  396.         :"r"(n),"ZR"((const FLOAT (*)[n])a0),"ZQ"((const FLOAT (*)[1])xo),"ZR"((FLOAT (*)[n])y),"m"(*alpha)

  397.         :"memory","cc","r0","r1","v0","v1","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  398.     );

  399. }

  400. 

  401. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)

  402. {

  403.     BLASLONG i;

  404.     for (i = 0; i < n; i++)

  405.     {

  406.         *dest += src[i];

  407.         dest  += inc_dest;

  408.     }

  409. }

  410. 

  411. int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)

  412. {

  413.     BLASLONG i;

  414.     FLOAT *a_ptr;

  415.     FLOAT *x_ptr;

  416.     FLOAT *y_ptr;

  417.     FLOAT *ap[4];

  418.     BLASLONG n1;

  419.     BLASLONG m1;

  420.     BLASLONG m2;

  421.     BLASLONG m3;

  422.     BLASLONG n2;

  423.     BLASLONG lda4 =  lda << 2;

  424.     FLOAT xbuffer[8],*ybuffer;

  425. 

  426.     if ( m < 1 ) return(0);

  427.     if ( n < 1 ) return(0);

  428. 

  429.     ybuffer = buffer;

  430.     

  431.     n1 = n >> 2 ;

  432.     n2 = n &  3 ;

  433. 

  434.     m3 = m & 3  ;

  435.     m1 = m & -4 ;

  436.     m2 = (m & (NBMAX-1)) - m3 ;

  437. 

  438.     y_ptr = y;

  439. 

  440.     BLASLONG NB = NBMAX;

  441. 

  442.     while ( NB == NBMAX )

  443.     {

  444.         

  445.         m1 -= NB;

  446.         if ( m1 < 0)

  447.         {

  448.             if ( m2 == 0 ) break;    

  449.             NB = m2;

  450.         }

  451.         

  452.         a_ptr = a;

  453.         x_ptr = x;

  454.         

  455.         ap[0] = a_ptr;

  456.         ap[1] = a_ptr + lda;

  457.         ap[2] = ap[1] + lda;

  458.         ap[3] = ap[2] + lda;

  459. 

  460.         if ( inc_y != 1 )

  461.             memset(ybuffer,0,NB*8);

  462.         else

  463.             ybuffer = y_ptr;

  464. 

  465.         if ( inc_x == 1 )

  466.         {

  467. 

  468. 

  469.             for( i = 0; i < n1 ; i++)

  470.             {

  471.                 dgemv_kernel_4x4(NB,ap,x_ptr,ybuffer,&alpha);

  472.                 ap[0] += lda4; 

  473.                 ap[1] += lda4; 

  474.                 ap[2] += lda4; 

  475.                 ap[3] += lda4; 

  476.                 a_ptr += lda4;

  477.                 x_ptr += 4;    

  478.             }

  479. 

  480.             if ( n2 & 2 )

  481.             {

  482.                 dgemv_kernel_4x2(NB,ap,x_ptr,ybuffer,&alpha);

  483.                 a_ptr += lda*2;

  484.                 x_ptr += 2;    

  485.             }

  486. 

  487. 

  488.             if ( n2 & 1 )

  489.             {

  490.                 dgemv_kernel_4x1(NB,a_ptr,x_ptr,ybuffer,&alpha);

  491.                 /* a_ptr += lda;

  492.                 x_ptr += 1; */    

  493. 

  494.             }

  495. 

  496. 

  497.         }

  498.         else

  499.         {

  500. 

  501.             for( i = 0; i < n1 ; i++)

  502.             {

  503.                 xbuffer[0] = x_ptr[0];

  504.                 x_ptr += inc_x;    

  505.                 xbuffer[1] =  x_ptr[0];

  506.                 x_ptr += inc_x;    

  507.                 xbuffer[2] =  x_ptr[0];

  508.                 x_ptr += inc_x;    

  509.                 xbuffer[3] = x_ptr[0];

  510.                 x_ptr += inc_x;    

  511.                 dgemv_kernel_4x4(NB,ap,xbuffer,ybuffer,&alpha);

  512.                 ap[0] += lda4; 

  513.                 ap[1] += lda4; 

  514.                 ap[2] += lda4; 

  515.                 ap[3] += lda4; 

  516.                 a_ptr += lda4;

  517.             }

  518. 

  519.             for( i = 0; i < n2 ; i++)

  520.             {

  521.                 xbuffer[0] = x_ptr[0];

  522.                 x_ptr += inc_x;    

  523.                 dgemv_kernel_4x1(NB,a_ptr,xbuffer,ybuffer,&alpha);

  524.                 a_ptr += lda;

  525. 

  526.             }

  527. 

  528.         }

  529. 

  530.         a     += NB;

  531.         if ( inc_y != 1 )

  532.         {

  533.             add_y(NB,ybuffer,y_ptr,inc_y);

  534.             y_ptr += NB * inc_y;

  535.         }

  536.         else

  537.             y_ptr += NB ;

  538. 

  539.     }

  540. 

  541.     if ( m3 == 0 ) return(0);

  542. 

  543.     if ( m3 == 3 )

  544.     {

  545.         a_ptr = a;

  546.         x_ptr = x;

  547.         FLOAT temp0 = 0.0;

  548.         FLOAT temp1 = 0.0;

  549.         FLOAT temp2 = 0.0;

  550.         if ( lda == 3 && inc_x ==1 )

  551.         {

  552. 

  553.             for( i = 0; i < ( n & -4 ); i+=4 )

  554.             {

  555. 

  556.                 temp0 += a_ptr[0] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  557.                 temp1 += a_ptr[1] * x_ptr[0] + a_ptr[4] * x_ptr[1];

  558.                 temp2 += a_ptr[2] * x_ptr[0] + a_ptr[5] * x_ptr[1];

  559. 

  560.                 temp0 += a_ptr[6] * x_ptr[2] + a_ptr[9]  * x_ptr[3];

  561.                 temp1 += a_ptr[7] * x_ptr[2] + a_ptr[10] * x_ptr[3];

  562.                 temp2 += a_ptr[8] * x_ptr[2] + a_ptr[11] * x_ptr[3];

  563. 

  564.                 a_ptr += 12;

  565.                 x_ptr += 4;

  566.             }

  567. 

  568.             for( ; i < n; i++ )

  569.             {

  570.                 temp0 += a_ptr[0] * x_ptr[0];

  571.                 temp1 += a_ptr[1] * x_ptr[0];

  572.                 temp2 += a_ptr[2] * x_ptr[0];

  573.                 a_ptr += 3;

  574.                 x_ptr ++;

  575.             }

  576. 

  577.         }

  578.         else

  579.         {

  580. 

  581.             for( i = 0; i < n; i++ )

  582.             {

  583.                 temp0 += a_ptr[0] * x_ptr[0];

  584.                 temp1 += a_ptr[1] * x_ptr[0];

  585.                 temp2 += a_ptr[2] * x_ptr[0];

  586.                 a_ptr += lda;

  587.                 x_ptr += inc_x;

  588. 

  589. 

  590.             }

  591. 

  592.         }

  593.         y_ptr[0] += alpha * temp0;

  594.         y_ptr += inc_y;

  595.         y_ptr[0] += alpha * temp1;

  596.         y_ptr += inc_y;

  597.         y_ptr[0] += alpha * temp2;

  598.         return(0);

  599.     }

  600. 

  601. 

  602.     if ( m3 == 2 )

  603.     {

  604.         a_ptr = a;

  605.         x_ptr = x;

  606.         FLOAT temp0 = 0.0;

  607.         FLOAT temp1 = 0.0;

  608.         if ( lda == 2 && inc_x ==1 )

  609.         {

  610. 

  611.             for( i = 0; i < (n & -4) ; i+=4 )

  612.             {

  613.                 temp0 += a_ptr[0] * x_ptr[0] + a_ptr[2] * x_ptr[1];

  614.                 temp1 += a_ptr[1] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  615.                 temp0 += a_ptr[4] * x_ptr[2] + a_ptr[6] * x_ptr[3];

  616.                 temp1 += a_ptr[5] * x_ptr[2] + a_ptr[7] * x_ptr[3];

  617.                 a_ptr += 8;

  618.                 x_ptr += 4;

  619. 

  620.             }

  621. 

  622. 

  623.             for( ; i < n; i++ )

  624.             {

  625.                 temp0 += a_ptr[0]   * x_ptr[0];

  626.                 temp1 += a_ptr[1]   * x_ptr[0];

  627.                 a_ptr += 2;

  628.                 x_ptr ++;

  629.             }

  630. 

  631.         }

  632.         else

  633.         {

  634. 

  635.             for( i = 0; i < n; i++ )

  636.             {

  637.                 temp0 += a_ptr[0] * x_ptr[0];

  638.                 temp1 += a_ptr[1] * x_ptr[0];

  639.                 a_ptr += lda;

  640.                 x_ptr += inc_x;

  641. 

  642. 

  643.             }

  644. 

  645.         }

  646.         y_ptr[0] += alpha * temp0;

  647.         y_ptr += inc_y;

  648.         y_ptr[0] += alpha * temp1;

  649.         return(0);

  650.     }

  651. 

  652.     if ( m3 == 1 )

  653.     {

  654.         a_ptr = a;

  655.         x_ptr = x;

  656.         FLOAT temp = 0.0;

  657.         if ( lda == 1 && inc_x ==1 )

  658.         {

  659. 

  660.             for( i = 0; i < (n & -4); i+=4 )

  661.             {

  662.                 temp += a_ptr[i] * x_ptr[i] + a_ptr[i+1] * x_ptr[i+1] + a_ptr[i+2] * x_ptr[i+2] + a_ptr[i+3] * x_ptr[i+3];

  663.     

  664.             }

  665. 

  666.             for( ; i < n; i++ )

  667.             {

  668.                 temp += a_ptr[i] * x_ptr[i];

  669.             }

  670. 

  671.         }

  672.         else

  673.         {

  674. 

  675.             for( i = 0; i < n; i++ )

  676.             {

  677.                 temp += a_ptr[0] * x_ptr[0];

  678.                 a_ptr += lda;

  679.                 x_ptr += inc_x;

  680.             }

  681. 

  682.         }

  683.         y_ptr[0] += alpha * temp;

  684.         return(0);

  685.     }

  686. 

  687. 

  688.     return(0);

  689. }

  690.