OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f4b82d7bc4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f4b82d7bc4'
${ noResults }
OpenBLAS/kernel/zarch/dgemv_t_4.c

824 lines
28 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2017, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. 

  30. #define NBMAX 2048

  31. 

  32. static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  33. {

  34.     __asm__ volatile (   

  35.         "vzero %%v0                      \n\t"

  36.         "vzero %%v1                      \n\t"

  37.         "vzero %%v2                      \n\t"

  38.         "vzero %%v3                      \n\t"

  39.         "xgr   %%r1,%%r1                 \n\t"

  40. 

  41.         "lghi    %%r0,-16                \n\t"

  42.         "ngr     %%r0,%0                 \n\t"

  43.         "ltgr    %%r0,%%r0               \n\t"

  44.         "jz      1f                      \n\t"

  45. 

  46.         "srlg  %%r0,%%r0,4               \n\t"

  47.         "0:                              \n\t"

  48.         "pfd 1,1024(%%r1,%1)             \n\t"

  49.         "pfd 1,1024(%%r1,%2)             \n\t"

  50.         "pfd 1,1024(%%r1,%3)             \n\t"

  51.         "pfd 1,1024(%%r1,%4)             \n\t"

  52.         "pfd 1,1024(%%r1,%5)             \n\t"

  53. 

  54.         "vl  %%v16,0(%%r1,%5)            \n\t"

  55.         "vl  %%v17,16(%%r1,%5)           \n\t"

  56.         "vl  %%v18,32(%%r1,%5)           \n\t"

  57.         "vl  %%v19,48(%%r1,%5)           \n\t"

  58.         "vl  %%v20,64(%%r1,%5)           \n\t"

  59.         "vl  %%v21,80(%%r1,%5)           \n\t"

  60.         "vl  %%v22,96(%%r1,%5)           \n\t"

  61.         "vl  %%v23,112(%%r1,%5)          \n\t"

  62. 

  63.         "vl  %%v24,0(%%r1,%1)            \n\t"

  64.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  65.         "vl  %%v25,0(%%r1,%2)            \n\t"

  66.         "vfmadb   %%v1,%%v16,%%v25,%%v1  \n\t"

  67.         "vl  %%v26,0(%%r1,%3)            \n\t"

  68.         "vfmadb   %%v2,%%v16,%%v26,%%v2  \n\t"

  69.         "vl  %%v27,0(%%r1,%4)            \n\t"

  70.         "vfmadb   %%v3,%%v16,%%v27,%%v3  \n\t"

  71. 

  72.         "vl  %%v28,16(%%r1,%1)           \n\t"

  73.         "vfmadb   %%v0,%%v17,%%v28,%%v0  \n\t"

  74.         "vl  %%v29,16(%%r1,%2)           \n\t"

  75.         "vfmadb   %%v1,%%v17,%%v29,%%v1  \n\t"

  76.         "vl  %%v30,16(%%r1,%3)           \n\t"

  77.         "vfmadb   %%v2,%%v17,%%v30,%%v2  \n\t"

  78.         "vl  %%v31,16(%%r1,%4)           \n\t"

  79.         "vfmadb   %%v3,%%v17,%%v31,%%v3  \n\t"

  80. 

  81.         "vl  %%v24,32(%%r1,%1)           \n\t"

  82.         "vfmadb   %%v0,%%v18,%%v24,%%v0  \n\t"

  83.         "vl  %%v25,32(%%r1,%2)           \n\t"

  84.         "vfmadb   %%v1,%%v18,%%v25,%%v1  \n\t"

  85.         "vl  %%v26,32(%%r1,%3)           \n\t"

  86.         "vfmadb   %%v2,%%v18,%%v26,%%v2  \n\t"

  87.         "vl  %%v27,32(%%r1,%4)           \n\t"

  88.         "vfmadb   %%v3,%%v18,%%v27,%%v3  \n\t"

  89. 

  90.         "vl  %%v28,48(%%r1,%1)           \n\t"

  91.         "vfmadb   %%v0,%%v19,%%v28,%%v0  \n\t"

  92.         "vl  %%v29,48(%%r1,%2)           \n\t"

  93.         "vfmadb   %%v1,%%v19,%%v29,%%v1  \n\t"

  94.         "vl  %%v30,48(%%r1,%3)           \n\t"

  95.         "vfmadb   %%v2,%%v19,%%v30,%%v2  \n\t"

  96.         "vl  %%v31,48(%%r1,%4)           \n\t"

  97.         "vfmadb   %%v3,%%v19,%%v31,%%v3  \n\t"

  98. 

  99.         "vl  %%v24,64(%%r1,%1)           \n\t"

  100.         "vfmadb   %%v0,%%v20,%%v24,%%v0  \n\t"

  101.         "vl  %%v25,64(%%r1,%2)           \n\t"

  102.         "vfmadb   %%v1,%%v20,%%v25,%%v1  \n\t"

  103.         "vl  %%v26,64(%%r1,%3)           \n\t"

  104.         "vfmadb   %%v2,%%v20,%%v26,%%v2  \n\t"

  105.         "vl  %%v27,64(%%r1,%4)           \n\t"

  106.         "vfmadb   %%v3,%%v20,%%v27,%%v3  \n\t"

  107. 

  108.         "vl  %%v28,80(%%r1,%1)           \n\t"

  109.         "vfmadb   %%v0,%%v21,%%v28,%%v0  \n\t"

  110.         "vl  %%v29,80(%%r1,%2)           \n\t"

  111.         "vfmadb   %%v1,%%v21,%%v29,%%v1  \n\t"

  112.         "vl  %%v30,80(%%r1,%3)           \n\t"

  113.         "vfmadb   %%v2,%%v21,%%v30,%%v2  \n\t"

  114.         "vl  %%v31,80(%%r1,%4)           \n\t"

  115.         "vfmadb   %%v3,%%v21,%%v31,%%v3  \n\t"

  116. 

  117.         "vl  %%v24,96(%%r1,%1)           \n\t"

  118.         "vfmadb   %%v0,%%v22,%%v24,%%v0  \n\t"

  119.         "vl  %%v25,96(%%r1,%2)           \n\t"

  120.         "vfmadb   %%v1,%%v22,%%v25,%%v1  \n\t"

  121.         "vl  %%v26,96(%%r1,%3)           \n\t"

  122.         "vfmadb   %%v2,%%v22,%%v26,%%v2  \n\t"

  123.         "vl  %%v27,96(%%r1,%4)           \n\t"

  124.         "vfmadb   %%v3,%%v22,%%v27,%%v3  \n\t"

  125. 

  126.         "vl  %%v28,112(%%r1,%1)          \n\t"

  127.         "vfmadb   %%v0,%%v23,%%v28,%%v0  \n\t"

  128.         "vl  %%v29,112(%%r1,%2)          \n\t"

  129.         "vfmadb   %%v1,%%v23,%%v29,%%v1  \n\t"

  130.         "vl  %%v30,112(%%r1,%3)          \n\t"

  131.         "vfmadb   %%v2,%%v23,%%v30,%%v2  \n\t"

  132.         "vl  %%v31,112(%%r1,%4)          \n\t"

  133.         "vfmadb   %%v3,%%v23,%%v31,%%v3  \n\t"

  134. 

  135.         "agfi   %%r1,128                 \n\t"

  136.         "brctg  %%r0,0b                  \n\t"

  137. 

  138.         "1:                              \n\t"

  139.         "lghi    %%r0,12                 \n\t"

  140.         "ngr     %%r0,%0                 \n\t"

  141.         "ltgr    %%r0,%%r0               \n\t"

  142.         "jz      3f                      \n\t"

  143. 

  144.         "srlg  %%r0,%%r0,2               \n\t"

  145.         "2:                              \n\t"

  146.         "vl  %%v16,0(%%r1,%5)            \n\t"

  147.         "vl  %%v17,16(%%r1,%5)           \n\t"

  148. 

  149.         "vl  %%v24,0(%%r1,%1)            \n\t"

  150.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  151.         "vl  %%v25,0(%%r1,%2)            \n\t"

  152.         "vfmadb   %%v1,%%v16,%%v25,%%v1  \n\t"

  153.         "vl  %%v26,0(%%r1,%3)            \n\t"

  154.         "vfmadb   %%v2,%%v16,%%v26,%%v2  \n\t"

  155.         "vl  %%v27,0(%%r1,%4)            \n\t"

  156.         "vfmadb   %%v3,%%v16,%%v27,%%v3  \n\t"

  157. 

  158.         "vl  %%v28,16(%%r1,%1)           \n\t"

  159.         "vfmadb   %%v0,%%v17,%%v28,%%v0  \n\t"

  160.         "vl  %%v29,16(%%r1,%2)           \n\t"

  161.         "vfmadb   %%v1,%%v17,%%v29,%%v1  \n\t"

  162.         "vl  %%v30,16(%%r1,%3)           \n\t"

  163.         "vfmadb   %%v2,%%v17,%%v30,%%v2  \n\t"

  164.         "vl  %%v31,16(%%r1,%4)           \n\t"

  165.         "vfmadb   %%v3,%%v17,%%v31,%%v3  \n\t"

  166. 

  167.         "agfi   %%r1,32                  \n\t"

  168.         "brctg  %%r0,2b                  \n\t"

  169. 

  170.         "3:                              \n\t"

  171.         "vrepg  %%v4,%%v0,1              \n\t"

  172.         "adbr   %%f0,%%f4                \n\t"

  173.         "std    %%f0,0(%6)               \n\t"

  174.         "vrepg  %%v4,%%v1,1              \n\t"

  175.         "adbr   %%f1,%%f4                \n\t"

  176.         "std    %%f1,8(%6)               \n\t"

  177.         "vrepg  %%v4,%%v2,1              \n\t"

  178.         "adbr   %%f2,%%f4                \n\t"

  179.         "std    %%f2,16(%6)              \n\t"

  180.         "vrepg  %%v4,%%v3,1              \n\t"

  181.         "adbr   %%f3,%%f4                \n\t"

  182.         "std    %%f3,24(%6)                  "

  183.         :

  184.         :"r"(n),"ZR"((const FLOAT (*)[n])ap[0]),"ZR"((const FLOAT (*)[n])ap[1]),"ZR"((const FLOAT (*)[n])ap[2]),"ZR"((const FLOAT (*)[n])ap[3]),"ZR"((const FLOAT (*)[n])x),"ZQ"((FLOAT (*)[4])y)

  185.         :"memory","cc","r0","r1","v0","v1","v2","v3","v4","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  186.     );

  187. }

  188. 

  189. static void dgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  190. {

  191.     __asm__ volatile (   

  192.         "vzero %%v0                      \n\t"

  193.         "vzero %%v1                      \n\t"

  194.         "xgr   %%r1,%%r1                 \n\t"

  195. 

  196.         "lghi    %%r0,-16                \n\t"

  197.         "ngr     %%r0,%0                 \n\t"

  198.         "ltgr    %%r0,%%r0               \n\t"

  199.         "jz      1f                      \n\t"

  200. 

  201.         "srlg  %%r0,%%r0,4               \n\t"

  202.         "0:                              \n\t"

  203.         "pfd 1,1024(%%r1,%1)             \n\t"

  204.         "pfd 1,1024(%%r1,%2)             \n\t"

  205.         "pfd 1,1024(%%r1,%3)             \n\t"

  206. 

  207.         "vl  %%v16,0(%%r1,%3)            \n\t"

  208.         "vl  %%v17,16(%%r1,%3)           \n\t"

  209.         "vl  %%v18,32(%%r1,%3)           \n\t"

  210.         "vl  %%v19,48(%%r1,%3)           \n\t"

  211.         "vl  %%v20,64(%%r1,%3)           \n\t"

  212.         "vl  %%v21,80(%%r1,%3)           \n\t"

  213.         "vl  %%v22,96(%%r1,%3)           \n\t"

  214.         "vl  %%v23,112(%%r1,%3)          \n\t"

  215. 

  216.         "vl  %%v24,0(%%r1,%1)            \n\t"

  217.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  218.         "vl  %%v25,0(%%r1,%2)            \n\t"

  219.         "vfmadb   %%v1,%%v16,%%v25,%%v1  \n\t"

  220. 

  221.         "vl  %%v26,16(%%r1,%1)           \n\t"

  222.         "vfmadb   %%v0,%%v17,%%v26,%%v0  \n\t"

  223.         "vl  %%v27,16(%%r1,%2)           \n\t"

  224.         "vfmadb   %%v1,%%v17,%%v27,%%v1  \n\t"

  225. 

  226.         "vl  %%v28,32(%%r1,%1)           \n\t"

  227.         "vfmadb   %%v0,%%v18,%%v28,%%v0  \n\t"

  228.         "vl  %%v29,32(%%r1,%2)           \n\t"

  229.         "vfmadb   %%v1,%%v18,%%v29,%%v1  \n\t"

  230. 

  231.         "vl  %%v30,48(%%r1,%1)           \n\t"

  232.         "vfmadb   %%v0,%%v19,%%v30,%%v0  \n\t"

  233.         "vl  %%v31,48(%%r1,%2)           \n\t"

  234.         "vfmadb   %%v1,%%v19,%%v31,%%v1  \n\t"

  235. 

  236.         "vl  %%v24,64(%%r1,%1)           \n\t"

  237.         "vfmadb   %%v0,%%v20,%%v24,%%v0  \n\t"

  238.         "vl  %%v25,64(%%r1,%2)           \n\t"

  239.         "vfmadb   %%v1,%%v20,%%v25,%%v1  \n\t"

  240. 

  241.         "vl  %%v26,80(%%r1,%1)           \n\t"

  242.         "vfmadb   %%v0,%%v21,%%v26,%%v0  \n\t"

  243.         "vl  %%v27,80(%%r1,%2)           \n\t"

  244.         "vfmadb   %%v1,%%v21,%%v27,%%v1  \n\t"

  245. 

  246.         "vl  %%v28,96(%%r1,%1)           \n\t"

  247.         "vfmadb   %%v0,%%v22,%%v28,%%v0  \n\t"

  248.         "vl  %%v29,96(%%r1,%2)           \n\t"

  249.         "vfmadb   %%v1,%%v22,%%v29,%%v1  \n\t"

  250. 

  251.         "vl  %%v30,112(%%r1,%1)          \n\t"

  252.         "vfmadb   %%v0,%%v23,%%v30,%%v0  \n\t"

  253.         "vl  %%v31,112(%%r1,%2)          \n\t"

  254.         "vfmadb   %%v1,%%v23,%%v31,%%v1  \n\t"

  255. 

  256.         "agfi   %%r1,128                 \n\t"

  257.         "brctg  %%r0,0b                  \n\t"

  258. 

  259.         "1:                              \n\t"

  260.         "lghi    %%r0,12                 \n\t"

  261.         "ngr     %%r0,%0                 \n\t"

  262.         "ltgr    %%r0,%%r0               \n\t"

  263.         "jz      3f                      \n\t"

  264. 

  265.         "srlg  %%r0,%%r0,2               \n\t"

  266.         "2:                              \n\t"

  267.         "vl  %%v16,0(%%r1,%3)            \n\t"

  268.         "vl  %%v17,16(%%r1,%3)           \n\t"

  269. 

  270.         "vl  %%v24,0(%%r1,%1)            \n\t"

  271.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  272.         "vl  %%v25,0(%%r1,%2)            \n\t"

  273.         "vfmadb   %%v1,%%v16,%%v25,%%v1  \n\t"

  274. 

  275.         "vl  %%v26,16(%%r1,%1)           \n\t"

  276.         "vfmadb   %%v0,%%v17,%%v26,%%v0  \n\t"

  277.         "vl  %%v27,16(%%r1,%2)           \n\t"

  278.         "vfmadb   %%v1,%%v17,%%v27,%%v1  \n\t"

  279. 

  280.         "agfi   %%r1,32                  \n\t"

  281.         "brctg  %%r0,2b                  \n\t"

  282. 

  283.         "3:                              \n\t"

  284.         "vrepg  %%v2,%%v0,1              \n\t"

  285.         "adbr   %%f0,%%f2                \n\t"

  286.         "std    %%f0,0(%4)               \n\t"

  287.         "vrepg  %%v2,%%v1,1              \n\t"

  288.         "adbr   %%f1,%%f2                \n\t"

  289.         "std    %%f1,8(%4)                   "

  290.         :

  291.         :"r"(n),"ZR"((const FLOAT (*)[n])ap[0]),"ZR"((const FLOAT (*)[n])ap[1]),"ZR"((const FLOAT (*)[n])x),"ZQ"((FLOAT (*)[2])y)

  292.         :"memory","cc","r0","r1","v0","v1","v2","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  293.     );

  294. }

  295. 

  296. static void dgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y)

  297. {

  298.     __asm__ volatile (   

  299.         "vzero %%v0                      \n\t"

  300.         "xgr   %%r1,%%r1                 \n\t"

  301. 

  302.         "lghi    %%r0,-16                \n\t"

  303.         "ngr     %%r0,%0                 \n\t"

  304.         "ltgr    %%r0,%%r0               \n\t"

  305.         "jz      1f                      \n\t"

  306. 

  307.         "srlg  %%r0,%%r0,4               \n\t"

  308.         "0:                              \n\t"

  309.         "pfd 1,1024(%%r1,%1)             \n\t"

  310.         "pfd 1,1024(%%r1,%2)             \n\t"

  311. 

  312.         "vl  %%v16,0(%%r1,%2)            \n\t"

  313.         "vl  %%v17,16(%%r1,%2)           \n\t"

  314.         "vl  %%v18,32(%%r1,%2)           \n\t"

  315.         "vl  %%v19,48(%%r1,%2)           \n\t"

  316.         "vl  %%v20,64(%%r1,%2)           \n\t"

  317.         "vl  %%v21,80(%%r1,%2)           \n\t"

  318.         "vl  %%v22,96(%%r1,%2)           \n\t"

  319.         "vl  %%v23,112(%%r1,%2)          \n\t"

  320. 

  321.         "vl  %%v24,0(%%r1,%1)            \n\t"

  322.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  323. 

  324.         "vl  %%v25,16(%%r1,%1)           \n\t"

  325.         "vfmadb   %%v0,%%v17,%%v25,%%v0  \n\t"

  326. 

  327.         "vl  %%v26,32(%%r1,%1)           \n\t"

  328.         "vfmadb   %%v0,%%v18,%%v26,%%v0  \n\t"

  329. 

  330.         "vl  %%v27,48(%%r1,%1)           \n\t"

  331.         "vfmadb   %%v0,%%v19,%%v27,%%v0  \n\t"

  332. 

  333.         "vl  %%v28,64(%%r1,%1)           \n\t"

  334.         "vfmadb   %%v0,%%v20,%%v28,%%v0  \n\t"

  335. 

  336.         "vl  %%v29,80(%%r1,%1)           \n\t"

  337.         "vfmadb   %%v0,%%v21,%%v29,%%v0  \n\t"

  338. 

  339.         "vl  %%v30,96(%%r1,%1)           \n\t"

  340.         "vfmadb   %%v0,%%v22,%%v30,%%v0  \n\t"

  341. 

  342.         "vl  %%v31,112(%%r1,%1)          \n\t"

  343.         "vfmadb   %%v0,%%v23,%%v31,%%v0  \n\t"

  344. 

  345.         "agfi   %%r1,128                 \n\t"

  346.         "brctg  %%r0,0b                  \n\t"

  347. 

  348.         "1:                              \n\t"

  349.         "lghi    %%r0,12                 \n\t"

  350.         "ngr     %%r0,%0                 \n\t"

  351.         "ltgr    %%r0,%%r0               \n\t"

  352.         "jz      3f                      \n\t"

  353. 

  354.         "srlg  %%r0,%%r0,2               \n\t"

  355.         "2:                              \n\t"

  356.         "vl  %%v16,0(%%r1,%2)            \n\t"

  357.         "vl  %%v17,16(%%r1,%2)           \n\t"

  358. 

  359.         "vl  %%v24,0(%%r1,%1)            \n\t"

  360.         "vfmadb   %%v0,%%v16,%%v24,%%v0  \n\t"

  361. 

  362.         "vl  %%v25,16(%%r1,%1)           \n\t"

  363.         "vfmadb   %%v0,%%v17,%%v25,%%v0  \n\t"

  364. 

  365.         "agfi   %%r1,32                  \n\t"

  366.         "brctg  %%r0,2b                  \n\t"

  367. 

  368.         "3:                              \n\t"

  369.         "vrepg  %%v1,%%v0,1              \n\t"

  370.         "adbr   %%f0,%%f1                \n\t"

  371.         "std    %%f0,0(%3)                   "

  372.         :

  373.         :"r"(n),"ZR"((const FLOAT (*)[n])a0),"ZR"((const FLOAT (*)[n])x),"ZQ"((FLOAT (*)[1])y)

  374.         :"memory","cc","r0","r1","v0","v1","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  375.     );

  376. }

  377. 

  378. static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)

  379. {

  380.     BLASLONG i;

  381.     for (i = 0; i < n; i++)

  382.     {

  383.         dest[i] = *src;

  384.         src += inc_src;

  385.     }

  386. }

  387.  

  388. static void add_y_kernel_4(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest)

  389. {

  390.     __asm__ volatile (   

  391.         "vlrepg %%v0,%1                   \n\t"

  392.         "xgr   %%r1,%%r1                  \n\t"

  393. 

  394.         "lghi    %%r0,-16                 \n\t"

  395.         "ngr     %%r0,%0                  \n\t"

  396.         "ltgr    %%r0,%%r0                \n\t"

  397.         "jz      1f                       \n\t"

  398. 

  399.         "srlg  %%r0,%%r0,4                \n\t"

  400.         "0:                               \n\t"

  401.         "pfd 1,1024(%%r1,%2)              \n\t"

  402.         "pfd 2,1024(%%r1,%3)              \n\t"

  403. 

  404.         "vl  %%v16,0(%%r1,%2)             \n\t"

  405.         "vl  %%v17,16(%%r1,%2)            \n\t"

  406.         "vl  %%v18,32(%%r1,%2)            \n\t"

  407.         "vl  %%v19,48(%%r1,%2)            \n\t"

  408.         "vl  %%v20,64(%%r1,%2)            \n\t"

  409.         "vl  %%v21,80(%%r1,%2)            \n\t"

  410.         "vl  %%v22,96(%%r1,%2)            \n\t"

  411.         "vl  %%v23,112(%%r1,%2)           \n\t"

  412. 

  413.         "vl  %%v24, 0(%%r1,%3)            \n\t"

  414.         "vfmadb   %%v24,%%v16,%%v0,%%v24  \n\t"

  415.         "vst  %%v24, 0(%%r1,%3)           \n\t"

  416.         "vl  %%v25, 16(%%r1,%3)           \n\t"

  417.         "vfmadb   %%v25,%%v17,%%v0,%%v25  \n\t"

  418.         "vst  %%v25, 16(%%r1,%3)          \n\t"

  419.         "vl  %%v26, 32(%%r1,%3)           \n\t"

  420.         "vfmadb   %%v26,%%v18,%%v0,%%v26  \n\t"

  421.         "vst  %%v26, 32(%%r1,%3)          \n\t"

  422.         "vl  %%v27, 48(%%r1,%3)           \n\t"

  423.         "vfmadb   %%v27,%%v19,%%v0,%%v27  \n\t"

  424.         "vst  %%v27, 48(%%r1,%3)          \n\t"

  425.         "vl  %%v28, 64(%%r1,%3)           \n\t"

  426.         "vfmadb   %%v28,%%v20,%%v0,%%v28  \n\t"

  427.         "vst  %%v28, 64(%%r1,%3)          \n\t"

  428.         "vl  %%v29, 80(%%r1,%3)           \n\t"

  429.         "vfmadb   %%v29,%%v21,%%v0,%%v29  \n\t"

  430.         "vst  %%v29, 80(%%r1,%3)          \n\t"

  431.         "vl  %%v30, 96(%%r1,%3)           \n\t"

  432.         "vfmadb   %%v30,%%v22,%%v0,%%v30  \n\t"

  433.         "vst  %%v30, 96(%%r1,%3)          \n\t"

  434.         "vl  %%v31, 112(%%r1,%3)          \n\t"

  435.         "vfmadb   %%v31,%%v23,%%v0,%%v31  \n\t"

  436.         "vst  %%v31, 112(%%r1,%3)         \n\t"

  437. 

  438.         "agfi   %%r1,128                  \n\t"

  439.         "brctg  %%r0,0b                   \n\t"

  440. 

  441.         "1:                               \n\t"

  442.         "lghi    %%r0,12                  \n\t"

  443.         "ngr     %%r0,%0                  \n\t"

  444.         "ltgr    %%r0,%%r0                \n\t"

  445.         "jz      3f                       \n\t"

  446. 

  447.         "srlg  %%r0,%%r0,2                \n\t"

  448.         "2:                               \n\t"

  449.         "vl  %%v16,0(%%r1,%2)             \n\t"

  450.         "vl  %%v17,16(%%r1,%2)            \n\t"

  451. 

  452.         "vl  %%v24, 0(%%r1,%3)            \n\t"

  453.         "vfmadb   %%v24,%%v16,%%v0,%%v24  \n\t"

  454.         "vst  %%v24, 0(%%r1,%3)           \n\t"

  455.         "vl  %%v25, 16(%%r1,%3)           \n\t"

  456.         "vfmadb   %%v25,%%v17,%%v0,%%v25  \n\t"

  457.         "vst  %%v25, 16(%%r1,%3)          \n\t"

  458. 

  459.         "agfi   %%r1,32                   \n\t"

  460.         "brctg  %%r0,2b                   \n\t"

  461. 

  462.         "3:                               \n\t"

  463.         "nop                                  "

  464.         :

  465.         :"r"(n),"m"(da),"ZR"((const FLOAT (*)[n])src),"ZR"((FLOAT (*)[n])dest)

  466.         :"memory","cc","r0","r1","v0","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  467.     );

  468. }

  469. static void add_y(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)

  470. {

  471.     if (inc_dest == 1)

  472.         add_y_kernel_4(n, da, src, dest);  

  473.     else

  474.     {

  475.         BLASLONG i;

  476.         for (i = 0; i < n; i++)

  477.         {

  478.             *dest += src[i] * da;

  479.             dest  += inc_dest;

  480.         }

  481.     }

  482. }

  483. 

  484. int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)

  485. {

  486.     BLASLONG register i;

  487.     BLASLONG register j;

  488.     FLOAT *a_ptr;

  489.     FLOAT *x_ptr;

  490.     FLOAT *y_ptr;

  491.     BLASLONG n0;

  492.     BLASLONG n1;

  493.     BLASLONG m1;

  494.     BLASLONG m2;

  495.     BLASLONG m3;

  496.     BLASLONG n2;

  497.     FLOAT ybuffer[2] __attribute__ ((aligned(16)));

  498.     FLOAT *xbuffer;

  499.     FLOAT *ytemp;

  500. 

  501.     if ( m < 1 ) return(0);

  502.     if ( n < 1 ) return(0);

  503. 

  504.     xbuffer = buffer;

  505.     ytemp   = buffer + (m < NBMAX ? m : NBMAX);

  506.     

  507.     n0 = n / NBMAX;

  508.     n1 = (n % NBMAX)  >> 2 ;

  509.     n2 = n & 3  ;

  510. 

  511.     m3 = m & 3  ;

  512.     m1 = m & -4 ;

  513.     m2 = (m & (NBMAX-1)) - m3 ;

  514. 

  515. 

  516.     BLASLONG NB = NBMAX;

  517. 

  518.     while ( NB == NBMAX )

  519.     {

  520.         m1 -= NB;

  521.         if ( m1 < 0)

  522.         {

  523.             if ( m2 == 0 ) break;    

  524.             NB = m2;

  525.         }

  526.         

  527.         y_ptr = y;

  528.         a_ptr = a;

  529.         x_ptr = x;

  530. 

  531.         if ( inc_x == 1 )

  532.             xbuffer = x_ptr;

  533.         else

  534.             copy_x(NB,x_ptr,xbuffer,inc_x);

  535. 

  536. 

  537.         FLOAT *ap[4];

  538.         FLOAT *yp;

  539.         BLASLONG register lda4 = 4 * lda;

  540.         ap[0] = a_ptr;

  541.         ap[1] = a_ptr + lda;

  542.         ap[2] = ap[1] + lda;

  543.         ap[3] = ap[2] + lda;

  544. 

  545.         if ( n0 > 0 )

  546.         {

  547.             BLASLONG nb1 = NBMAX / 4;

  548.             for( j=0; j<n0; j++)

  549.             {

  550. 

  551.                 yp = ytemp;

  552.                 for( i = 0; i < nb1  ; i++)

  553.                 {

  554.                     dgemv_kernel_4x4(NB,ap,xbuffer,yp);

  555.                     ap[0] += lda4 ;

  556.                     ap[1] += lda4 ;

  557.                     ap[2] += lda4 ;

  558.                     ap[3] += lda4 ;

  559.                     yp += 4;

  560.                 }

  561.                 add_y(nb1*4, alpha, ytemp, y_ptr, inc_y );

  562.                 y_ptr += nb1 * inc_y * 4;

  563.                 a_ptr += nb1 * lda4 ;

  564. 

  565.             }

  566. 

  567.         }

  568. 

  569. 

  570.         yp = ytemp;

  571. 

  572.         for( i = 0; i < n1 ; i++)

  573.         {

  574.             dgemv_kernel_4x4(NB,ap,xbuffer,yp);

  575.             ap[0] += lda4 ;

  576.             ap[1] += lda4 ;

  577.             ap[2] += lda4 ;

  578.             ap[3] += lda4 ;

  579.             yp += 4;

  580.         }

  581.         if ( n1 > 0 )

  582.         {

  583.             add_y(n1*4, alpha, ytemp, y_ptr, inc_y );

  584.             y_ptr += n1 * inc_y * 4;

  585.             a_ptr += n1 * lda4 ;

  586.         }

  587. 

  588.         if ( n2 & 2 )

  589.         {

  590. 

  591.             dgemv_kernel_4x2(NB,ap,xbuffer,ybuffer);

  592.             a_ptr  += lda * 2;

  593.             *y_ptr += ybuffer[0] * alpha;

  594.             y_ptr  += inc_y;

  595.             *y_ptr += ybuffer[1] * alpha;

  596.             y_ptr  += inc_y;

  597. 

  598.         }

  599. 

  600.         if ( n2 & 1 )

  601.         {

  602. 

  603.             dgemv_kernel_4x1(NB,a_ptr,xbuffer,ybuffer);

  604.             // a_ptr  += lda;

  605.             *y_ptr += ybuffer[0] * alpha;

  606.             // y_ptr  += inc_y;

  607. 

  608.         }

  609.         a += NB;

  610.         x += NB * inc_x;    

  611.     }

  612. 

  613.     if ( m3 == 0 ) return(0);

  614. 

  615.     x_ptr = x;

  616.     a_ptr = a;

  617.     if ( m3 == 3 )

  618.     {

  619.         FLOAT xtemp0 = *x_ptr * alpha;

  620.         x_ptr += inc_x;

  621.         FLOAT xtemp1 = *x_ptr * alpha;

  622.         x_ptr += inc_x;

  623.         FLOAT xtemp2 = *x_ptr * alpha;

  624. 

  625.         FLOAT *aj = a_ptr;

  626.         y_ptr = y;

  627. 

  628.         if ( lda == 3 && inc_y == 1 )

  629.         {

  630. 

  631.             for ( j=0; j< ( n & -4) ; j+=4 )

  632.             {

  633. 

  634.                 y_ptr[j]   += aj[0] * xtemp0 + aj[1]  * xtemp1 + aj[2]  * xtemp2;

  635.                 y_ptr[j+1] += aj[3] * xtemp0 + aj[4]  * xtemp1 + aj[5]  * xtemp2;

  636.                 y_ptr[j+2] += aj[6] * xtemp0 + aj[7]  * xtemp1 + aj[8]  * xtemp2;

  637.                 y_ptr[j+3] += aj[9] * xtemp0 + aj[10] * xtemp1 + aj[11] * xtemp2;

  638.                  aj        += 12;

  639.             }

  640. 

  641.             for ( ; j<n; j++ )

  642.             {

  643.                 y_ptr[j]  += aj[0] * xtemp0 + aj[1] * xtemp1 + aj[2] * xtemp2;

  644.                  aj        += 3;

  645.             }

  646. 

  647.         }

  648.         else

  649.         {

  650. 

  651.             if ( inc_y == 1 )

  652.             {

  653. 

  654.                 BLASLONG register lda2 = lda << 1;

  655.                 BLASLONG register lda4 = lda << 2;

  656.                 BLASLONG register lda3 = lda2 + lda;

  657. 

  658.                 for ( j=0; j< ( n & -4 ); j+=4 )

  659.                 {

  660. 

  661.                     y_ptr[j]    += *aj        * xtemp0 + *(aj+1)      * xtemp1 + *(aj+2)      * xtemp2;

  662.                     y_ptr[j+1]  += *(aj+lda)  * xtemp0 + *(aj+lda+1)  * xtemp1 + *(aj+lda+2)  * xtemp2;

  663.                     y_ptr[j+2]  += *(aj+lda2) * xtemp0 + *(aj+lda2+1) * xtemp1 + *(aj+lda2+2) * xtemp2;

  664.                     y_ptr[j+3]  += *(aj+lda3) * xtemp0 + *(aj+lda3+1) * xtemp1 + *(aj+lda3+2) * xtemp2;

  665.                      aj          += lda4;

  666.                 }

  667. 

  668.                 for ( ; j< n ; j++ )

  669.                 {

  670. 

  671.                     y_ptr[j]    += *aj * xtemp0 + *(aj+1) * xtemp1 + *(aj+2) * xtemp2 ;

  672.                      aj          += lda;

  673.                 }

  674. 

  675.             }

  676.             else

  677.             {

  678. 

  679.                 for ( j=0; j<n; j++ )

  680.                 {

  681.                     *y_ptr += *aj * xtemp0 + *(aj+1) * xtemp1 + *(aj+2) * xtemp2;

  682.                      y_ptr += inc_y;

  683.                      aj    += lda;

  684.                 }

  685. 

  686. 

  687.             }

  688. 

  689.         }

  690.         return(0);

  691.     }

  692. 

  693.     if ( m3 == 2 )

  694.     {

  695.         FLOAT xtemp0 = *x_ptr * alpha;

  696.         x_ptr += inc_x;

  697.         FLOAT xtemp1 = *x_ptr * alpha;

  698. 

  699.         FLOAT *aj = a_ptr;

  700.         y_ptr = y;

  701. 

  702.         if ( lda == 2 && inc_y == 1 )

  703.         {

  704. 

  705.             for ( j=0; j< ( n & -4) ; j+=4 )

  706.             {

  707.                 y_ptr[j]   += aj[0] * xtemp0 + aj[1] * xtemp1 ;

  708.                 y_ptr[j+1] += aj[2] * xtemp0 + aj[3] * xtemp1 ;

  709.                 y_ptr[j+2] += aj[4] * xtemp0 + aj[5] * xtemp1 ;

  710.                 y_ptr[j+3] += aj[6] * xtemp0 + aj[7] * xtemp1 ;

  711.                  aj         += 8;

  712. 

  713.             }

  714. 

  715.             for ( ; j<n; j++ )

  716.             {

  717.                 y_ptr[j] += aj[0] * xtemp0 + aj[1] * xtemp1 ;

  718.                  aj       += 2;

  719.             }

  720. 

  721.         }

  722.         else

  723.         {

  724.             if ( inc_y == 1 )

  725.             {

  726. 

  727.                 BLASLONG register lda2 = lda << 1;

  728.                 BLASLONG register lda4 = lda << 2;

  729.                 BLASLONG register lda3 = lda2 + lda;

  730. 

  731.                 for ( j=0; j< ( n & -4 ); j+=4 )

  732.                 {

  733. 

  734.                     y_ptr[j]    += *aj        * xtemp0 + *(aj+1)      * xtemp1 ;

  735.                     y_ptr[j+1]  += *(aj+lda)  * xtemp0 + *(aj+lda+1)  * xtemp1 ;

  736.                     y_ptr[j+2]  += *(aj+lda2) * xtemp0 + *(aj+lda2+1) * xtemp1 ;

  737.                     y_ptr[j+3]  += *(aj+lda3) * xtemp0 + *(aj+lda3+1) * xtemp1 ;

  738.                      aj          += lda4;

  739.                 }

  740. 

  741.                 for ( ; j< n ; j++ )

  742.                 {

  743. 

  744.                     y_ptr[j]    += *aj * xtemp0 + *(aj+1) * xtemp1 ;

  745.                      aj          += lda;

  746.                 }

  747. 

  748.             }

  749.             else

  750.             {

  751.                 for ( j=0; j<n; j++ )

  752.                 {

  753.                     *y_ptr += *aj * xtemp0 + *(aj+1) * xtemp1 ;

  754.                      y_ptr += inc_y;

  755.                      aj    += lda;

  756.                 }

  757.             }

  758. 

  759.         }

  760.         return(0);

  761. 

  762.     }

  763. 

  764.     FLOAT xtemp = *x_ptr * alpha;

  765.     FLOAT *aj = a_ptr;

  766.     y_ptr = y;

  767.     if ( lda == 1 && inc_y == 1 )

  768.     {

  769.         for ( j=0; j< ( n & -4) ; j+=4 )

  770.         {

  771.             y_ptr[j]   += aj[j]   * xtemp;

  772.             y_ptr[j+1] += aj[j+1] * xtemp;

  773.             y_ptr[j+2] += aj[j+2] * xtemp;

  774.             y_ptr[j+3] += aj[j+3] * xtemp;

  775.         }

  776.         for ( ; j<n   ; j++ )

  777.         {

  778.             y_ptr[j] += aj[j] * xtemp;

  779.         }

  780. 

  781. 

  782. 

  783.     }

  784.     else

  785.     {

  786.         if ( inc_y == 1 )

  787.         {

  788. 

  789.             BLASLONG register lda2 = lda << 1;

  790.             BLASLONG register lda4 = lda << 2;

  791.             BLASLONG register lda3 = lda2 + lda;

  792.             for ( j=0; j< ( n & -4 ); j+=4 )

  793.             {

  794.                 y_ptr[j]    += *aj        * xtemp;

  795.                 y_ptr[j+1]  += *(aj+lda)  * xtemp;

  796.                 y_ptr[j+2]  += *(aj+lda2) * xtemp;

  797.                 y_ptr[j+3]  += *(aj+lda3) * xtemp;

  798.                  aj          += lda4  ;

  799.             }

  800. 

  801.             for ( ; j<n; j++ )

  802.             {

  803.                 y_ptr[j]  += *aj * xtemp;

  804.                  aj        += lda;

  805.             }

  806. 

  807.         }

  808.         else

  809.         {

  810.             for ( j=0; j<n; j++ )

  811.             {

  812.                 *y_ptr += *aj * xtemp;

  813.                  y_ptr += inc_y;

  814.                  aj    += lda;

  815.             }

  816. 

  817.         }

  818.     }

  819. 

  820.     return(0);

  821. }

  822.