OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f4b82d7bc4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f4b82d7bc4'
${ noResults }
OpenBLAS/kernel/zarch/cgemv_n_4.c

744 lines
23 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include <stdlib.h>

  29. #include <stdio.h>

  30. #include "common.h"

  31. 

  32. #define NBMAX 2048

  33. 

  34. static void cgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  35. {

  36.     __asm__ volatile (

  37. 	"vlrepg     %%v16,0(%5)           \n\t"

  38. 	"vlrepg     %%v17,8(%5)           \n\t"

  39. 	"vlrepg     %%v18,16(%5)          \n\t"

  40. 	"vlrepg     %%v19,24(%5)          \n\t"

  41. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  42. 	"vlef   %%v20,4(%5),0             \n\t"

  43. 	"vlef   %%v20,4(%5),2             \n\t"

  44. 	"vflcsb %%v20,%%v20               \n\t"

  45. 	"vlef   %%v20,0(%5),1             \n\t"

  46. 	"vlef   %%v20,0(%5),3             \n\t"

  47. 

  48. 	"vlef   %%v21,12(%5),0            \n\t"

  49. 	"vlef   %%v21,12(%5),2            \n\t"

  50. 	"vflcsb %%v21,%%v21               \n\t"

  51. 	"vlef   %%v21,8(%5),1             \n\t"

  52. 	"vlef   %%v21,8(%5),3             \n\t"

  53. 

  54. 	"vlef   %%v22,20(%5),0            \n\t"

  55. 	"vlef   %%v22,20(%5),2            \n\t"

  56. 	"vflcsb %%v22,%%v22               \n\t"

  57. 	"vlef   %%v22,16(%5),1            \n\t"

  58. 	"vlef   %%v22,16(%5),3            \n\t"

  59. 

  60. 	"vlef   %%v23,28(%5),0            \n\t"

  61. 	"vlef   %%v23,28(%5),2            \n\t"

  62. 	"vflcsb %%v23,%%v23               \n\t"

  63. 	"vlef   %%v23,24(%5),1            \n\t"

  64. 	"vlef   %%v23,24(%5),3            \n\t"

  65. #else

  66. 	"vlef   %%v20,0(%5),1             \n\t"

  67. 	"vlef   %%v20,0(%5),3             \n\t"

  68. 	"vflcsb %%v20,%%v20               \n\t"

  69. 	"vlef   %%v20,4(%5),0             \n\t"

  70. 	"vlef   %%v20,4(%5),2             \n\t"

  71. 

  72. 	"vlef   %%v21,8(%5),1             \n\t"

  73. 	"vlef   %%v21,8(%5),3             \n\t"

  74. 	"vflcsb %%v21,%%v21               \n\t"

  75. 	"vlef   %%v21,12(%5),0            \n\t"

  76. 	"vlef   %%v21,12(%5),2            \n\t"

  77. 

  78. 	"vlef   %%v22,16(%5),1            \n\t"

  79. 	"vlef   %%v22,16(%5),3            \n\t"

  80. 	"vflcsb %%v22,%%v22               \n\t"

  81. 	"vlef   %%v22,20(%5),0            \n\t"

  82. 	"vlef   %%v22,20(%5),2            \n\t"

  83. 

  84. 	"vlef   %%v23,24(%5),1            \n\t"

  85. 	"vlef   %%v23,24(%5),3            \n\t"

  86. 	"vflcsb %%v23,%%v23               \n\t"

  87. 	"vlef   %%v23,28(%5),0            \n\t"

  88. 	"vlef   %%v23,28(%5),2            \n\t"

  89. #endif

  90. 	"xgr   %%r1,%%r1                  \n\t"

  91. 	"srlg  %%r0,%0,1                  \n\t"

  92. 	"0:                               \n\t"

  93. 	"pfd 1,1024(%%r1,%1)              \n\t"

  94. 	"pfd 1,1024(%%r1,%2)              \n\t"

  95. 	"pfd 1,1024(%%r1,%3)              \n\t"

  96. 	"pfd 1,1024(%%r1,%4)              \n\t"

  97. 	"pfd 2,1024(%%r1,%6)              \n\t"

  98. 

  99. 	"vlef   %%v24,0(%%r1,%1),0        \n\t"

  100. 	"vlef   %%v24,0(%%r1,%1),1        \n\t"

  101. 	"vlef   %%v24,8(%%r1,%1),2        \n\t"

  102. 	"vlef   %%v24,8(%%r1,%1),3        \n\t"

  103. 	"vlef   %%v25,4(%%r1,%1),0        \n\t"

  104. 	"vlef   %%v25,4(%%r1,%1),1        \n\t"

  105. 	"vlef   %%v25,12(%%r1,%1),2       \n\t"

  106. 	"vlef   %%v25,12(%%r1,%1),3       \n\t"

  107. 	"vlef   %%v26,0(%%r1,%2),0        \n\t"

  108. 	"vlef   %%v26,0(%%r1,%2),1        \n\t"

  109. 	"vlef   %%v26,8(%%r1,%2),2        \n\t"

  110. 	"vlef   %%v26,8(%%r1,%2),3        \n\t"

  111. 	"vlef   %%v27,4(%%r1,%2),0        \n\t"

  112. 	"vlef   %%v27,4(%%r1,%2),1        \n\t"

  113. 	"vlef   %%v27,12(%%r1,%2),2       \n\t"

  114. 	"vlef   %%v27,12(%%r1,%2),3       \n\t"

  115. 

  116. 	"vl  %%v0,0(%%r1,%6)              \n\t"

  117. 	"vfmasb   %%v0,%%v24,%%v16,%%v0   \n\t"

  118. 	"vfmasb   %%v0,%%v25,%%v20,%%v0   \n\t"

  119. 	"vfmasb   %%v0,%%v26,%%v17,%%v0   \n\t"

  120. 	"vfmasb   %%v0,%%v27,%%v21,%%v0   \n\t"

  121. 

  122. 	"vlef   %%v28,0(%%r1,%3),0        \n\t"

  123. 	"vlef   %%v28,0(%%r1,%3),1        \n\t"

  124. 	"vlef   %%v28,8(%%r1,%3),2        \n\t"

  125. 	"vlef   %%v28,8(%%r1,%3),3        \n\t"

  126. 	"vlef   %%v29,4(%%r1,%3),0        \n\t"

  127. 	"vlef   %%v29,4(%%r1,%3),1        \n\t"

  128. 	"vlef   %%v29,12(%%r1,%3),2       \n\t"

  129. 	"vlef   %%v29,12(%%r1,%3),3       \n\t"

  130. 	"vlef   %%v30,0(%%r1,%4),0        \n\t"

  131. 	"vlef   %%v30,0(%%r1,%4),1        \n\t"

  132. 	"vlef   %%v30,8(%%r1,%4),2        \n\t"

  133. 	"vlef   %%v30,8(%%r1,%4),3        \n\t"

  134. 	"vlef   %%v31,4(%%r1,%4),0        \n\t"

  135. 	"vlef   %%v31,4(%%r1,%4),1        \n\t"

  136. 	"vlef   %%v31,12(%%r1,%4),2       \n\t"

  137. 	"vlef   %%v31,12(%%r1,%4),3       \n\t"

  138. 

  139.         "vfmasb   %%v0,%%v28,%%v18,%%v0   \n\t"

  140.         "vfmasb   %%v0,%%v29,%%v22,%%v0   \n\t"

  141.         "vfmasb   %%v0,%%v30,%%v19,%%v0   \n\t"

  142.         "vfmasb   %%v0,%%v31,%%v23,%%v0   \n\t"

  143.         "vst %%v0,0(%%r1,%6)              \n\t"

  144.         

  145.         "agfi   %%r1,16                   \n\t"

  146.         "brctg  %%r0,0b                   \n\t"

  147.         :

  148.         :"r"(n),"ZR"((const FLOAT (*)[n * 2])ap[0]),"ZR"((const FLOAT (*)[n * 2])ap[1]),"ZR"((const FLOAT (*)[n * 2])ap[2]),"ZR"((const FLOAT (*)[n * 2])ap[3]),"ZQ"((const FLOAT (*)[8])x),"ZR"((FLOAT (*)[n * 2])y)

  149.         :"memory","cc","r0","r1","v0","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26","v27","v28","v29","v30","v31"

  150.     );

  151. }

  152. 

  153. static void cgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  154. {

  155.     __asm__ volatile (

  156. 	"vlrepg     %%v16,0(%3)           \n\t"

  157. 	"vlrepg     %%v17,8(%3)           \n\t"

  158. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  159. 	"vlef   %%v18,4(%3),0             \n\t"

  160. 	"vlef   %%v18,4(%3),2             \n\t"

  161. 	"vflcsb %%v18,%%v18               \n\t"

  162. 	"vlef   %%v18,0(%3),1             \n\t"

  163. 	"vlef   %%v18,0(%3),3             \n\t"

  164. 

  165. 	"vlef   %%v19,12(%3),0            \n\t"

  166. 	"vlef   %%v19,12(%3),2            \n\t"

  167. 	"vflcsb %%v19,%%v19               \n\t"

  168. 	"vlef   %%v19,8(%3),1             \n\t"

  169. 	"vlef   %%v19,8(%3),3             \n\t"

  170. #else

  171. 	"vlef   %%v18,0(%3),1             \n\t"

  172. 	"vlef   %%v18,0(%3),3             \n\t"

  173. 	"vflcsb %%v18,%%v18               \n\t"

  174. 	"vlef   %%v18,4(%3),0             \n\t"

  175. 	"vlef   %%v18,4(%3),2             \n\t"

  176. 

  177. 	"vlef   %%v19,8(%3),1             \n\t"

  178. 	"vlef   %%v19,8(%3),3             \n\t"

  179. 	"vflcsb %%v19,%%v19               \n\t"

  180. 	"vlef   %%v19,12(%3),0            \n\t"

  181. 	"vlef   %%v19,12(%3),2            \n\t"

  182. #endif

  183. 	"xgr   %%r1,%%r1                  \n\t"

  184. 	"srlg  %%r0,%0,1                  \n\t"

  185. 	"0:                               \n\t"

  186. 	"pfd 1,1024(%%r1,%1)              \n\t"

  187. 	"pfd 1,1024(%%r1,%2)              \n\t"

  188. 	"pfd 2,1024(%%r1,%4)              \n\t"

  189. 

  190. 	"vlef   %%v20,0(%%r1,%1),0        \n\t"

  191. 	"vlef   %%v20,0(%%r1,%1),1        \n\t"

  192. 	"vlef   %%v20,8(%%r1,%1),2        \n\t"

  193. 	"vlef   %%v20,8(%%r1,%1),3        \n\t"

  194. 	"vlef   %%v21,4(%%r1,%1),0        \n\t"

  195. 	"vlef   %%v21,4(%%r1,%1),1        \n\t"

  196. 	"vlef   %%v21,12(%%r1,%1),2       \n\t"

  197. 	"vlef   %%v21,12(%%r1,%1),3       \n\t"

  198. 	"vlef   %%v22,0(%%r1,%2),0        \n\t"

  199. 	"vlef   %%v22,0(%%r1,%2),1        \n\t"

  200. 	"vlef   %%v22,8(%%r1,%2),2        \n\t"

  201. 	"vlef   %%v22,8(%%r1,%2),3        \n\t"

  202. 	"vlef   %%v23,4(%%r1,%2),0        \n\t"

  203. 	"vlef   %%v23,4(%%r1,%2),1        \n\t"

  204. 	"vlef   %%v23,12(%%r1,%2),2       \n\t"

  205. 	"vlef   %%v23,12(%%r1,%2),3       \n\t"

  206. 

  207.         "vl  %%v0,0(%%r1,%4)              \n\t"

  208.         "vfmasb   %%v0,%%v20,%%v16,%%v0   \n\t"

  209.         "vfmasb   %%v0,%%v21,%%v18,%%v0   \n\t"

  210.         "vfmasb   %%v0,%%v22,%%v17,%%v0   \n\t"

  211.         "vfmasb   %%v0,%%v23,%%v19,%%v0   \n\t"

  212.         "vst %%v0,0(%%r1,%4)              \n\t"

  213.         

  214.         "agfi   %%r1,16                   \n\t"

  215.         "brctg  %%r0,0b                   \n\t"

  216.         :

  217.         :"r"(n),"ZR"((const FLOAT (*)[n * 2])ap[0]),"ZR"((const FLOAT (*)[n * 2])ap[1]),"ZQ"((const FLOAT (*)[4])x),"ZR"((FLOAT (*)[n * 2])y)

  218.         :"memory","cc","r0","r1","v0","v16","v17","v18","v19","v20","v21","v22","v23"

  219.     );

  220. }

  221. 

  222. static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)

  223. {

  224.     __asm__ volatile (

  225. 	"vlrepg     %%v16,0(%2)           \n\t"

  226. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  227.         "vlef   %%v17,4(%2),0             \n\t"

  228. 	"vlef   %%v17,4(%2),2             \n\t"

  229.         "vflcsb %%v17,%%v17               \n\t"

  230.         "vlef   %%v17,0(%2),1             \n\t"

  231. 	"vlef   %%v17,0(%2),3             \n\t"

  232. #else

  233.         "vlef   %%v17,0(%2),1             \n\t"

  234. 	"vlef   %%v17,0(%2),3             \n\t"

  235.         "vflcsb %%v17,%%v17               \n\t"

  236.         "vlef   %%v17,4(%2),0             \n\t"

  237. 	"vlef   %%v17,4(%2),2             \n\t"

  238. #endif

  239.         "xgr   %%r1,%%r1                  \n\t"

  240.         "srlg  %%r0,%0,1                  \n\t"

  241.         "0:                               \n\t"

  242.         "pfd 1,1024(%%r1,%1)              \n\t"

  243.         "pfd 2,1024(%%r1,%3)              \n\t"

  244. 

  245. 	"vlef   %%v18,0(%%r1,%1),0        \n\t"

  246. 	"vlef   %%v18,0(%%r1,%1),1        \n\t"

  247. 	"vlef   %%v18,8(%%r1,%1),2        \n\t"

  248. 	"vlef   %%v18,8(%%r1,%1),3        \n\t"

  249. 	"vlef   %%v19,4(%%r1,%1),0        \n\t"

  250. 	"vlef   %%v19,4(%%r1,%1),1        \n\t"

  251. 	"vlef   %%v19,12(%%r1,%1),2       \n\t"

  252. 	"vlef   %%v19,12(%%r1,%1),3       \n\t"

  253. 

  254.         "vl  %%v0,0(%%r1,%3)              \n\t"

  255.         "vfmasb   %%v0,%%v18,%%v16,%%v0   \n\t"

  256.         "vfmasb   %%v0,%%v19,%%v17,%%v0   \n\t"

  257.         "vst %%v0,0(%%r1,%3)              \n\t"

  258.         

  259.         "agfi   %%r1,16                   \n\t"

  260.         "brctg  %%r0,0b                   \n\t"

  261.         :

  262.         :"r"(n),"ZR"((const FLOAT (*)[n * 2])ap),"ZQ"((const FLOAT (*)[2])x),"ZR"((FLOAT (*)[n * 2])y)

  263.         :"memory","cc","r0","r1","v0","v16","v17","v18","v19"

  264.     );

  265. }

  266. 

  267. static void add_y_4(BLASLONG n, FLOAT *src, FLOAT *dest, FLOAT alpha_r, FLOAT alpha_i)

  268. {

  269.     __asm__ volatile (

  270. #if !defined(XCONJ) 

  271. 	"vlrepf %%v0,%3                 \n\t"

  272. 	"vlef   %%v1,%4,0               \n\t"

  273. 	"vlef   %%v1,%4,2               \n\t"

  274.         "vflcsb %%v1,%%v1               \n\t"

  275. 	"vlef   %%v1,%4,1               \n\t"

  276.         "vlef   %%v1,%4,3               \n\t"

  277. #else

  278.         "vlef   %%v0,%3,1               \n\t"

  279. 	"vlef   %%v0,%3,3               \n\t"

  280.         "vflcsb %%v0,%%v0               \n\t"

  281.         "vlef   %%v0,%3,0               \n\t"

  282. 	"vlef   %%v0,%3,2               \n\t"

  283.         "vlrepf %%v1,%4                 \n\t"

  284. #endif

  285.         "xgr   %%r1,%%r1                \n\t"

  286.         "srlg  %%r0,%0,2                \n\t"

  287.         "0:                             \n\t"

  288.         "pfd 1,1024(%%r1,%1)            \n\t"

  289.         "pfd 2,1024(%%r1,%2)            \n\t"

  290. 

  291.         "vl   %%v16,0(%%r1,%1)          \n\t"

  292.         "vl   %%v17,16(%%r1,%1)         \n\t"

  293.         "vl   %%v18,0(%%r1,%2)          \n\t"

  294.         "vl   %%v19,16(%%r1,%2)         \n\t"

  295. 	"verllg   %%v20,%%v16,32        \n\t"

  296.         "verllg   %%v21,%%v17,32        \n\t"

  297. 

  298.         "vfmasb %%v22,%%v16,%%v0,%%v18  \n\t"

  299.         "vfmasb %%v23,%%v17,%%v0,%%v19  \n\t"

  300. 

  301.         "vfmasb %%v22,%%v20,%%v1,%%v22  \n\t"

  302.         "vfmasb %%v23,%%v21,%%v1,%%v23  \n\t"

  303. 

  304.         "vst %%v22,0(%%r1,%2)           \n\t"

  305.         "vst %%v23,16(%%r1,%2)          \n\t"

  306.         

  307.         "agfi   %%r1,32                 \n\t"

  308.         "brctg  %%r0,0b                     "

  309.         :

  310.         :"r"(n),"ZR"((const FLOAT (*)[n * 2])src),"ZR"((FLOAT (*)[n * 2])dest),"m"(alpha_r),"m"(alpha_i)

  311.         :"memory","cc","r0","r1","v0","v1","v16","v17","v18","v19","v20","v21","v22","v23"

  312.     );

  313. }

  314. 

  315. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT alpha_r, FLOAT alpha_i)

  316. {

  317. 	BLASLONG i;

  318. 

  319. 	if ( inc_dest != 2 )

  320. 	{

  321. 

  322. 		FLOAT temp_r;

  323. 		FLOAT temp_i;

  324. 		for ( i=0; i<n; i++ )

  325. 		{

  326. #if !defined(XCONJ) 

  327. 			temp_r = alpha_r * src[0] - alpha_i * src[1];

  328. 			temp_i = alpha_r * src[1] + alpha_i * src[0];

  329. #else

  330. 			temp_r =  alpha_r * src[0] + alpha_i * src[1];

  331. 			temp_i = -alpha_r * src[1] + alpha_i * src[0];

  332. #endif

  333. 

  334. 			*dest += temp_r;

  335. 			*(dest+1) += temp_i;

  336. 

  337. 			src+=2;

  338. 			dest += inc_dest;

  339. 		}

  340. 		return;

  341. 	}

  342. 

  343. 	add_y_4(n, src, dest, alpha_r, alpha_i);

  344. }

  345. 

  346. int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r,FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)

  347. {

  348. 	BLASLONG i;

  349. 	FLOAT *a_ptr;

  350. 	FLOAT *x_ptr;

  351. 	FLOAT *y_ptr;

  352. 	FLOAT *ap[4];

  353. 	BLASLONG n1;

  354. 	BLASLONG m1;

  355. 	BLASLONG m2;

  356. 	BLASLONG m3;

  357. 	BLASLONG n2;

  358. 	BLASLONG lda4;

  359. 	FLOAT xbuffer[8],*ybuffer;

  360. 

  361. 	if ( m < 1 ) return(0);

  362. 	if ( n < 1 ) return(0);

  363. 

  364. 	ybuffer = buffer;

  365. 	

  366. 	inc_x *= 2;

  367. 	inc_y *= 2;

  368. 	lda   *= 2;

  369. 	lda4  = 4 * lda;

  370. 

  371. 	n1 = n / 4 ;

  372. 	n2 = n % 4 ;

  373. 	

  374. 	m3 = m % 4;

  375. 	m1 = m - ( m % 4 );

  376. 	m2 = (m % NBMAX) - (m % 4) ;

  377. 	

  378. 	y_ptr = y;

  379. 

  380. 	BLASLONG NB = NBMAX;

  381. 

  382. 	while ( NB == NBMAX )

  383. 	{

  384. 		

  385. 		m1 -= NB;

  386. 		if ( m1 < 0)

  387. 		{

  388. 			if ( m2 == 0 ) break;	

  389. 			NB = m2;

  390. 		}

  391. 		

  392. 		a_ptr = a;

  393. 		ap[0] = a_ptr;

  394. 		ap[1] = a_ptr + lda;

  395. 		ap[2] = ap[1] + lda;

  396. 		ap[3] = ap[2] + lda;

  397. 		x_ptr = x;

  398. 		//zero_y(NB,ybuffer);

  399. 		memset(ybuffer,0,NB*8);

  400. 

  401. 		if ( inc_x == 2 )

  402. 		{

  403. 

  404. 			for( i = 0; i < n1 ; i++)

  405. 			{

  406. 				cgemv_kernel_4x4(NB,ap,x_ptr,ybuffer);

  407. 				ap[0] += lda4;

  408. 				ap[1] += lda4;

  409. 				ap[2] += lda4;

  410. 				ap[3] += lda4;

  411. 				a_ptr += lda4;

  412. 				x_ptr += 8;	

  413. 			}

  414. 

  415. 			if ( n2 & 2 )

  416. 			{

  417. 				cgemv_kernel_4x2(NB,ap,x_ptr,ybuffer);

  418. 				x_ptr += 4;	

  419. 				a_ptr += 2 * lda;

  420. 

  421. 			}

  422. 

  423. 			if ( n2 & 1 )

  424. 			{

  425. 				cgemv_kernel_4x1(NB,a_ptr,x_ptr,ybuffer);

  426. 				/* x_ptr += 2;	

  427. 				a_ptr += lda; */

  428. 

  429. 			}

  430. 		}

  431. 		else

  432. 		{

  433. 

  434. 			for( i = 0; i < n1 ; i++)

  435. 			{

  436. 

  437. 				xbuffer[0] = x_ptr[0];

  438. 				xbuffer[1] = x_ptr[1];

  439. 				x_ptr += inc_x;	

  440. 				xbuffer[2] = x_ptr[0];

  441. 				xbuffer[3] = x_ptr[1];

  442. 				x_ptr += inc_x;	

  443. 				xbuffer[4] = x_ptr[0];

  444. 				xbuffer[5] = x_ptr[1];

  445. 				x_ptr += inc_x;	

  446. 				xbuffer[6] = x_ptr[0];

  447. 				xbuffer[7] = x_ptr[1];

  448. 				x_ptr += inc_x;	

  449. 

  450. 				cgemv_kernel_4x4(NB,ap,xbuffer,ybuffer);

  451. 				ap[0] += lda4;

  452. 				ap[1] += lda4;

  453. 				ap[2] += lda4;

  454. 				ap[3] += lda4;

  455. 				a_ptr += lda4;

  456. 			}

  457. 

  458. 			for( i = 0; i < n2 ; i++)

  459. 			{

  460. 				xbuffer[0] = x_ptr[0];

  461. 				xbuffer[1] = x_ptr[1];

  462. 				x_ptr += inc_x;	

  463. 				cgemv_kernel_4x1(NB,a_ptr,xbuffer,ybuffer);

  464. 				a_ptr += 1 * lda;

  465. 

  466. 			}

  467. 

  468. 		}

  469. 

  470. 		add_y(NB,ybuffer,y_ptr,inc_y,alpha_r,alpha_i);

  471. 		a     += 2 * NB;

  472. 		y_ptr += NB * inc_y;

  473. 	}

  474. 

  475. 	if ( m3 == 0 ) return(0);

  476. 

  477. 	if ( m3 == 1 )

  478. 	{

  479. 		a_ptr = a;

  480. 		x_ptr = x;

  481. 		FLOAT temp_r = 0.0;

  482. 		FLOAT temp_i = 0.0;

  483. 

  484. 		if ( lda == 2 && inc_x == 2 )

  485. 		{

  486. 

  487. 

  488. 			for( i=0 ; i < (n & -2); i+=2 )

  489. 			{

  490. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  491. 				temp_r += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  492. 				temp_i += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  493. 				temp_r += a_ptr[2] * x_ptr[2] - a_ptr[3] * x_ptr[3];

  494. 				temp_i += a_ptr[2] * x_ptr[3] + a_ptr[3] * x_ptr[2];

  495. #else

  496. 				temp_r += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  497. 				temp_i += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  498. 				temp_r += a_ptr[2] * x_ptr[2] + a_ptr[3] * x_ptr[3];

  499. 				temp_i += a_ptr[2] * x_ptr[3] - a_ptr[3] * x_ptr[2];

  500. #endif

  501. 

  502. 				a_ptr += 4;

  503. 				x_ptr += 4;

  504. 			}

  505. 

  506. 

  507. 

  508. 			for( ; i < n; i++ )

  509. 			{

  510. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  511. 				temp_r += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  512. 				temp_i += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  513. #else

  514. 				temp_r += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  515. 				temp_i += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  516. #endif

  517. 

  518. 				a_ptr += 2;

  519. 				x_ptr += 2;

  520. 			}

  521. 

  522. 

  523. 		}

  524. 		else

  525. 		{

  526. 

  527. 			for( i = 0; i < n; i++ )

  528. 			{

  529. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  530. 				temp_r += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  531. 				temp_i += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  532. #else

  533. 				temp_r += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  534. 				temp_i += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  535. #endif

  536. 

  537. 				a_ptr += lda;

  538. 				x_ptr += inc_x;

  539. 			}

  540. 

  541. 		}

  542. #if !defined(XCONJ) 

  543. 		y_ptr[0] += alpha_r * temp_r - alpha_i * temp_i;

  544. 		y_ptr[1] += alpha_r * temp_i + alpha_i * temp_r;

  545. #else

  546. 		y_ptr[0] += alpha_r * temp_r + alpha_i * temp_i;

  547. 		y_ptr[1] -= alpha_r * temp_i - alpha_i * temp_r;

  548. #endif

  549. 		return(0);

  550. 	}

  551. 

  552. 	if ( m3 == 2 )

  553. 	{

  554. 		a_ptr = a;

  555. 		x_ptr = x;

  556. 		FLOAT temp_r0 = 0.0;

  557. 		FLOAT temp_i0 = 0.0;

  558. 		FLOAT temp_r1 = 0.0;

  559. 		FLOAT temp_i1 = 0.0;

  560. 

  561. 		if ( lda == 4 && inc_x == 2 )

  562. 		{

  563. 

  564. 			for( i = 0; i < (n & -2); i+=2 )

  565. 			{

  566. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  567. 

  568. 				temp_r0 += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  569. 				temp_i0 += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  570. 				temp_r1 += a_ptr[2] * x_ptr[0] - a_ptr[3] * x_ptr[1];

  571. 				temp_i1 += a_ptr[2] * x_ptr[1] + a_ptr[3] * x_ptr[0];

  572. 

  573. 				temp_r0 += a_ptr[4] * x_ptr[2] - a_ptr[5] * x_ptr[3];

  574. 				temp_i0 += a_ptr[4] * x_ptr[3] + a_ptr[5] * x_ptr[2];

  575. 				temp_r1 += a_ptr[6] * x_ptr[2] - a_ptr[7] * x_ptr[3];

  576. 				temp_i1 += a_ptr[6] * x_ptr[3] + a_ptr[7] * x_ptr[2];

  577. 

  578. #else

  579. 				temp_r0 += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  580. 				temp_i0 += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  581. 				temp_r1 += a_ptr[2] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  582. 				temp_i1 += a_ptr[2] * x_ptr[1] - a_ptr[3] * x_ptr[0];

  583. 

  584. 				temp_r0 += a_ptr[4] * x_ptr[2] + a_ptr[5] * x_ptr[3];

  585. 				temp_i0 += a_ptr[4] * x_ptr[3] - a_ptr[5] * x_ptr[2];

  586. 				temp_r1 += a_ptr[6] * x_ptr[2] + a_ptr[7] * x_ptr[3];

  587. 				temp_i1 += a_ptr[6] * x_ptr[3] - a_ptr[7] * x_ptr[2];

  588. 

  589. #endif

  590. 

  591. 				a_ptr += 8;

  592. 				x_ptr += 4;

  593. 			}

  594. 

  595. 

  596. 			for( ; i < n; i++ )

  597. 			{

  598. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  599. 				temp_r0 += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  600. 				temp_i0 += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  601. 				temp_r1 += a_ptr[2] * x_ptr[0] - a_ptr[3] * x_ptr[1];

  602. 				temp_i1 += a_ptr[2] * x_ptr[1] + a_ptr[3] * x_ptr[0];

  603. #else

  604. 				temp_r0 += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  605. 				temp_i0 += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  606. 				temp_r1 += a_ptr[2] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  607. 				temp_i1 += a_ptr[2] * x_ptr[1] - a_ptr[3] * x_ptr[0];

  608. #endif

  609. 

  610. 				a_ptr += 4;

  611. 				x_ptr += 2;

  612. 			}

  613. 

  614. 

  615. 		}

  616. 		else

  617. 		{

  618. 

  619. 			for( i=0 ; i < n; i++ )

  620. 			{

  621. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  622. 				temp_r0 += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  623. 				temp_i0 += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  624. 				temp_r1 += a_ptr[2] * x_ptr[0] - a_ptr[3] * x_ptr[1];

  625. 				temp_i1 += a_ptr[2] * x_ptr[1] + a_ptr[3] * x_ptr[0];

  626. #else

  627. 				temp_r0 += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  628. 				temp_i0 += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  629. 				temp_r1 += a_ptr[2] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  630. 				temp_i1 += a_ptr[2] * x_ptr[1] - a_ptr[3] * x_ptr[0];

  631. #endif

  632. 

  633. 				a_ptr += lda;

  634. 				x_ptr += inc_x;

  635. 			}

  636. 

  637. 

  638. 		}

  639. #if !defined(XCONJ) 

  640. 		y_ptr[0] += alpha_r * temp_r0 - alpha_i * temp_i0;

  641. 		y_ptr[1] += alpha_r * temp_i0 + alpha_i * temp_r0;

  642. 		y_ptr    += inc_y;

  643. 		y_ptr[0] += alpha_r * temp_r1 - alpha_i * temp_i1;

  644. 		y_ptr[1] += alpha_r * temp_i1 + alpha_i * temp_r1;

  645. #else

  646. 		y_ptr[0] += alpha_r * temp_r0 + alpha_i * temp_i0;

  647. 		y_ptr[1] -= alpha_r * temp_i0 - alpha_i * temp_r0;

  648. 		y_ptr    += inc_y;

  649. 		y_ptr[0] += alpha_r * temp_r1 + alpha_i * temp_i1;

  650. 		y_ptr[1] -= alpha_r * temp_i1 - alpha_i * temp_r1;

  651. #endif

  652. 		return(0);

  653. 	}

  654. 

  655. 

  656. 	if ( m3 == 3 )

  657. 	{

  658. 		a_ptr = a;

  659. 		x_ptr = x;

  660. 		FLOAT temp_r0 = 0.0;

  661. 		FLOAT temp_i0 = 0.0;

  662. 		FLOAT temp_r1 = 0.0;

  663. 		FLOAT temp_i1 = 0.0;

  664. 		FLOAT temp_r2 = 0.0;

  665. 		FLOAT temp_i2 = 0.0;

  666. 

  667. 		if ( lda == 6 && inc_x == 2 )

  668. 		{

  669. 

  670. 			for( i=0 ; i < n; i++ )

  671. 			{

  672. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  673. 				temp_r0 += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  674. 				temp_i0 += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  675. 				temp_r1 += a_ptr[2] * x_ptr[0] - a_ptr[3] * x_ptr[1];

  676. 				temp_i1 += a_ptr[2] * x_ptr[1] + a_ptr[3] * x_ptr[0];

  677. 				temp_r2 += a_ptr[4] * x_ptr[0] - a_ptr[5] * x_ptr[1];

  678. 				temp_i2 += a_ptr[4] * x_ptr[1] + a_ptr[5] * x_ptr[0];

  679. #else

  680. 				temp_r0 += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  681. 				temp_i0 += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  682. 				temp_r1 += a_ptr[2] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  683. 				temp_i1 += a_ptr[2] * x_ptr[1] - a_ptr[3] * x_ptr[0];

  684. 				temp_r2 += a_ptr[4] * x_ptr[0] + a_ptr[5] * x_ptr[1];

  685. 				temp_i2 += a_ptr[4] * x_ptr[1] - a_ptr[5] * x_ptr[0];

  686. #endif

  687. 

  688. 				a_ptr += 6;

  689. 				x_ptr += 2;

  690. 			}

  691. 

  692. 

  693. 		}

  694. 		else

  695. 		{

  696. 

  697. 			for( i = 0; i < n; i++ )

  698. 			{

  699. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  700. 				temp_r0 += a_ptr[0] * x_ptr[0] - a_ptr[1] * x_ptr[1];

  701. 				temp_i0 += a_ptr[0] * x_ptr[1] + a_ptr[1] * x_ptr[0];

  702. 				temp_r1 += a_ptr[2] * x_ptr[0] - a_ptr[3] * x_ptr[1];

  703. 				temp_i1 += a_ptr[2] * x_ptr[1] + a_ptr[3] * x_ptr[0];

  704. 				temp_r2 += a_ptr[4] * x_ptr[0] - a_ptr[5] * x_ptr[1];

  705. 				temp_i2 += a_ptr[4] * x_ptr[1] + a_ptr[5] * x_ptr[0];

  706. #else

  707. 				temp_r0 += a_ptr[0] * x_ptr[0] + a_ptr[1] * x_ptr[1];

  708. 				temp_i0 += a_ptr[0] * x_ptr[1] - a_ptr[1] * x_ptr[0];

  709. 				temp_r1 += a_ptr[2] * x_ptr[0] + a_ptr[3] * x_ptr[1];

  710. 				temp_i1 += a_ptr[2] * x_ptr[1] - a_ptr[3] * x_ptr[0];

  711. 				temp_r2 += a_ptr[4] * x_ptr[0] + a_ptr[5] * x_ptr[1];

  712. 				temp_i2 += a_ptr[4] * x_ptr[1] - a_ptr[5] * x_ptr[0];

  713. #endif

  714. 

  715. 				a_ptr += lda;

  716. 				x_ptr += inc_x;

  717. 			}

  718. 

  719. 		}

  720. #if !defined(XCONJ) 

  721. 		y_ptr[0] += alpha_r * temp_r0 - alpha_i * temp_i0;

  722. 		y_ptr[1] += alpha_r * temp_i0 + alpha_i * temp_r0;

  723. 		y_ptr    += inc_y;

  724. 		y_ptr[0] += alpha_r * temp_r1 - alpha_i * temp_i1;

  725. 		y_ptr[1] += alpha_r * temp_i1 + alpha_i * temp_r1;

  726. 		y_ptr    += inc_y;

  727. 		y_ptr[0] += alpha_r * temp_r2 - alpha_i * temp_i2;

  728. 		y_ptr[1] += alpha_r * temp_i2 + alpha_i * temp_r2;

  729. #else

  730. 		y_ptr[0] += alpha_r * temp_r0 + alpha_i * temp_i0;

  731. 		y_ptr[1] -= alpha_r * temp_i0 - alpha_i * temp_r0;

  732. 		y_ptr    += inc_y;

  733. 		y_ptr[0] += alpha_r * temp_r1 + alpha_i * temp_i1;

  734. 		y_ptr[1] -= alpha_r * temp_i1 - alpha_i * temp_r1;

  735. 		y_ptr    += inc_y;

  736. 		y_ptr[0] += alpha_r * temp_r2 + alpha_i * temp_i2;

  737. 		y_ptr[1] -= alpha_r * temp_i2 - alpha_i * temp_r2;

  738. #endif

  739. 		return(0);

  740. 	}

  741. 

  742. 	return(0);

  743. }