OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 925d4e1dc6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '925d4e1dc6'
${ noResults }
OpenBLAS/kernel/x86_64/dgemv_t_bulldozer.S

1141 lines
27 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. #include "l2param.h"

  42. 

  43. 

  44. #define A_PRE 256

  45. 

  46. #define VMOVUPS_A1(OFF, ADDR, REGS) vmovups     OFF(ADDR), REGS

  47. #define VMOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS) vmovups        OFF(ADDR, BASE, SCALE), REGS

  48. #define VMOVUPS_XL1(OFF, ADDR, REGS) vmovups    OFF(ADDR), REGS

  49. 

  50. 

  51. 

  52. #if GEMV_UNROLL < 2

  53. #undef  GEMV_UNROLL

  54. #define GEMV_UNROLL 2

  55. #endif

  56. 

  57. #ifndef WINDOWS_ABI

  58. 

  59. #define STACKSIZE	128

  60. 

  61. #define OLD_M	  %rdi

  62. #define OLD_N	  %rsi

  63. #define OLD_A	  %rcx

  64. #define OLD_LDA	  %r8

  65. #define STACK_INCX	 8 + STACKSIZE(%rsp)

  66. #define STACK_Y		16 + STACKSIZE(%rsp)

  67. #define STACK_INCY	24 + STACKSIZE(%rsp)

  68. #define STACK_BUFFER	32 + STACKSIZE(%rsp)

  69. #define MMM	56(%rsp)

  70. #define NN	64(%rsp)

  71. #define AA	72(%rsp)

  72. #define LDAX	80(%rsp)

  73. #else

  74. 

  75. #define STACKSIZE	256

  76. 

  77. #define OLD_M	  %rcx

  78. #define OLD_N	  %rdx

  79. #define OLD_A		 40 + STACKSIZE(%rsp)

  80. #define OLD_LDA		 48 + STACKSIZE(%rsp)

  81. #define OLD_X		 56 + STACKSIZE(%rsp)

  82. #define STACK_INCX	 64 + STACKSIZE(%rsp)

  83. #define STACK_Y		 72 + STACKSIZE(%rsp)

  84. #define STACK_INCY	 80 + STACKSIZE(%rsp)

  85. #define STACK_BUFFER	 88 + STACKSIZE(%rsp)

  86. //Temp variables for M,N,A,LDA

  87. #define MMM	224(%rsp)

  88. #define NN	232(%rsp)

  89. #define AA	240(%rsp)

  90. #define LDAX	248(%rsp)

  91. 

  92. #endif

  93. 

  94. #define LDA	%r8

  95. #define X	%r9

  96. 

  97. #define INCX	%rsi

  98. #define INCY	%rdi

  99. 

  100. #define M	%r10

  101. #define N	%r11

  102. #define A	%r12

  103. #define Y	%r14

  104. #define BUFFER	%r13

  105. 

  106. #define I	%rax

  107. #define A1	%rbx

  108. #define A2	%rcx

  109. #define LDA3	%rdx

  110. #define Y1	%rbp

  111. #define X1	%r15

  112. 

  113. #define MM	M

  114. 

  115. #define ALPHA	%xmm15

  116. 

  117. 	PROLOGUE

  118. 	PROFCODE

  119. 

  120. 	subq	$STACKSIZE, %rsp

  121. 	movq	%rbx,  0(%rsp)

  122. 	movq	%rbp,  8(%rsp)

  123. 	movq	%r12, 16(%rsp)

  124. 	movq	%r13, 24(%rsp)

  125. 	movq	%r14, 32(%rsp)

  126. 	movq	%r15, 40(%rsp)

  127. 

  128. #ifdef WINDOWS_ABI

  129. 	movq	%rdi,    48(%rsp)

  130. 	movq	%rsi,    56(%rsp)

  131. 	vmovups	%xmm6,   64(%rsp)

  132. 	vmovups	%xmm7,   80(%rsp)

  133. 	vmovups	%xmm8,   96(%rsp)

  134. 	vmovups	%xmm9,  112(%rsp)

  135. 	vmovups	%xmm10, 128(%rsp)

  136. 	vmovups	%xmm11, 144(%rsp)

  137. 	vmovups	%xmm12, 160(%rsp)

  138. 	vmovups	%xmm13, 176(%rsp)

  139. 	vmovups	%xmm14, 192(%rsp)

  140. 	vmovups	%xmm15, 208(%rsp)

  141. 

  142. 	movq	OLD_M,	      M

  143. 	movq	OLD_N,        N

  144. 	movq	OLD_A,        A

  145. 	movq	OLD_LDA,      LDA

  146. 	movq	OLD_X,        X

  147. 

  148. 	movq	M,	      MMM

  149. 	movq	N,            NN

  150. 	movq	A,            AA

  151. 	movq	LDA,	      LDAX

  152. 

  153. #else

  154. 	movq	OLD_M,	      MMM

  155. 	movq	OLD_N,        NN

  156. 	movq	OLD_A,        AA

  157. 	movq	OLD_LDA,      LDAX

  158. #endif

  159. #ifdef HAVE_SSE3

  160. #ifndef WINDOWS_ABI

  161. 	movddup	%xmm0, ALPHA

  162. #else

  163. 	movddup	%xmm3, ALPHA

  164. #endif

  165. #else

  166. #ifndef WINDOWS_ABI

  167. 	vmovups	%xmm0, ALPHA

  168. #else

  169. 	vmovups	%xmm3, ALPHA

  170. #endif

  171. 	unpcklpd ALPHA, ALPHA

  172. #endif

  173. 

  174. 

  175. 

  176. .L0x:

  177. 	xorq	M,M

  178. 	addq	$1,M

  179. 	salq	$22,M

  180. 	subq	M,MMM

  181. 	jge .L00

  182. 

  183. 	movq	MMM,%rax

  184. 	addq	M,%rax

  185. 	jle	.L999x

  186. 	movq	%rax,M

  187. 

  188. .L00:

  189. 	movq	LDAX,LDA

  190. 	movq	NN,N

  191. 	movq	AA,A

  192. 	movq	STACK_INCX,   INCX

  193. 	movq	STACK_Y,      Y

  194. 	movq	STACK_INCY,   INCY

  195. 	movq	STACK_BUFFER, BUFFER

  196. 

  197. 	leaq	-1(INCX), %rax

  198. 

  199. 	leaq	(,LDA,  SIZE), LDA

  200. 	leaq	(,INCX, SIZE), INCX

  201. 	leaq	(,INCY, SIZE), INCY

  202. 

  203. 	leaq	(LDA, LDA, 2), LDA3

  204. 

  205. 	subq	$-16 * SIZE, A

  206. 

  207. 	testq	M, M

  208. 	jle	.L999

  209. 	testq	N, N

  210. 	jle	.L999

  211. 

  212. 	movq	BUFFER, X1

  213. 

  214. 	movq	M,  I

  215. 	sarq	$3, I

  216. 	jle	.L05

  217. 	ALIGN_4

  218. 

  219. .L02:

  220. 	vmovsd	(X), %xmm0

  221. 	addq	INCX, X

  222. 	vmovhpd	(X), %xmm0 , %xmm0

  223. 	addq	INCX, X

  224. 

  225. 	vmovsd	(X), %xmm1

  226. 	addq	INCX, X

  227. 	vmovhpd	(X), %xmm1 , %xmm1

  228. 	addq	INCX, X

  229. 

  230. 	vmovsd	(X), %xmm2

  231. 	addq	INCX, X

  232. 	vmovhpd	(X), %xmm2 , %xmm2

  233. 	addq	INCX, X

  234. 

  235. 	vmovsd	(X), %xmm3

  236. 	addq	INCX, X

  237. 	vmovhpd	(X), %xmm3 , %xmm3

  238. 	addq	INCX, X

  239. 

  240. 	vmovups	%xmm0, 0 * SIZE(X1)

  241. 	vmovups	%xmm1, 2 * SIZE(X1)

  242. 	vmovups	%xmm2, 4 * SIZE(X1)

  243. 	vmovups	%xmm3, 6 * SIZE(X1)

  244. 

  245. 	addq	$8 * SIZE, X1

  246. 	decq	I

  247. 	jg	.L02

  248. 	ALIGN_4

  249. 

  250. .L05:

  251. 	movq	M,  I

  252. 	andq	$7, I

  253. 	jle	.L10

  254. 	ALIGN_2

  255. 

  256. .L06:

  257. 	vmovsd	(X), %xmm0

  258. 	addq	INCX, X

  259. 	vmovsd	%xmm0, 0 * SIZE(X1)

  260. 	addq	$SIZE, X1

  261. 	decq	I

  262. 	jg	.L06

  263. 	ALIGN_4

  264. 

  265. .L10:

  266. 	movq	Y, Y1

  267. 

  268. 

  269. #if GEMV_UNROLL >= 8

  270. 	cmpq	$8, N

  271. 	jl	.L20

  272. 	ALIGN_3

  273. 

  274. .L11:

  275. 	subq	$8, N

  276. 

  277. 	leaq	16 * SIZE(BUFFER), X1

  278. 

  279. 	movq	A, A1

  280. 	leaq	(A1, LDA, 4), A2

  281. 	leaq	(A1, LDA, 8), A

  282. 

  283. 	vxorps %xmm0 , 	%xmm0, %xmm0

  284. 	vxorps %xmm1 , 	%xmm1, %xmm1

  285. 	vxorps %xmm2 , 	%xmm2, %xmm2

  286. 	vxorps %xmm3 , 	%xmm3, %xmm3

  287. 	vxorps %xmm4 , 	%xmm4, %xmm4

  288. 	vxorps %xmm5 , 	%xmm5, %xmm5

  289. 	vxorps %xmm6 , 	%xmm6, %xmm6

  290. 	vxorps %xmm7 , 	%xmm7, %xmm7

  291. 

  292. 

  293. 	movq	M,  I

  294. 	sarq	$3,  I

  295. 	jle	.L15

  296. 

  297. 

  298. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  299. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  300. 

  301. 	decq	I

  302. 	jle	.L13

  303. 	ALIGN_4

  304. 

  305. .L12:

  306. 

  307. 	prefetchnta     A_PRE(A1)

  308. 	prefetchnta     A_PRE(A2)

  309. 	vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  310. 	vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  311. 	vfmaddpd %xmm2 , -16 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  312. 	vfmaddpd %xmm3 , -16 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  313. 

  314. 	prefetchnta     A_PRE(A1,LDA,1)

  315. 	prefetchnta     A_PRE(A2,LDA,1)

  316. 	vfmaddpd %xmm4 , -16 * SIZE(A2)           , %xmm12 , %xmm4

  317. 	vfmaddpd %xmm5 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  318. 	vfmaddpd %xmm6 , -16 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  319. 	vfmaddpd %xmm7 , -16 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  320. 

  321. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  322. 

  323. 	prefetchnta     A_PRE(A1,LDA,2)

  324. 	prefetchnta     A_PRE(A2,LDA,2)

  325. 	vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  326. 	vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  327. 	vfmaddpd %xmm2 , -14 * SIZE(A1 , LDA , 2) , %xmm13 , %xmm2

  328. 	vfmaddpd %xmm3 , -14 * SIZE(A1 , LDA3, 1) , %xmm13 , %xmm3

  329. 

  330. 	prefetchnta     A_PRE(A1,LDA3,1)

  331. 	prefetchnta     A_PRE(A2,LDA3,1)

  332. 	vfmaddpd %xmm4 , -14 * SIZE(A2)           , %xmm13 , %xmm4

  333. 	vfmaddpd %xmm5 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm5

  334. 	vfmaddpd %xmm6 , -14 * SIZE(A2 , LDA , 2) , %xmm13 , %xmm6

  335. 	vfmaddpd %xmm7 , -14 * SIZE(A2 , LDA3, 1) , %xmm13 , %xmm7

  336. 

  337. 	prefetchnta     A_PRE(X1)

  338. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  339. 

  340. 	vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  341. 	vfmaddpd %xmm1 , -12 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  342. 	vfmaddpd %xmm2 , -12 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  343. 	vfmaddpd %xmm3 , -12 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  344. 

  345. 	vfmaddpd %xmm4 , -12 * SIZE(A2)           , %xmm12 , %xmm4

  346. 	vfmaddpd %xmm5 , -12 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  347. 	vfmaddpd %xmm6 , -12 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  348. 	vfmaddpd %xmm7 , -12 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  349. 

  350. 	VMOVUPS_XL1(-8 * SIZE, X1, %xmm12)

  351. 

  352. 	vfmaddpd %xmm0 , -10 * SIZE(A1)           , %xmm13 , %xmm0

  353. 	vfmaddpd %xmm1 , -10 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  354. 	vfmaddpd %xmm2 , -10 * SIZE(A1 , LDA , 2) , %xmm13 , %xmm2

  355. 	vfmaddpd %xmm3 , -10 * SIZE(A1 , LDA3, 1) , %xmm13 , %xmm3

  356. 

  357. 	vfmaddpd %xmm4 , -10 * SIZE(A2)           , %xmm13 , %xmm4

  358. 	vfmaddpd %xmm5 , -10 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm5

  359. 	vfmaddpd %xmm6 , -10 * SIZE(A2 , LDA , 2) , %xmm13 , %xmm6

  360. 	vfmaddpd %xmm7 , -10 * SIZE(A2 , LDA3, 1) , %xmm13 , %xmm7

  361. 

  362. 

  363. 	VMOVUPS_XL1(-6 * SIZE, X1, %xmm13)

  364. 

  365. 	addq	$8 * SIZE, A1

  366. 	addq	$8 * SIZE, A2

  367. 	addq	$8 * SIZE, X1

  368. 

  369. 	decq	I

  370. 	jg	.L12

  371. 	ALIGN_4

  372. 

  373. .L13:

  374. 

  375. 	vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  376. 	vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  377. 	vfmaddpd %xmm2 , -16 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  378. 	vfmaddpd %xmm3 , -16 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  379. 

  380. 	vfmaddpd %xmm4 , -16 * SIZE(A2)           , %xmm12 , %xmm4

  381. 	vfmaddpd %xmm5 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  382. 	vfmaddpd %xmm6 , -16 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  383. 	vfmaddpd %xmm7 , -16 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  384. 

  385. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  386. 

  387. 	vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  388. 	vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  389. 	vfmaddpd %xmm2 , -14 * SIZE(A1 , LDA , 2) , %xmm13 , %xmm2

  390. 	vfmaddpd %xmm3 , -14 * SIZE(A1 , LDA3, 1) , %xmm13 , %xmm3

  391. 

  392. 	vfmaddpd %xmm4 , -14 * SIZE(A2)           , %xmm13 , %xmm4

  393. 	vfmaddpd %xmm5 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm5

  394. 	vfmaddpd %xmm6 , -14 * SIZE(A2 , LDA , 2) , %xmm13 , %xmm6

  395. 	vfmaddpd %xmm7 , -14 * SIZE(A2 , LDA3, 1) , %xmm13 , %xmm7

  396. 

  397. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  398. 

  399. 	vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  400. 	vfmaddpd %xmm1 , -12 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  401. 	vfmaddpd %xmm2 , -12 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  402. 	vfmaddpd %xmm3 , -12 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  403. 

  404. 	vfmaddpd %xmm4 , -12 * SIZE(A2)           , %xmm12 , %xmm4

  405. 	vfmaddpd %xmm5 , -12 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  406. 	vfmaddpd %xmm6 , -12 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  407. 	vfmaddpd %xmm7 , -12 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  408. 

  409. 	vfmaddpd %xmm0 , -10 * SIZE(A1)           , %xmm13 , %xmm0

  410. 	vfmaddpd %xmm1 , -10 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  411. 	vfmaddpd %xmm2 , -10 * SIZE(A1 , LDA , 2) , %xmm13 , %xmm2

  412. 	vfmaddpd %xmm3 , -10 * SIZE(A1 , LDA3, 1) , %xmm13 , %xmm3

  413. 

  414. 	vfmaddpd %xmm4 , -10 * SIZE(A2)           , %xmm13 , %xmm4

  415. 	vfmaddpd %xmm5 , -10 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm5

  416. 	vfmaddpd %xmm6 , -10 * SIZE(A2 , LDA , 2) , %xmm13 , %xmm6

  417. 	vfmaddpd %xmm7 , -10 * SIZE(A2 , LDA3, 1) , %xmm13 , %xmm7

  418. 

  419. 

  420. 

  421. 	addq	$8 * SIZE, A1

  422. 	addq	$8 * SIZE, A2

  423. 	addq	$8 * SIZE, X1

  424. 	ALIGN_4

  425. 

  426. .L15:

  427. 	testq	$4, M

  428. 	jle	.L16

  429. 

  430. 

  431. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  432. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  433. 

  434. 

  435. 	vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  436. 	vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  437. 	vfmaddpd %xmm2 , -16 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  438. 	vfmaddpd %xmm3 , -16 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  439. 

  440. 	vfmaddpd %xmm4 , -16 * SIZE(A2)           , %xmm12 , %xmm4

  441. 	vfmaddpd %xmm5 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  442. 	vfmaddpd %xmm6 , -16 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  443. 	vfmaddpd %xmm7 , -16 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  444. 

  445. 	vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  446. 	vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  447. 	vfmaddpd %xmm2 , -14 * SIZE(A1 , LDA , 2) , %xmm13 , %xmm2

  448. 	vfmaddpd %xmm3 , -14 * SIZE(A1 , LDA3, 1) , %xmm13 , %xmm3

  449. 

  450. 	vfmaddpd %xmm4 , -14 * SIZE(A2)           , %xmm13 , %xmm4

  451. 	vfmaddpd %xmm5 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm5

  452. 	vfmaddpd %xmm6 , -14 * SIZE(A2 , LDA , 2) , %xmm13 , %xmm6

  453. 	vfmaddpd %xmm7 , -14 * SIZE(A2 , LDA3, 1) , %xmm13 , %xmm7

  454. 

  455. 

  456. 	addq	$4 * SIZE, A1

  457. 	addq	$4 * SIZE, A2

  458. 	addq	$4 * SIZE, X1

  459. 	ALIGN_4

  460. 

  461. .L16:

  462. 	testq	$2, M

  463. 	jle	.L17

  464. 

  465. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  466. 

  467. 	vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  468. 	vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  469. 	vfmaddpd %xmm2 , -16 * SIZE(A1 , LDA , 2) , %xmm12 , %xmm2

  470. 	vfmaddpd %xmm3 , -16 * SIZE(A1 , LDA3, 1) , %xmm12 , %xmm3

  471. 

  472. 	vfmaddpd %xmm4 , -16 * SIZE(A2)           , %xmm12 , %xmm4

  473. 	vfmaddpd %xmm5 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm5

  474. 	vfmaddpd %xmm6 , -16 * SIZE(A2 , LDA , 2) , %xmm12 , %xmm6

  475. 	vfmaddpd %xmm7 , -16 * SIZE(A2 , LDA3, 1) , %xmm12 , %xmm7

  476. 

  477. 	addq	$2 * SIZE, A1

  478. 	addq	$2 * SIZE, A2

  479. 	addq	$2 * SIZE, X1

  480. 	ALIGN_4

  481. 

  482. .L17:

  483. 	testq	$1, M

  484. 	je	.L18

  485. 

  486. 	vmovsd	-16 * SIZE(X1), %xmm12

  487. 

  488.         vmovsd   -16 * SIZE(A1), %xmm8

  489.         vmovsd   -16 * SIZE(A1, LDA), %xmm9

  490.         vmovsd   -16 * SIZE(A1, LDA, 2), %xmm10

  491.         vmovsd   -16 * SIZE(A1, LDA3), %xmm11

  492. 

  493. 	vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0

  494. 	vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1

  495. 	vfmaddpd %xmm2, %xmm10, %xmm12, %xmm2

  496. 	vfmaddpd %xmm3, %xmm11, %xmm12, %xmm3

  497. 

  498.         vmovsd   -16 * SIZE(A2), %xmm8

  499.         vmovsd   -16 * SIZE(A2, LDA), %xmm9

  500.         vmovsd   -16 * SIZE(A2, LDA, 2), %xmm10

  501.         vmovsd   -16 * SIZE(A2, LDA3), %xmm11

  502. 

  503. 	vfmaddpd %xmm4, %xmm8 , %xmm12, %xmm4

  504. 	vfmaddpd %xmm5, %xmm9 , %xmm12, %xmm5

  505. 	vfmaddpd %xmm6, %xmm10, %xmm12, %xmm6

  506. 	vfmaddpd %xmm7, %xmm11, %xmm12, %xmm7

  507. 

  508. 	ALIGN_4

  509. 

  510. .L18:

  511. 	vhaddpd	%xmm1, %xmm0 , %xmm0

  512. 	vhaddpd	%xmm3, %xmm2 , %xmm2

  513. 	vhaddpd	%xmm5, %xmm4 , %xmm4

  514. 	vhaddpd	%xmm7, %xmm6 , %xmm6

  515. 

  516. 	vmulpd	ALPHA, %xmm0 , %xmm0

  517. 	vmulpd	ALPHA, %xmm2 , %xmm2

  518. 	vmulpd	ALPHA, %xmm4 , %xmm4

  519. 	vmulpd	ALPHA, %xmm6 , %xmm6

  520. 

  521. 	cmpq	$SIZE, INCY

  522. 	jne	.L19

  523. 

  524. 	vaddpd	0 * SIZE(Y),  %xmm0 , %xmm0

  525. 	vaddpd	2 * SIZE(Y),  %xmm2 , %xmm2

  526. 	vaddpd	4 * SIZE(Y),  %xmm4 , %xmm4

  527. 	vaddpd	6 * SIZE(Y),  %xmm6 , %xmm6

  528. 	addq	$8 * SIZE, Y

  529. 

  530. 	vmovups	%xmm0,  0 * SIZE(Y1)

  531. 	vmovups	%xmm2,  2 * SIZE(Y1)

  532. 	vmovups	%xmm4,  4 * SIZE(Y1)

  533. 	vmovups	%xmm6,  6 * SIZE(Y1)

  534. 	addq	$8 * SIZE, Y1

  535. 

  536. 	cmpq	$8, N

  537. 	jge	.L11

  538. 	jmp	.L20

  539. 	ALIGN_4

  540. 

  541. .L19:

  542. 

  543. 	vmovsd	(Y), %xmm8

  544. 	addq	INCY, Y

  545. 	vmovhpd	(Y), %xmm8 , %xmm8

  546. 	addq	INCY, Y

  547. 	vmovsd	(Y), %xmm9

  548. 	addq	INCY, Y

  549. 	vmovhpd	(Y), %xmm9 , %xmm9

  550. 	addq	INCY, Y

  551. 	vmovsd	(Y), %xmm10

  552. 	addq	INCY, Y

  553. 	vmovhpd	(Y), %xmm10 , %xmm10

  554. 	addq	INCY, Y

  555. 	vmovsd	(Y), %xmm11

  556. 	addq	INCY, Y

  557. 	vmovhpd	(Y), %xmm11 , %xmm11

  558. 	addq	INCY, Y

  559. 

  560. 	vaddpd	%xmm8,  %xmm0 , %xmm0

  561. 	vaddpd	%xmm9,  %xmm2 , %xmm2

  562. 	vaddpd	%xmm10, %xmm4 , %xmm4

  563. 	vaddpd	%xmm11, %xmm6 , %xmm6

  564. 

  565. 	vmovlpd	%xmm0, (Y1)

  566. 	addq	INCY, Y1

  567. 	vmovhpd	%xmm0, (Y1)

  568. 	addq	INCY, Y1

  569. 	vmovlpd	%xmm2, (Y1)

  570. 	addq	INCY, Y1

  571. 	vmovhpd	%xmm2, (Y1)

  572. 	addq	INCY, Y1

  573. 	vmovlpd	%xmm4, (Y1)

  574. 	addq	INCY, Y1

  575. 	vmovhpd	%xmm4, (Y1)

  576. 	addq	INCY, Y1

  577. 	vmovlpd	%xmm6, (Y1)

  578. 	addq	INCY, Y1

  579. 	vmovhpd	%xmm6, (Y1)

  580. 	addq	INCY, Y1

  581. 

  582. 	cmpq	$8, N

  583. 	jge	.L11

  584. 	ALIGN_4

  585. 

  586. .L20:

  587. #endif

  588. 

  589. #if GEMV_UNROLL >= 4

  590. 

  591. 	cmpq	$4, N

  592. 	jl	.L30

  593. 

  594. #if GEMV_UNROLL == 4

  595. 	ALIGN_3

  596. 

  597. .L21:

  598. #endif

  599. 	subq	$4, N

  600. 

  601. 	leaq	16 * SIZE(BUFFER), X1

  602. 

  603. 	movq	A, A1

  604. 	leaq	(A1, LDA, 2), A2

  605. 	leaq	(A1, LDA, 4), A

  606. 

  607. 	vxorps %xmm0 , 	%xmm0, %xmm0

  608. 	vxorps %xmm1 , 	%xmm1, %xmm1

  609. 	vxorps %xmm2 , 	%xmm2, %xmm2

  610. 	vxorps %xmm3 , 	%xmm3, %xmm3

  611. 

  612. 

  613. 	movq	M,  I

  614. 	sarq	$3,  I

  615. 	jle	.L25

  616. 

  617. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  618. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  619. 

  620. 	decq	I

  621. 	jle	.L23

  622. 	ALIGN_4

  623. 

  624. .L22:

  625. 

  626. 	prefetchnta     A_PRE(A1)

  627. 	prefetchnta     A_PRE(A2)

  628.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  629.         vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  630.         vfmaddpd %xmm2 , -16 * SIZE(A2)           , %xmm12 , %xmm2

  631.         vfmaddpd %xmm3 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  632. 

  633. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  634. 

  635. 	prefetchnta     A_PRE(A1,LDA,1)

  636. 	prefetchnta     A_PRE(A2,LDA,1)

  637.         vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  638.         vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  639.         vfmaddpd %xmm2 , -14 * SIZE(A2)           , %xmm13 , %xmm2

  640.         vfmaddpd %xmm3 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm3

  641. 

  642. 	prefetchnta     A_PRE(X1)

  643. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  644. 

  645.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  646.         vfmaddpd %xmm1 , -12 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  647.         vfmaddpd %xmm2 , -12 * SIZE(A2)           , %xmm12 , %xmm2

  648.         vfmaddpd %xmm3 , -12 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  649. 

  650. 	VMOVUPS_XL1( -8 * SIZE, X1, %xmm12)

  651. 

  652.         vfmaddpd %xmm0 , -10 * SIZE(A1)           , %xmm13 , %xmm0

  653.         vfmaddpd %xmm1 , -10 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  654.         vfmaddpd %xmm2 , -10 * SIZE(A2)           , %xmm13 , %xmm2

  655.         vfmaddpd %xmm3 , -10 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm3

  656. 

  657. 	VMOVUPS_XL1( -6 * SIZE, X1, %xmm13)

  658. 

  659. 	addq	$8 * SIZE, A1

  660. 	addq	$8 * SIZE, A2

  661. 	addq	$8 * SIZE, X1

  662. 

  663. 	decq	I

  664. 	jg	.L22

  665. 	ALIGN_4

  666. 

  667. .L23:

  668. 

  669.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  670.         vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  671.         vfmaddpd %xmm2 , -16 * SIZE(A2)           , %xmm12 , %xmm2

  672.         vfmaddpd %xmm3 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  673. 

  674. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  675. 

  676.         vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  677.         vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  678.         vfmaddpd %xmm2 , -14 * SIZE(A2)           , %xmm13 , %xmm2

  679.         vfmaddpd %xmm3 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm3

  680. 

  681. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  682. 

  683.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  684.         vfmaddpd %xmm1 , -12 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  685.         vfmaddpd %xmm2 , -12 * SIZE(A2)           , %xmm12 , %xmm2

  686.         vfmaddpd %xmm3 , -12 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  687. 

  688.         vfmaddpd %xmm0 , -10 * SIZE(A1)           , %xmm13 , %xmm0

  689.         vfmaddpd %xmm1 , -10 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  690.         vfmaddpd %xmm2 , -10 * SIZE(A2)           , %xmm13 , %xmm2

  691.         vfmaddpd %xmm3 , -10 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm3

  692. 

  693. 

  694. 	addq	$8 * SIZE, A1

  695. 	addq	$8 * SIZE, A2

  696. 	addq	$8 * SIZE, X1

  697. 	ALIGN_4

  698. 

  699. .L25:

  700. 	testq	$4, M

  701. 	jle	.L26

  702. 

  703. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  704. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  705. 

  706.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  707.         vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  708.         vfmaddpd %xmm2 , -16 * SIZE(A2)           , %xmm12 , %xmm2

  709.         vfmaddpd %xmm3 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  710. 

  711.         vfmaddpd %xmm0 , -14 * SIZE(A1)           , %xmm13 , %xmm0

  712.         vfmaddpd %xmm1 , -14 * SIZE(A1 , LDA , 1) , %xmm13 , %xmm1

  713.         vfmaddpd %xmm2 , -14 * SIZE(A2)           , %xmm13 , %xmm2

  714.         vfmaddpd %xmm3 , -14 * SIZE(A2 , LDA , 1) , %xmm13 , %xmm3

  715. 

  716. 

  717. 	addq	$4 * SIZE, A1

  718. 	addq	$4 * SIZE, A2

  719. 	addq	$4 * SIZE, X1

  720. 	ALIGN_4

  721. 

  722. .L26:

  723. 	testq	$2, M

  724. 	jle	.L27

  725. 

  726. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  727. 

  728.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  729.         vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1

  730.         vfmaddpd %xmm2 , -16 * SIZE(A2)           , %xmm12 , %xmm2

  731.         vfmaddpd %xmm3 , -16 * SIZE(A2 , LDA , 1) , %xmm12 , %xmm3

  732. 

  733. 	addq	$2 * SIZE, A1

  734. 	addq	$2 * SIZE, A2

  735. 	addq	$2 * SIZE, X1

  736. 	ALIGN_4

  737. 

  738. .L27:

  739. 	testq	$1, M

  740. 	je	.L28

  741. 

  742. 	vmovsd	-16 * SIZE(X1), %xmm12

  743. 

  744. 	vmovsd	-16 * SIZE(A1), %xmm8

  745. 	vmovsd	-16 * SIZE(A1, LDA), %xmm9

  746. 	vmovsd	-16 * SIZE(A2), %xmm10

  747. 	vmovsd	-16 * SIZE(A2, LDA), %xmm11

  748. 

  749. 	vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0

  750. 	vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1

  751. 	vfmaddpd %xmm2, %xmm10, %xmm12, %xmm2

  752. 	vfmaddpd %xmm3, %xmm11, %xmm12, %xmm3

  753. 

  754. 	ALIGN_4

  755. 

  756. .L28:

  757. 	vhaddpd	%xmm1, %xmm0 , %xmm0

  758. 	vhaddpd	%xmm3, %xmm2 , %xmm2

  759. 

  760. 	vmulpd	ALPHA, %xmm0 , %xmm0

  761. 	vmulpd	ALPHA, %xmm2 , %xmm2

  762. 

  763. 	cmpq	$SIZE, INCY

  764. 	jne	.L29

  765. 

  766. 	vmovups	 0 * SIZE(Y), %xmm4

  767. 	vmovups  2 * SIZE(Y), %xmm5

  768. 	addq	$4 * SIZE, Y

  769. 

  770. 	vaddpd	%xmm4, %xmm0 , %xmm0

  771. 	vaddpd	%xmm5, %xmm2 , %xmm2

  772. 

  773. 	vmovups	%xmm0,  0 * SIZE(Y1)

  774. 	vmovups	%xmm2,  2 * SIZE(Y1)

  775. 	addq	$4 * SIZE, Y1

  776. 

  777. #if GEMV_UNROLL == 4

  778. 	cmpq	$4, N

  779. 	jge	.L21

  780. #endif

  781. 	jmp	.L30

  782. 	ALIGN_4

  783. 

  784. .L29:

  785. 

  786. 	vmovsd	(Y), %xmm4

  787. 	addq	INCY, Y

  788. 	vmovhpd	(Y), %xmm4 , %xmm4

  789. 	addq	INCY, Y

  790. 	vmovsd	(Y), %xmm5

  791. 	addq	INCY, Y

  792. 	vmovhpd	(Y), %xmm5 , %xmm5

  793. 	addq	INCY, Y

  794. 

  795. 	vaddpd	%xmm4, %xmm0 , %xmm0

  796. 	vaddpd	%xmm5, %xmm2 , %xmm2

  797. 

  798. 	vmovlpd	%xmm0, (Y1)

  799. 	addq	INCY, Y1

  800. 	vmovhpd	%xmm0, (Y1)

  801. 	addq	INCY, Y1

  802. 	vmovlpd	%xmm2, (Y1)

  803. 	addq	INCY, Y1

  804. 	vmovhpd	%xmm2, (Y1)

  805. 	addq	INCY, Y1

  806. 

  807. #if GEMV_UNROLL == 4

  808. 	cmpq	$4, N

  809. 	jge	.L21

  810. #endif

  811. 	ALIGN_4

  812. 

  813. .L30:

  814. #endif

  815. 

  816. #if GEMV_UNROLL >= 2

  817. 

  818. 	cmpq	$2, N

  819. 	jl	.L40

  820. 

  821. #if GEMV_UNROLL == 2

  822. 	ALIGN_3

  823. 

  824. .L31:

  825. #endif

  826. 

  827. 	subq	$2, N

  828. 

  829. 	leaq	16 * SIZE(BUFFER), X1

  830. 

  831. 	movq	A, A1

  832. 	leaq	(A1, LDA), A2

  833. 	leaq	(A1, LDA, 2), A

  834. 

  835.         vxorps %xmm0 ,  %xmm0, %xmm0

  836.         vxorps %xmm1 ,  %xmm1, %xmm1

  837.         vxorps %xmm2 ,  %xmm2, %xmm2

  838.         vxorps %xmm3 ,  %xmm3, %xmm3

  839. 

  840. 

  841. 	movq	M,  I

  842. 	sarq	$3,  I

  843. 	jle	.L35

  844. 

  845. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  846. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  847. 

  848. 	decq	I

  849. 	jle	.L33

  850. 	ALIGN_4

  851. 

  852. .L32:

  853. 

  854.         prefetchnta     A_PRE(A1)

  855.         prefetchnta     A_PRE(A2)

  856.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  857.         vfmaddpd %xmm1 , -16 * SIZE(A2)           , %xmm12 , %xmm1

  858. 

  859.         prefetchnta     A_PRE(X1)

  860. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  861. 

  862.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  863.         vfmaddpd %xmm3 , -14 * SIZE(A2)           , %xmm13 , %xmm3

  864. 

  865. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  866. 

  867.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  868.         vfmaddpd %xmm1 , -12 * SIZE(A2)           , %xmm12 , %xmm1

  869. 

  870. 	VMOVUPS_XL1( -8 * SIZE, X1, %xmm12)

  871. 

  872.         vfmaddpd %xmm2 , -10 * SIZE(A1)           , %xmm13 , %xmm2

  873.         vfmaddpd %xmm3 , -10 * SIZE(A2)           , %xmm13 , %xmm3

  874. 

  875. 	VMOVUPS_XL1( -6 * SIZE, X1, %xmm13)

  876. 

  877. 	addq	$8 * SIZE, A1

  878. 	addq	$8 * SIZE, A2

  879. 	addq	$8 * SIZE, X1

  880. 

  881. 	decq	I

  882. 	jg	.L32

  883. 	ALIGN_4

  884. 

  885. .L33:

  886. 

  887.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  888.         vfmaddpd %xmm1 , -16 * SIZE(A2)           , %xmm12 , %xmm1

  889. 

  890. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  891. 

  892.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  893.         vfmaddpd %xmm3 , -14 * SIZE(A2)           , %xmm13 , %xmm3

  894. 

  895. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  896. 

  897.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  898.         vfmaddpd %xmm1 , -12 * SIZE(A2)           , %xmm12 , %xmm1

  899. 

  900.         vfmaddpd %xmm2 , -10 * SIZE(A1)           , %xmm13 , %xmm2

  901.         vfmaddpd %xmm3 , -10 * SIZE(A2)           , %xmm13 , %xmm3

  902. 

  903. 	addq	$8 * SIZE, A1

  904. 	addq	$8 * SIZE, A2

  905. 	addq	$8 * SIZE, X1

  906. 	ALIGN_4

  907. 

  908. .L35:

  909. 	testq	$4, M

  910. 	jle	.L36

  911. 

  912. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  913. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  914. 

  915.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  916.         vfmaddpd %xmm1 , -16 * SIZE(A2)           , %xmm12 , %xmm1

  917. 

  918.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  919.         vfmaddpd %xmm3 , -14 * SIZE(A2)           , %xmm13 , %xmm3

  920. 

  921. 

  922. 	addq	$4 * SIZE, A1

  923. 	addq	$4 * SIZE, A2

  924. 	addq	$4 * SIZE, X1

  925. 	ALIGN_4

  926. 

  927. .L36:

  928. 	testq	$2, M

  929. 	jle	.L37

  930. 

  931. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  932. 

  933.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  934.         vfmaddpd %xmm1 , -16 * SIZE(A2)           , %xmm12 , %xmm1

  935. 

  936. 	addq	$2 * SIZE, A1

  937. 	addq	$2 * SIZE, A2

  938. 	addq	$2 * SIZE, X1

  939. 	ALIGN_4

  940. 

  941. .L37:

  942. 	testq	$1, M

  943. 	je	.L38

  944. 

  945. 	vmovsd	-16 * SIZE(X1), %xmm12

  946. 

  947. 	vmovsd	-16 * SIZE(A1), %xmm8

  948. 	vmovsd	-16 * SIZE(A2), %xmm9

  949. 

  950.         vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0

  951.         vfmaddpd %xmm1, %xmm9 , %xmm12, %xmm1

  952. 

  953. 	ALIGN_4

  954. 

  955. .L38:

  956. 	vaddpd	%xmm2, %xmm0 , %xmm0

  957. 	vaddpd	%xmm3, %xmm1 , %xmm1

  958. 

  959. 	vhaddpd	%xmm1, %xmm0 , %xmm0

  960. 

  961. 	mulpd	ALPHA, %xmm0

  962. 

  963. 	vmovsd	(Y), %xmm4

  964. 	addq	INCY, Y

  965. 	vmovhpd	(Y), %xmm4 , %xmm4

  966. 	addq	INCY, Y

  967. 

  968. 	vaddpd	%xmm4, %xmm0 , %xmm0

  969. 

  970. 	vmovlpd	%xmm0, (Y1)

  971. 	addq	INCY, Y1

  972. 	vmovhpd	%xmm0, (Y1)

  973. 	addq	INCY, Y1

  974. 

  975. #if GEMV_UNROLL == 2

  976. 	cmpq	$2, N

  977. 	jge	.L31

  978. #endif

  979. 	ALIGN_4

  980. 

  981. .L40:

  982. 	cmpq	$1, N

  983. 	jl	.L999

  984. 

  985. #endif

  986. 

  987. 	leaq	16 * SIZE(BUFFER), X1

  988. 

  989. 	movq	A, A1

  990. 

  991.         vxorps %xmm0 ,  %xmm0, %xmm0

  992.         vxorps %xmm1 ,  %xmm1, %xmm1

  993.         vxorps %xmm2 ,  %xmm2, %xmm2

  994.         vxorps %xmm3 ,  %xmm3, %xmm3

  995. 

  996. 

  997. 

  998. 	movq	M,  I

  999. 	sarq	$3,  I

  1000. 	jle	.L45

  1001. 

  1002. 

  1003. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  1004. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  1005. 

  1006. 	decq	I

  1007. 	jle	.L43

  1008. 	ALIGN_4

  1009. 

  1010. .L42:

  1011. 

  1012. 	prefetchnta     A_PRE(A1)

  1013.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  1014.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  1015. 

  1016. 	prefetchnta     A_PRE(X1)

  1017. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  1018. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  1019. 

  1020.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  1021.         vfmaddpd %xmm2 , -10 * SIZE(A1)           , %xmm13 , %xmm2

  1022. 

  1023. 	VMOVUPS_XL1( -8 * SIZE, X1, %xmm12)

  1024. 	VMOVUPS_XL1( -6 * SIZE, X1, %xmm13)

  1025. 

  1026. 	addq	$8 * SIZE, A1

  1027. 	addq	$8 * SIZE, X1

  1028. 

  1029. 	decq	I

  1030. 	jg	.L42

  1031. 	ALIGN_4

  1032. 

  1033. .L43:

  1034. 

  1035.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  1036.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  1037. 

  1038. 	VMOVUPS_XL1(-12 * SIZE, X1, %xmm12)

  1039. 	VMOVUPS_XL1(-10 * SIZE, X1, %xmm13)

  1040. 

  1041.         vfmaddpd %xmm0 , -12 * SIZE(A1)           , %xmm12 , %xmm0

  1042.         vfmaddpd %xmm2 , -10 * SIZE(A1)           , %xmm13 , %xmm2

  1043. 

  1044. 

  1045. 

  1046. 	addq	$8 * SIZE, A1

  1047. 	addq	$8 * SIZE, X1

  1048. 	ALIGN_4

  1049. 

  1050. .L45:

  1051. 	testq	$4, M

  1052. 	jle	.L46

  1053. 

  1054. 

  1055. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  1056. 	VMOVUPS_XL1(-14 * SIZE, X1, %xmm13)

  1057. 

  1058.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  1059.         vfmaddpd %xmm2 , -14 * SIZE(A1)           , %xmm13 , %xmm2

  1060. 

  1061. 	addq	$4 * SIZE, A1

  1062. 	addq	$4 * SIZE, X1

  1063. 	ALIGN_4

  1064. 

  1065. .L46:

  1066. 	testq	$2, M

  1067. 	jle	.L47

  1068. 

  1069. 	VMOVUPS_XL1(-16 * SIZE, X1, %xmm12)

  1070. 

  1071.         vfmaddpd %xmm0 , -16 * SIZE(A1)           , %xmm12 , %xmm0

  1072. 

  1073. 	addq	$2 * SIZE, A1

  1074. 	addq	$2 * SIZE, X1

  1075. 	ALIGN_4

  1076. 

  1077. .L47:

  1078. 	testq	$1, M

  1079. 	je	.L48

  1080. 

  1081. 	vmovsd	-16 * SIZE(X1), %xmm12

  1082. 	vmovsd	-16 * SIZE(A1), %xmm8

  1083. 

  1084. 	vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0

  1085. 	ALIGN_4

  1086. 

  1087. .L48:

  1088. 	vaddpd	%xmm2, %xmm0 , %xmm0

  1089. 	vaddpd	%xmm3, %xmm1 , %xmm1

  1090. 

  1091. 	vaddpd	%xmm1, %xmm0 , %xmm0

  1092. 

  1093. 	vhaddpd	%xmm1, %xmm0 , %xmm0

  1094. 

  1095. 	vmulsd	ALPHA, %xmm0 , %xmm0

  1096. 

  1097. 	vmovsd	(Y), %xmm4

  1098. 	addq	INCY, Y

  1099. 

  1100. 	vaddsd	%xmm4, %xmm0 , %xmm0

  1101. 

  1102. 	vmovlpd	%xmm0, (Y1)

  1103. 	addq	INCY, Y1

  1104. 

  1105. 	ALIGN_4

  1106. 

  1107. .L999:

  1108. 	leaq	(, M, SIZE), %rax

  1109. 	addq %rax,AA

  1110. 	jmp .L0x;

  1111. 	ALIGN_4

  1112. 

  1113. .L999x:

  1114. 	movq	  0(%rsp), %rbx

  1115. 	movq	  8(%rsp), %rbp

  1116. 	movq	 16(%rsp), %r12

  1117. 	movq	 24(%rsp), %r13

  1118. 	movq	 32(%rsp), %r14

  1119. 	movq	 40(%rsp), %r15

  1120. 

  1121. #ifdef WINDOWS_ABI

  1122. 	movq	 48(%rsp), %rdi

  1123. 	movq	 56(%rsp), %rsi

  1124. 	vmovups	 64(%rsp), %xmm6

  1125. 	vmovups	 80(%rsp), %xmm7

  1126. 	vmovups	 96(%rsp), %xmm8

  1127. 	vmovups	112(%rsp), %xmm9

  1128. 	vmovups	128(%rsp), %xmm10

  1129. 	vmovups	144(%rsp), %xmm11

  1130. 	vmovups	160(%rsp), %xmm12

  1131. 	vmovups	176(%rsp), %xmm13

  1132. 	vmovups	192(%rsp), %xmm14

  1133. 	vmovups	208(%rsp), %xmm15

  1134. #endif

  1135. 

  1136. 	addq	$STACKSIZE, %rsp

  1137. 	ret

  1138. 	ALIGN_4

  1139. 

  1140. 	EPILOGUE