OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86_64/zgemv_n.S

2753 lines
54 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. #include "l2param.h"

  42. 

  43. #ifndef WINDOWS_ABI

  44. 

  45. #define STACKSIZE	128

  46. 

  47. #define OLD_INCX	 8 + STACKSIZE(%rsp)

  48. #define OLD_Y		16 + STACKSIZE(%rsp)

  49. #define OLD_INCY	24 + STACKSIZE(%rsp)

  50. #define OLD_BUFFER	32 + STACKSIZE(%rsp)

  51. #define ALPHA_R		48	      (%rsp)

  52. #define ALPHA_I		56	      (%rsp)

  53. 

  54. #define MMM	64(%rsp)

  55. #define NN	72(%rsp)

  56. #define AA	80(%rsp)

  57. #define XX	88(%rsp)

  58. #define LDAX	96(%rsp)

  59. #define ALPHAR	104(%rsp)

  60. #define	ALPHAI	112(%rsp)

  61. 

  62. #define M	  %rdi

  63. #define N	  %rsi

  64. #define A	  %rcx

  65. #define LDA	  %r8

  66. #define X	  %r9

  67. #define INCX	  %rdx

  68. #define Y	  %rbp

  69. #define INCY	  %r10

  70. 

  71. #else

  72. 

  73. #define STACKSIZE	304

  74. 

  75. #define OLD_ALPHA_I	 40 + STACKSIZE(%rsp)

  76. #define OLD_A		 48 + STACKSIZE(%rsp)

  77. #define OLD_LDA		 56 + STACKSIZE(%rsp)

  78. #define OLD_X		 64 + STACKSIZE(%rsp)

  79. #define OLD_INCX	 72 + STACKSIZE(%rsp)

  80. #define OLD_Y		 80 + STACKSIZE(%rsp)

  81. #define OLD_INCY	 88 + STACKSIZE(%rsp)

  82. #define OLD_BUFFER	 96 + STACKSIZE(%rsp)

  83. #define ALPHA_R		224	       (%rsp)

  84. #define ALPHA_I		232	       (%rsp)

  85. 

  86. #define MMM	240(%rsp)

  87. #define NN	248(%rsp)

  88. #define AA	256(%rsp)

  89. #define	XX	264(%rsp)

  90. #define LDAX	272(%rsp)

  91. #define ALPHAR	280(%rsp)

  92. #define ALPHAI	288(%rsp)

  93. 

  94. #define M	  %rcx

  95. #define N	  %rdx

  96. #define A	  %r8

  97. #define LDA	  %r9

  98. #define X	  %rdi

  99. #define INCX	  %rsi

  100. #define Y	  %rbp

  101. #define INCY	  %r10

  102. 

  103. #endif

  104. 

  105. #define I	%rax

  106. #define A1	%r12

  107. #define A2	%r13

  108. 

  109. #define Y1	%r14

  110. #define BUFFER	%r15

  111. 

  112. #define J	%r11

  113. 

  114. #undef SUBPD

  115. 

  116. #if (!defined(CONJ) && !defined(XCONJ)) || (defined(CONJ) && defined(XCONJ))

  117. #define SUBPD	   subpd

  118. #else

  119. #define SUBPD	   addpd

  120. #endif

  121. 

  122. 	PROLOGUE

  123. 	PROFCODE

  124. 

  125. 	subq	$STACKSIZE, %rsp

  126. 	movq	%rbx,  0(%rsp)

  127. 	movq	%rbp,  8(%rsp)

  128. 	movq	%r12, 16(%rsp)

  129. 	movq	%r13, 24(%rsp)

  130. 	movq	%r14, 32(%rsp)

  131. 	movq	%r15, 40(%rsp)

  132. 

  133. #ifdef WINDOWS_ABI

  134. 	movq	%rdi,    48(%rsp)

  135. 	movq	%rsi,    56(%rsp)

  136. 	movups	%xmm6,   64(%rsp)

  137. 	movups	%xmm7,   80(%rsp)

  138. 	movups	%xmm8,   96(%rsp)

  139. 	movups	%xmm9,  112(%rsp)

  140. 	movups	%xmm10, 128(%rsp)

  141. 	movups	%xmm11, 144(%rsp)

  142. 	movups	%xmm12, 160(%rsp)

  143. 	movups	%xmm13, 176(%rsp)

  144. 	movups	%xmm14, 192(%rsp)

  145. 	movups	%xmm15, 208(%rsp)

  146. 

  147. 	movq	OLD_A,     A

  148. 	movq	OLD_LDA,   LDA

  149. 	movq	OLD_X,     X

  150. 

  151. 	movapd	%xmm3,       %xmm0

  152. 	movsd	OLD_ALPHA_I, %xmm1

  153. #endif

  154. 

  155. 	movq	A, AA

  156. 	movq	N, NN

  157. 	movq	M, MMM

  158. 	movq	LDA, LDAX

  159. 	movq	X, XX

  160. 	movq	OLD_Y,     Y

  161. 	movsd	%xmm0,ALPHAR

  162. 	movsd	%xmm1,ALPHAI

  163. 

  164. .L0t:

  165. 	xorq	I,I

  166. 	addq	$1,I

  167. 	salq	$18,I

  168. 	subq	I,MMM

  169. 	movq	I,M

  170. 	movsd	ALPHAR,%xmm0

  171. 	movsd	ALPHAI,%xmm1

  172. 	jge	.L00t

  173. 

  174. 	movq	MMM,M

  175. 	addq	I,M

  176. 	jle	.L999x

  177. 

  178. .L00t:

  179. 	movq	AA, A

  180. 	movq	NN, N

  181. 	movq	LDAX, LDA

  182. 	movq	XX, X

  183. 

  184. 	movq	OLD_INCX,  INCX

  185. #	movq	OLD_Y,     Y

  186. 	movq	OLD_INCY,  INCY

  187. 	movq	OLD_BUFFER, BUFFER

  188. 

  189. 	salq	$ZBASE_SHIFT,   LDA

  190. 	salq	$ZBASE_SHIFT,   INCX

  191. 	salq	$ZBASE_SHIFT,   INCY

  192. 

  193. 	movlpd	%xmm0, ALPHA_R

  194. 	movlpd	%xmm1, ALPHA_I

  195. 

  196. 	subq	$-16 * SIZE, A

  197. 

  198. 	testq	M, M

  199. 	jle	.L999

  200. 	testq	N, N

  201. 	jle	.L999

  202. 	ALIGN_3

  203. 

  204. 	movq	BUFFER, Y1

  205. 

  206. 	pxor	%xmm4, %xmm4

  207. 

  208. 	movq	M,   %rax

  209. 	addq	$8,  %rax

  210. 	sarq	$3,  %rax

  211. 	ALIGN_3

  212. 

  213. .L01:

  214. 	movapd	%xmm4,  0 * SIZE(Y1)

  215. 	movapd	%xmm4,  2 * SIZE(Y1)

  216. 	movapd	%xmm4,  4 * SIZE(Y1)

  217. 	movapd	%xmm4,  6 * SIZE(Y1)

  218. 	movapd	%xmm4,  8 * SIZE(Y1)

  219. 	movapd	%xmm4, 10 * SIZE(Y1)

  220. 	movapd	%xmm4, 12 * SIZE(Y1)

  221. 	movapd	%xmm4, 14 * SIZE(Y1)

  222. 

  223. 	subq	$-16 * SIZE, Y1

  224. 	decq	%rax

  225. 	jg	.L01

  226. 	ALIGN_3

  227. 

  228. .L10:

  229. #ifdef ALIGNED_ACCESS

  230. 	testq	$SIZE, A

  231. 	jne	.L100

  232. #endif

  233. 

  234. #if GEMV_UNROLL >= 4

  235. 

  236. 	cmpq	$4, N

  237. 	jl	.L20

  238. 	ALIGN_3

  239. 

  240. .L11:

  241. 	subq	$4, N

  242. 

  243. 	leaq	16 * SIZE(BUFFER), Y1

  244. 	movq	A,  A1

  245. 	leaq	(A,  LDA, 2), A2

  246. 	leaq	(A,  LDA, 4), A

  247. 

  248. 	movsd	0 * SIZE(X), %xmm8

  249. 	movhpd	1 * SIZE(X), %xmm8

  250. 	addq	INCX, X

  251. 	movsd	0 * SIZE(X), %xmm10

  252. 	movhpd	1 * SIZE(X), %xmm10

  253. 	addq	INCX, X

  254. 	movsd	0 * SIZE(X), %xmm12

  255. 	movhpd	1 * SIZE(X), %xmm12

  256. 	addq	INCX, X

  257. 	movsd	0 * SIZE(X), %xmm14

  258. 	movhpd	1 * SIZE(X), %xmm14

  259. 	addq	INCX, X

  260. 

  261. 	pcmpeqb	%xmm5, %xmm5

  262. 	psllq	$63,   %xmm5

  263. 	shufps	$0xc0, %xmm5, %xmm5

  264. 

  265. 	pshufd	$0x4e, %xmm8,  %xmm9

  266. 	pshufd	$0x4e, %xmm10, %xmm11

  267. 	pshufd	$0x4e, %xmm12, %xmm13

  268. 	pshufd	$0x4e, %xmm14, %xmm15

  269. 

  270. #ifdef HAVE_SSE3

  271. 	movddup	ALPHA_R, %xmm6

  272. 	movddup	ALPHA_I, %xmm7

  273. #else

  274. 	movsd	ALPHA_R, %xmm6

  275. 	unpcklpd %xmm6, %xmm6

  276. 	movsd	ALPHA_I, %xmm7

  277. 	unpcklpd %xmm7, %xmm7

  278. #endif

  279. 

  280. 	xorpd	 %xmm5, %xmm9

  281. 	xorpd	 %xmm5, %xmm11

  282. 	xorpd	 %xmm5, %xmm13

  283. 	xorpd	 %xmm5, %xmm15

  284. 

  285. 	mulpd	 %xmm6, %xmm8

  286. 	mulpd	 %xmm7, %xmm9

  287. 	mulpd	 %xmm6, %xmm10

  288. 	mulpd	 %xmm7, %xmm11

  289. 

  290. 	mulpd	 %xmm6, %xmm12

  291. 	mulpd	 %xmm7, %xmm13

  292. 	mulpd	 %xmm6, %xmm14

  293. 	mulpd	 %xmm7, %xmm15

  294. 

  295. #ifndef XCONJ

  296. 	subpd	 %xmm9,  %xmm8

  297. 	subpd	 %xmm11, %xmm10

  298. 	subpd	 %xmm13, %xmm12

  299. 	subpd	 %xmm15, %xmm14

  300. #else

  301. 	addpd	 %xmm9,  %xmm8

  302. 	addpd	 %xmm11, %xmm10

  303. 	addpd	 %xmm13, %xmm12

  304. 	addpd	 %xmm15, %xmm14

  305. #endif

  306. 

  307. 	pshufd	 $0xee, %xmm8,  %xmm9

  308. 	pshufd	 $0x44, %xmm8,  %xmm8

  309. 

  310. 	pshufd	 $0xee, %xmm10, %xmm11

  311. 	pshufd	 $0x44, %xmm10, %xmm10

  312. 

  313. 	pshufd	 $0xee, %xmm12, %xmm13

  314. 	pshufd	 $0x44, %xmm12, %xmm12

  315. 

  316. 	pshufd	 $0xee, %xmm14, %xmm15

  317. 	pshufd	 $0x44, %xmm14, %xmm14

  318. 

  319. #ifndef CONJ

  320. 	xorpd	 %xmm5, %xmm9

  321. 	xorpd	 %xmm5, %xmm11

  322. 	xorpd	 %xmm5, %xmm13

  323. 	xorpd	 %xmm5, %xmm15

  324. #else

  325. 	xorpd	 %xmm5, %xmm8

  326. 	xorpd	 %xmm5, %xmm10

  327. 	xorpd	 %xmm5, %xmm12

  328. 	xorpd	 %xmm5, %xmm14

  329. #endif

  330. 

  331. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  332. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  333. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  334. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  335. 	ALIGN_3

  336. 

  337. 	movq	M,   I

  338. 	sarq	$2,  I

  339. 	jle	.L15

  340. 

  341. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  342. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  343. 

  344. 	decq	 I

  345. 	jle	 .L14

  346. 	ALIGN_3

  347. 

  348. .L13:

  349. #ifdef PREFETCH

  350. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1)

  351. #endif

  352. 

  353. 	pshufd	 $0x4e, %xmm4, %xmm5

  354. 	mulpd	 %xmm8,  %xmm4

  355. 	addpd	 %xmm4,  %xmm0

  356. 	MOVUPS_A1(-12 * SIZE, A1, %xmm4)

  357. 	pshufd	 $0x4e, %xmm6, %xmm7

  358. 	mulpd	 %xmm8,  %xmm6

  359. 	addpd	 %xmm6,  %xmm1

  360. 	MOVUPS_A1(-10 * SIZE, A1, %xmm6)

  361. 

  362. 	mulpd	 %xmm9,  %xmm5

  363. 	SUBPD	 %xmm5,  %xmm0

  364. 	mulpd	 %xmm9,  %xmm7

  365. 	SUBPD	 %xmm7,  %xmm1

  366. 

  367. 	pshufd	 $0x4e, %xmm4, %xmm5

  368. 	mulpd	 %xmm8,  %xmm4

  369. 	addpd	 %xmm4,  %xmm2

  370. 	MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm4)

  371. 	pshufd	 $0x4e, %xmm6, %xmm7

  372. 	mulpd	 %xmm8,  %xmm6

  373. 	addpd	 %xmm6,  %xmm3

  374. 	MOVUPS_A2(-14 * SIZE, A1, LDA, 1, %xmm6)

  375. 

  376. 	mulpd	 %xmm9,  %xmm5

  377. 	SUBPD	 %xmm5,  %xmm2

  378. 	mulpd	 %xmm9,  %xmm7

  379. 	SUBPD	 %xmm7,  %xmm3

  380. 

  381. #ifdef PREFETCH

  382. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1, LDA)

  383. #endif

  384. 

  385. 	pshufd	 $0x4e, %xmm4, %xmm5

  386. 	mulpd	 %xmm10, %xmm4

  387. 	addpd	 %xmm4,  %xmm0

  388. 	MOVUPS_A2(-12 * SIZE, A1, LDA, 1, %xmm4)

  389. 	pshufd	 $0x4e, %xmm6, %xmm7

  390. 	mulpd	 %xmm10, %xmm6

  391. 	addpd	 %xmm6,  %xmm1

  392. 	MOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm6)

  393. 

  394. 	mulpd	 %xmm11, %xmm5

  395. 	SUBPD	 %xmm5,  %xmm0

  396. 	mulpd	 %xmm11, %xmm7

  397. 	SUBPD	 %xmm7,  %xmm1

  398. 

  399. 	pshufd	 $0x4e, %xmm4, %xmm5

  400. 	mulpd	 %xmm10, %xmm4

  401. 	addpd	 %xmm4,  %xmm2

  402. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  403. 	pshufd	 $0x4e, %xmm6, %xmm7

  404. 	mulpd	 %xmm10, %xmm6

  405. 	addpd	 %xmm6,  %xmm3

  406. 	MOVUPS_A1(-14 * SIZE, A2, %xmm6)

  407. 

  408. 	mulpd	 %xmm11, %xmm5

  409. 	SUBPD	 %xmm5,  %xmm2

  410. 	mulpd	 %xmm11, %xmm7

  411. 	SUBPD	 %xmm7,  %xmm3

  412. 

  413. #ifdef PREFETCH

  414. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2)

  415. #endif

  416. 

  417. 	pshufd	 $0x4e, %xmm4, %xmm5

  418. 	mulpd	 %xmm12, %xmm4

  419. 	addpd	 %xmm4,  %xmm0

  420. 	MOVUPS_A1(-12 * SIZE, A2, %xmm4)

  421. 	pshufd	 $0x4e, %xmm6, %xmm7

  422. 	mulpd	 %xmm12, %xmm6

  423. 	addpd	 %xmm6,  %xmm1

  424. 	MOVUPS_A1(-10 * SIZE, A2, %xmm6)

  425. 

  426. 	mulpd	 %xmm13, %xmm5

  427. 	SUBPD	 %xmm5,  %xmm0

  428. 	mulpd	 %xmm13, %xmm7

  429. 	SUBPD	 %xmm7,  %xmm1

  430. 

  431. 	pshufd	 $0x4e, %xmm4, %xmm5

  432. 	mulpd	 %xmm12, %xmm4

  433. 	addpd	 %xmm4,  %xmm2

  434. 	MOVUPS_A2(-16 * SIZE, A2, LDA, 1, %xmm4)

  435. 	pshufd	 $0x4e, %xmm6, %xmm7

  436. 	mulpd	 %xmm12, %xmm6

  437. 	addpd	 %xmm6,  %xmm3

  438. 	MOVUPS_A2(-14 * SIZE, A2, LDA, 1, %xmm6)

  439. 

  440. 	mulpd	 %xmm13, %xmm5

  441. 	SUBPD	 %xmm5,  %xmm2

  442. 	mulpd	 %xmm13, %xmm7

  443. 	SUBPD	 %xmm7,  %xmm3

  444. 

  445. #ifdef PREFETCH

  446. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2, LDA)

  447. #endif

  448. 

  449. 	pshufd	 $0x4e, %xmm4, %xmm5

  450. 	mulpd	 %xmm14, %xmm4

  451. 	addpd	 %xmm4,  %xmm0

  452. 	MOVUPS_A2(-12 * SIZE, A2, LDA, 1, %xmm4)

  453. 	pshufd	 $0x4e, %xmm6, %xmm7

  454. 	mulpd	 %xmm14, %xmm6

  455. 	addpd	 %xmm6,  %xmm1

  456. 	MOVUPS_A2(-10 * SIZE, A2, LDA, 1, %xmm6)

  457. 

  458. 	mulpd	 %xmm15, %xmm5

  459. 	SUBPD	 %xmm5,  %xmm0

  460. 	mulpd	 %xmm15, %xmm7

  461. 	SUBPD	 %xmm7,  %xmm1

  462. 

  463. 	pshufd	 $0x4e, %xmm4, %xmm5

  464. 	mulpd	 %xmm14, %xmm4

  465. 	addpd	 %xmm4,  %xmm2

  466. 	MOVUPS_A1( -8 * SIZE, A1, %xmm4)

  467. 	pshufd	 $0x4e, %xmm6, %xmm7

  468. 	mulpd	 %xmm14, %xmm6

  469. 	addpd	 %xmm6,  %xmm3

  470. 	MOVUPS_A1( -6 * SIZE, A1, %xmm6)

  471. 

  472. 	mulpd	 %xmm15, %xmm5

  473. 	SUBPD	 %xmm5,  %xmm2

  474. 	mulpd	 %xmm15, %xmm7

  475. 	SUBPD	 %xmm7,  %xmm3

  476. 

  477. #ifdef PREFETCHW

  478. 	PREFETCHW	(PREFETCHSIZE) - 128 + PREOFFSET(Y1)

  479. #endif

  480. 

  481. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  482. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  483. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  484. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  485. 

  486. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  487. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  488. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  489. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  490. 

  491. 	subq	 $-8 * SIZE, A1

  492. 	subq	 $-8 * SIZE, A2

  493. 	subq	 $-8 * SIZE, Y1

  494. 

  495. 	subq	 $1, I

  496. 	BRANCH

  497. 	jg	.L13

  498. 	ALIGN_3

  499. 

  500. .L14:

  501. 	pshufd	 $0x4e, %xmm4, %xmm5

  502. 	mulpd	 %xmm8,  %xmm4

  503. 	addpd	 %xmm4,  %xmm0

  504. 	MOVUPS_A1(-12 * SIZE, A1, %xmm4)

  505. 	pshufd	 $0x4e, %xmm6, %xmm7

  506. 	mulpd	 %xmm8,  %xmm6

  507. 	addpd	 %xmm6,  %xmm1

  508. 	MOVUPS_A1(-10 * SIZE, A1, %xmm6)

  509. 

  510. 	mulpd	 %xmm9,  %xmm5

  511. 	SUBPD	 %xmm5,  %xmm0

  512. 	mulpd	 %xmm9,  %xmm7

  513. 	SUBPD	 %xmm7,  %xmm1

  514. 

  515. 	pshufd	 $0x4e, %xmm4, %xmm5

  516. 	mulpd	 %xmm8,  %xmm4

  517. 	addpd	 %xmm4,  %xmm2

  518. 	MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm4)

  519. 	pshufd	 $0x4e, %xmm6, %xmm7

  520. 	mulpd	 %xmm8,  %xmm6

  521. 	addpd	 %xmm6,  %xmm3

  522. 	MOVUPS_A2(-14 * SIZE, A1, LDA, 1, %xmm6)

  523. 

  524. 	mulpd	 %xmm9,  %xmm5

  525. 	SUBPD	 %xmm5,  %xmm2

  526. 	mulpd	 %xmm9,  %xmm7

  527. 	SUBPD	 %xmm7,  %xmm3

  528. 

  529. 	pshufd	 $0x4e, %xmm4, %xmm5

  530. 	mulpd	 %xmm10, %xmm4

  531. 	addpd	 %xmm4,  %xmm0

  532. 	MOVUPS_A2(-12 * SIZE, A1, LDA, 1, %xmm4)

  533. 	pshufd	 $0x4e, %xmm6, %xmm7

  534. 	mulpd	 %xmm10, %xmm6

  535. 	addpd	 %xmm6,  %xmm1

  536. 	MOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm6)

  537. 

  538. 	mulpd	 %xmm11, %xmm5

  539. 	SUBPD	 %xmm5,  %xmm0

  540. 	mulpd	 %xmm11, %xmm7

  541. 	SUBPD	 %xmm7,  %xmm1

  542. 

  543. 	pshufd	 $0x4e, %xmm4, %xmm5

  544. 	mulpd	 %xmm10, %xmm4

  545. 	addpd	 %xmm4,  %xmm2

  546. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  547. 	pshufd	 $0x4e, %xmm6, %xmm7

  548. 	mulpd	 %xmm10, %xmm6

  549. 	addpd	 %xmm6,  %xmm3

  550. 	MOVUPS_A1(-14 * SIZE, A2, %xmm6)

  551. 

  552. 	mulpd	 %xmm11, %xmm5

  553. 	SUBPD	 %xmm5,  %xmm2

  554. 	mulpd	 %xmm11, %xmm7

  555. 	SUBPD	 %xmm7,  %xmm3

  556. 

  557. 	pshufd	 $0x4e, %xmm4, %xmm5

  558. 	mulpd	 %xmm12, %xmm4

  559. 	addpd	 %xmm4,  %xmm0

  560. 	MOVUPS_A1(-12 * SIZE, A2, %xmm4)

  561. 	pshufd	 $0x4e, %xmm6, %xmm7

  562. 	mulpd	 %xmm12, %xmm6

  563. 	addpd	 %xmm6,  %xmm1

  564. 	MOVUPS_A1(-10 * SIZE, A2, %xmm6)

  565. 

  566. 	mulpd	 %xmm13, %xmm5

  567. 	SUBPD	 %xmm5,  %xmm0

  568. 	mulpd	 %xmm13, %xmm7

  569. 	SUBPD	 %xmm7,  %xmm1

  570. 

  571. 	pshufd	 $0x4e, %xmm4, %xmm5

  572. 	mulpd	 %xmm12, %xmm4

  573. 	addpd	 %xmm4,  %xmm2

  574. 	MOVUPS_A2(-16 * SIZE, A2, LDA, 1, %xmm4)

  575. 	pshufd	 $0x4e, %xmm6, %xmm7

  576. 	mulpd	 %xmm12, %xmm6

  577. 	addpd	 %xmm6,  %xmm3

  578. 	MOVUPS_A2(-14 * SIZE, A2, LDA, 1, %xmm6)

  579. 

  580. 	mulpd	 %xmm13, %xmm5

  581. 	SUBPD	 %xmm5,  %xmm2

  582. 	mulpd	 %xmm13, %xmm7

  583. 	SUBPD	 %xmm7,  %xmm3

  584. 

  585. 	pshufd	 $0x4e, %xmm4, %xmm5

  586. 	mulpd	 %xmm14, %xmm4

  587. 	addpd	 %xmm4,  %xmm0

  588. 	MOVUPS_A2(-12 * SIZE, A2, LDA, 1, %xmm4)

  589. 	pshufd	 $0x4e, %xmm6, %xmm7

  590. 	mulpd	 %xmm14, %xmm6

  591. 	addpd	 %xmm6,  %xmm1

  592. 	MOVUPS_A2(-10 * SIZE, A2, LDA, 1, %xmm6)

  593. 

  594. 	mulpd	 %xmm15, %xmm5

  595. 	SUBPD	 %xmm5,  %xmm0

  596. 	mulpd	 %xmm15, %xmm7

  597. 	SUBPD	 %xmm7,  %xmm1

  598. 

  599. 	pshufd	 $0x4e, %xmm4, %xmm5

  600. 	mulpd	 %xmm14, %xmm4

  601. 	addpd	 %xmm4,  %xmm2

  602. 	pshufd	 $0x4e, %xmm6, %xmm7

  603. 	mulpd	 %xmm14, %xmm6

  604. 	addpd	 %xmm6,  %xmm3

  605. 

  606. 	mulpd	 %xmm15, %xmm5

  607. 	SUBPD	 %xmm5,  %xmm2

  608. 	mulpd	 %xmm15, %xmm7

  609. 	SUBPD	 %xmm7,  %xmm3

  610. 

  611. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  612. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  613. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  614. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  615. 

  616. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  617. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  618. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  619. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  620. 

  621. 	subq	 $-8 * SIZE, A1

  622. 	subq	 $-8 * SIZE, A2

  623. 	subq	 $-8 * SIZE, Y1

  624. 	ALIGN_3

  625. 

  626. .L15:

  627. 	testq	$2, M

  628. 	je	.L17

  629. 

  630. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  631. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  632. 

  633. 	pshufd	 $0x4e, %xmm4, %xmm5

  634. 	mulpd	 %xmm8,  %xmm4

  635. 	addpd	 %xmm4,  %xmm0

  636. 	MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm4)

  637. 	pshufd	 $0x4e, %xmm6, %xmm7

  638. 	mulpd	 %xmm8,  %xmm6

  639. 	addpd	 %xmm6,  %xmm1

  640. 	MOVUPS_A2(-14 * SIZE, A1, LDA, 1, %xmm6)

  641. 

  642. 	mulpd	 %xmm9,  %xmm5

  643. 	SUBPD	 %xmm5,  %xmm0

  644. 	mulpd	 %xmm9,  %xmm7

  645. 	SUBPD	 %xmm7,  %xmm1

  646. 

  647. 	pshufd	 $0x4e, %xmm4, %xmm5

  648. 	mulpd	 %xmm10, %xmm4

  649. 	addpd	 %xmm4,  %xmm0

  650. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  651. 	pshufd	 $0x4e, %xmm6, %xmm7

  652. 	mulpd	 %xmm10, %xmm6

  653. 	addpd	 %xmm6,  %xmm1

  654. 	MOVUPS_A1(-14 * SIZE, A2, %xmm6)

  655. 

  656. 	mulpd	 %xmm11, %xmm5

  657. 	SUBPD	 %xmm5,  %xmm0

  658. 	mulpd	 %xmm11, %xmm7

  659. 	SUBPD	 %xmm7,  %xmm1

  660. 

  661. 	pshufd	 $0x4e, %xmm4, %xmm5

  662. 	mulpd	 %xmm12, %xmm4

  663. 	addpd	 %xmm4,  %xmm0

  664. 	MOVUPS_A2(-16 * SIZE, A2, LDA, 1, %xmm4)

  665. 	pshufd	 $0x4e, %xmm6, %xmm7

  666. 	mulpd	 %xmm12, %xmm6

  667. 	addpd	 %xmm6,  %xmm1

  668. 	MOVUPS_A2(-14 * SIZE, A2, LDA, 1, %xmm6)

  669. 

  670. 	mulpd	 %xmm13, %xmm5

  671. 	SUBPD	 %xmm5,  %xmm0

  672. 	mulpd	 %xmm13, %xmm7

  673. 	SUBPD	 %xmm7,  %xmm1

  674. 

  675. 	pshufd	 $0x4e, %xmm4, %xmm5

  676. 	mulpd	 %xmm14, %xmm4

  677. 	addpd	 %xmm4,  %xmm0

  678. 	mulpd	 %xmm15, %xmm5

  679. 	SUBPD	 %xmm5,  %xmm0

  680. 

  681. 	pshufd	 $0x4e, %xmm6, %xmm7

  682. 	mulpd	 %xmm14, %xmm6

  683. 	addpd	 %xmm6,  %xmm1

  684. 	mulpd	 %xmm15, %xmm7

  685. 	SUBPD	 %xmm7,  %xmm1

  686. 

  687. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  688. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  689. 	movapd	 %xmm2, %xmm0

  690. 

  691. 	addq	 $4 * SIZE, A1

  692. 	addq	 $4 * SIZE, A2

  693. 	addq	 $4 * SIZE, Y1

  694. 	ALIGN_3

  695. 

  696. .L17:

  697. 	testq	$1, M

  698. 	je	.L19

  699. 

  700. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  701. 	MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm6)

  702. 

  703. 	pshufd	 $0x4e, %xmm4, %xmm5

  704. 	mulpd	 %xmm8,  %xmm4

  705. 	addpd	 %xmm4,  %xmm0

  706. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  707. 	mulpd	 %xmm9,  %xmm5

  708. 	SUBPD	 %xmm5,  %xmm0

  709. 

  710. 	pshufd	 $0x4e, %xmm6, %xmm7

  711. 	mulpd	 %xmm10, %xmm6

  712. 	addpd	 %xmm6,  %xmm0

  713. 	MOVUPS_A2(-16 * SIZE, A2, LDA, 1, %xmm6)

  714. 	mulpd	 %xmm11, %xmm7

  715. 	SUBPD	 %xmm7,  %xmm0

  716. 

  717. 	pshufd	 $0x4e, %xmm4, %xmm5

  718. 	mulpd	 %xmm12, %xmm4

  719. 	addpd	 %xmm4,  %xmm0

  720. 	mulpd	 %xmm13, %xmm5

  721. 	SUBPD	 %xmm5,  %xmm0

  722. 

  723. 	pshufd	 $0x4e, %xmm6, %xmm7

  724. 	mulpd	 %xmm14, %xmm6

  725. 	addpd	 %xmm6,  %xmm0

  726. 	mulpd	 %xmm15, %xmm7

  727. 	SUBPD	 %xmm7,  %xmm0

  728. 

  729. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  730. 	ALIGN_3

  731. 

  732. .L19:

  733. 	cmpq	$4, N

  734. 	jge	.L11

  735. 	ALIGN_3

  736. 

  737. .L20:

  738. #endif

  739. 

  740. #if GEMV_UNROLL >= 2

  741. 

  742. 	cmpq	$2, N

  743. 	jl	.L30

  744. 

  745. #if GEMV_UNROLL == 2

  746. 	ALIGN_3

  747. 

  748. .L21:

  749. #endif

  750. 

  751. 	subq	$2, N

  752. 

  753. 	leaq	16 * SIZE(BUFFER), Y1

  754. 	movq	A,  A1

  755. 	leaq	(A,  LDA, 1), A2

  756. 	leaq	(A,  LDA, 2), A

  757. 

  758. 	movsd	0 * SIZE(X), %xmm12

  759. 	movhpd	1 * SIZE(X), %xmm12

  760. 	addq	INCX, X

  761. 	movsd	0 * SIZE(X), %xmm14

  762. 	movhpd	1 * SIZE(X), %xmm14

  763. 	addq	INCX, X

  764. 

  765. 	pcmpeqb	%xmm11, %xmm11

  766. 	psllq	$63,    %xmm11

  767. 	shufps	$0xc0, %xmm11, %xmm11

  768. 

  769. 	pshufd	$0x4e, %xmm12, %xmm13

  770. 	pshufd	$0x4e, %xmm14, %xmm15

  771. 

  772. #ifdef HAVE_SSE3

  773. 	movddup	ALPHA_R, %xmm8

  774. 	movddup	ALPHA_I, %xmm9

  775. #else

  776. 	movsd	ALPHA_R, %xmm8

  777. 	unpcklpd %xmm8, %xmm8

  778. 	movsd	ALPHA_I, %xmm9

  779. 	unpcklpd %xmm9, %xmm9

  780. #endif

  781. 

  782. 	xorpd	 %xmm11, %xmm13

  783. 	xorpd	 %xmm11, %xmm15

  784. 

  785. 	mulpd	 %xmm8, %xmm12

  786. 	mulpd	 %xmm9, %xmm13

  787. 	mulpd	 %xmm8, %xmm14

  788. 	mulpd	 %xmm9, %xmm15

  789. 

  790. #ifndef XCONJ

  791. 	subpd	 %xmm13, %xmm12

  792. 	subpd	 %xmm15, %xmm14

  793. #else

  794. 	addpd	 %xmm13, %xmm12

  795. 	addpd	 %xmm15, %xmm14

  796. #endif

  797. 

  798. 	pshufd	 $0xee, %xmm12, %xmm13

  799. 	pshufd	 $0x44, %xmm12, %xmm12

  800. 

  801. 	pshufd	 $0xee, %xmm14, %xmm15

  802. 	pshufd	 $0x44, %xmm14, %xmm14

  803. 

  804. #ifndef CONJ

  805. 	xorpd	 %xmm11, %xmm13

  806. 	xorpd	 %xmm11, %xmm15

  807. #else

  808. 	xorpd	 %xmm11, %xmm12

  809. 	xorpd	 %xmm11, %xmm14

  810. #endif

  811. 

  812. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  813. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  814. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  815. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  816. 	ALIGN_3

  817. 

  818. 	movq	M,   I

  819. 	sarq	$2,  I

  820. 	jle	.L25

  821. 

  822. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  823. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  824. 	MOVUPS_A1(-12 * SIZE, A1, %xmm8)

  825. 	MOVUPS_A1(-10 * SIZE, A1, %xmm10)

  826. 

  827. 	decq	 I

  828. 	jle	 .L24

  829. 	ALIGN_3

  830. 

  831. .L23:

  832. #ifdef PREFETCH

  833. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2)

  834. #endif

  835. 

  836. 	pshufd	 $0x4e, %xmm4,  %xmm5

  837. 	mulpd	 %xmm12, %xmm4

  838. 	addpd	 %xmm4,  %xmm0

  839. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  840. 	pshufd	 $0x4e, %xmm6,  %xmm7

  841. 	mulpd	 %xmm12, %xmm6

  842. 	addpd	 %xmm6,  %xmm1

  843. 	MOVUPS_A1(-14 * SIZE, A2, %xmm6)

  844. 

  845. 	pshufd	 $0x4e, %xmm8,  %xmm9

  846. 	mulpd	 %xmm12, %xmm8

  847. 	addpd	 %xmm8,  %xmm2

  848. 	MOVUPS_A1(-12 * SIZE, A2, %xmm8)

  849. 	pshufd	 $0x4e, %xmm10, %xmm11

  850. 	mulpd	 %xmm12, %xmm10

  851. 	addpd	 %xmm10, %xmm3

  852. 	MOVUPS_A1(-10 * SIZE, A2, %xmm10)

  853. 

  854. 	mulpd	 %xmm13, %xmm5

  855. 	SUBPD	 %xmm5,  %xmm0

  856. 	mulpd	 %xmm13, %xmm7

  857. 	SUBPD	 %xmm7,  %xmm1

  858. 

  859. 	mulpd	 %xmm13, %xmm9

  860. 	SUBPD	 %xmm9,  %xmm2

  861. 	mulpd	 %xmm13, %xmm11

  862. 	SUBPD	 %xmm11, %xmm3

  863. 

  864. #ifdef PREFETCH

  865. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1)

  866. #endif

  867. 

  868. 	pshufd	 $0x4e, %xmm4,  %xmm5

  869. 	mulpd	 %xmm14, %xmm4

  870. 	addpd	 %xmm4,  %xmm0

  871. 	MOVUPS_A1( -8 * SIZE, A1, %xmm4)

  872. 	pshufd	 $0x4e, %xmm6,  %xmm7

  873. 	mulpd	 %xmm14, %xmm6

  874. 	addpd	 %xmm6,  %xmm1

  875. 	MOVUPS_A1( -6 * SIZE, A1, %xmm6)

  876. 

  877. 	pshufd	 $0x4e, %xmm8,  %xmm9

  878. 	mulpd	 %xmm14, %xmm8

  879. 	addpd	 %xmm8,  %xmm2

  880. 	MOVUPS_A1( -4 * SIZE, A1, %xmm8)

  881. 	pshufd	 $0x4e, %xmm10, %xmm11

  882. 	mulpd	 %xmm14, %xmm10

  883. 	addpd	 %xmm10, %xmm3

  884. 	MOVUPS_A1( -2 * SIZE, A1, %xmm10)

  885. 

  886. 	mulpd	 %xmm15, %xmm5

  887. 	SUBPD	 %xmm5,  %xmm0

  888. 	mulpd	 %xmm15, %xmm7

  889. 	SUBPD	 %xmm7,  %xmm1

  890. 

  891. 	mulpd	 %xmm15, %xmm9

  892. 	SUBPD	 %xmm9,  %xmm2

  893. 	mulpd	 %xmm15, %xmm11

  894. 	SUBPD	 %xmm11, %xmm3

  895. 

  896. #ifdef PREFETCHW

  897. 	PREFETCHW	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(Y1)

  898. #endif

  899. 

  900. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  901. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  902. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  903. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  904. 

  905. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  906. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  907. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  908. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  909. 

  910. 	subq	 $-8 * SIZE, A1

  911. 	subq	 $-8 * SIZE, A2

  912. 	subq	 $-8 * SIZE, Y1

  913. 

  914. 	subq	 $1, I

  915. 	BRANCH

  916. 	jg	.L23

  917. 	ALIGN_3

  918. 

  919. .L24:

  920. 	pshufd	 $0x4e, %xmm4,  %xmm5

  921. 	mulpd	 %xmm12, %xmm4

  922. 	addpd	 %xmm4,  %xmm0

  923. 	MOVUPS_A1(-16 * SIZE, A2, %xmm4)

  924. 	pshufd	 $0x4e, %xmm6,  %xmm7

  925. 	mulpd	 %xmm12, %xmm6

  926. 	addpd	 %xmm6,  %xmm1

  927. 	MOVUPS_A1(-14 * SIZE, A2, %xmm6)

  928. 

  929. 	pshufd	 $0x4e, %xmm8,  %xmm9

  930. 	mulpd	 %xmm12, %xmm8

  931. 	addpd	 %xmm8,  %xmm2

  932. 	MOVUPS_A1(-12 * SIZE, A2, %xmm8)

  933. 	pshufd	 $0x4e, %xmm10, %xmm11

  934. 	mulpd	 %xmm12, %xmm10

  935. 	addpd	 %xmm10, %xmm3

  936. 	MOVUPS_A1(-10 * SIZE, A2, %xmm10)

  937. 

  938. 	mulpd	 %xmm13, %xmm5

  939. 	SUBPD	 %xmm5,  %xmm0

  940. 	mulpd	 %xmm13, %xmm7

  941. 	SUBPD	 %xmm7,  %xmm1

  942. 

  943. 	mulpd	 %xmm13, %xmm9

  944. 	SUBPD	 %xmm9,  %xmm2

  945. 	mulpd	 %xmm13, %xmm11

  946. 	SUBPD	 %xmm11, %xmm3

  947. 

  948. 	pshufd	 $0x4e, %xmm4,  %xmm5

  949. 	mulpd	 %xmm14, %xmm4

  950. 	addpd	 %xmm4,  %xmm0

  951. 	pshufd	 $0x4e, %xmm6,  %xmm7

  952. 	mulpd	 %xmm14, %xmm6

  953. 	addpd	 %xmm6,  %xmm1

  954. 

  955. 	pshufd	 $0x4e, %xmm8,  %xmm9

  956. 	mulpd	 %xmm14, %xmm8

  957. 	addpd	 %xmm8,  %xmm2

  958. 	pshufd	 $0x4e, %xmm10, %xmm11

  959. 	mulpd	 %xmm14, %xmm10

  960. 	addpd	 %xmm10, %xmm3

  961. 

  962. 	mulpd	 %xmm15, %xmm5

  963. 	SUBPD	 %xmm5,  %xmm0

  964. 	mulpd	 %xmm15, %xmm7

  965. 	SUBPD	 %xmm7,  %xmm1

  966. 

  967. 	mulpd	 %xmm15, %xmm9

  968. 	SUBPD	 %xmm9,  %xmm2

  969. 	mulpd	 %xmm15, %xmm11

  970. 	SUBPD	 %xmm11, %xmm3

  971. 

  972. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  973. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  974. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  975. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  976. 

  977. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  978. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  979. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  980. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  981. 

  982. 	subq	 $-8 * SIZE, A1

  983. 	subq	 $-8 * SIZE, A2

  984. 	subq	 $-8 * SIZE, Y1

  985. 	ALIGN_3

  986. 

  987. .L25:

  988. 	testq	$2, M

  989. 	je	.L27

  990. 

  991. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  992. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  993. 	MOVUPS_A1(-16 * SIZE, A2, %xmm8)

  994. 	MOVUPS_A1(-14 * SIZE, A2, %xmm10)

  995. 

  996. 	pshufd	 $0x4e, %xmm4,  %xmm5

  997. 	mulpd	 %xmm12, %xmm4

  998. 	addpd	 %xmm4,  %xmm0

  999. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1000. 	mulpd	 %xmm12, %xmm6

  1001. 	addpd	 %xmm6,  %xmm1

  1002. 

  1003. 	mulpd	 %xmm13, %xmm5

  1004. 	SUBPD	 %xmm5,  %xmm0

  1005. 	mulpd	 %xmm13, %xmm7

  1006. 	SUBPD	 %xmm7,  %xmm1

  1007. 

  1008. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1009. 	mulpd	 %xmm14, %xmm8

  1010. 	addpd	 %xmm8,  %xmm0

  1011. 	pshufd	 $0x4e, %xmm10, %xmm11

  1012. 	mulpd	 %xmm14, %xmm10

  1013. 	addpd	 %xmm10, %xmm1

  1014. 

  1015. 	mulpd	 %xmm15, %xmm9

  1016. 	SUBPD	 %xmm9,  %xmm0

  1017. 	mulpd	 %xmm15, %xmm11

  1018. 	SUBPD	 %xmm11, %xmm1

  1019. 

  1020. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1021. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1022. 	movapd	 %xmm2, %xmm0

  1023. 

  1024. 	addq	 $4 * SIZE, A1

  1025. 	addq	 $4 * SIZE, A2

  1026. 	addq	 $4 * SIZE, Y1

  1027. 	ALIGN_3

  1028. 

  1029. .L27:

  1030. 	testq	$1, M

  1031. #if GEMV_UNROLL == 2

  1032. 	je	.L29

  1033. #else

  1034. 	je	.L30

  1035. #endif

  1036. 

  1037. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  1038. 	MOVUPS_A1(-16 * SIZE, A2, %xmm6)

  1039. 

  1040. 	pshufd	 $0x4e, %xmm4, %xmm5

  1041. 	mulpd	 %xmm12, %xmm4

  1042. 	addpd	 %xmm4,  %xmm0

  1043. 	mulpd	 %xmm13, %xmm5

  1044. 	SUBPD	 %xmm5,  %xmm0

  1045. 

  1046. 	pshufd	 $0x4e, %xmm6, %xmm7

  1047. 	mulpd	 %xmm14, %xmm6

  1048. 	addpd	 %xmm6,  %xmm0

  1049. 	mulpd	 %xmm15, %xmm7

  1050. 	SUBPD	 %xmm7,  %xmm0

  1051. 

  1052. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1053. 

  1054. #if GEMV_UNROLL == 2

  1055. 	ALIGN_3

  1056. 

  1057. .L29:

  1058. 	cmpq	$2, N

  1059. 	jge	.L21

  1060. #endif

  1061. 	ALIGN_3

  1062. 

  1063. .L30:

  1064. #endif

  1065. 

  1066. 	cmpq	$1, N

  1067. 	jl	.L980

  1068. 

  1069. #if GEMV_UNROLL == 1

  1070. .L31:

  1071. 	decq	N

  1072. #endif

  1073. 

  1074. 	leaq	16 * SIZE(BUFFER), Y1

  1075. 	movq	A,  A1

  1076. #if GEMV_UNROLL == 1

  1077. 	addq	LDA, A

  1078. #endif

  1079. 

  1080. 	movsd	0 * SIZE(X), %xmm12

  1081. 	movhpd	1 * SIZE(X), %xmm12

  1082. 	addq	INCX, X

  1083. 

  1084. 	pcmpeqb	%xmm11, %xmm11

  1085. 	psllq	$63,    %xmm11

  1086. 	shufps	$0xc0, %xmm11, %xmm11

  1087. 

  1088. 	pshufd	$0x4e, %xmm12, %xmm13

  1089. 

  1090. #ifdef HAVE_SSE3

  1091. 	movddup	ALPHA_R, %xmm8

  1092. 	movddup	ALPHA_I, %xmm9

  1093. #else

  1094. 	movsd	ALPHA_R, %xmm8

  1095. 	unpcklpd %xmm8, %xmm8

  1096. 	movsd	ALPHA_I, %xmm9

  1097. 	unpcklpd %xmm9, %xmm9

  1098. #endif

  1099. 

  1100. 	xorpd	 %xmm11, %xmm13

  1101. 

  1102. 	mulpd	 %xmm8, %xmm12

  1103. 	mulpd	 %xmm9, %xmm13

  1104. 

  1105. #ifndef XCONJ

  1106. 	subpd	 %xmm13, %xmm12

  1107. #else

  1108. 	addpd	 %xmm13, %xmm12

  1109. #endif

  1110. 

  1111. 	pshufd	 $0xee, %xmm12, %xmm13

  1112. 	pshufd	 $0x44, %xmm12, %xmm12

  1113. 

  1114. #ifndef CONJ

  1115. 	xorpd	 %xmm11, %xmm13

  1116. #else

  1117. 	xorpd	 %xmm11, %xmm12

  1118. #endif

  1119. 

  1120. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  1121. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  1122. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  1123. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  1124. 

  1125. 	movq	M,   I

  1126. 	sarq	$2,  I

  1127. 	jle	.L35

  1128. 

  1129. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  1130. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  1131. 	MOVUPS_A1(-12 * SIZE, A1, %xmm8)

  1132. 	MOVUPS_A1(-10 * SIZE, A1, %xmm10)

  1133. 

  1134. 	decq	 I

  1135. 	jle	 .L34

  1136. 	ALIGN_3

  1137. 

  1138. .L33:

  1139. #ifdef PREFETCH

  1140. 	PREFETCH	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1)

  1141. #endif

  1142. 

  1143. 	pshufd	 $0x4e, %xmm4,  %xmm5

  1144. 	mulpd	 %xmm12, %xmm4

  1145. 	addpd	 %xmm4,  %xmm0

  1146. 	MOVUPS_A1( -8 * SIZE, A1, %xmm4)

  1147. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1148. 	mulpd	 %xmm12, %xmm6

  1149. 	addpd	 %xmm6,  %xmm1

  1150. 	MOVUPS_A1( -6 * SIZE, A1, %xmm6)

  1151. 

  1152. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1153. 	mulpd	 %xmm12, %xmm8

  1154. 	addpd	 %xmm8,  %xmm2

  1155. 	MOVUPS_A1( -4 * SIZE, A1, %xmm8)

  1156. 	pshufd	 $0x4e, %xmm10, %xmm11

  1157. 	mulpd	 %xmm12, %xmm10

  1158. 	addpd	 %xmm10, %xmm3

  1159. 	MOVUPS_A1( -2 * SIZE, A1, %xmm10)

  1160. 

  1161. 	mulpd	 %xmm13, %xmm5

  1162. 	SUBPD	 %xmm5,  %xmm0

  1163. 	mulpd	 %xmm13, %xmm7

  1164. 	SUBPD	 %xmm7,  %xmm1

  1165. 

  1166. 	mulpd	 %xmm13, %xmm9

  1167. 	SUBPD	 %xmm9,  %xmm2

  1168. 	mulpd	 %xmm13, %xmm11

  1169. 	SUBPD	 %xmm11, %xmm3

  1170. 

  1171. #ifdef PREFETCHW

  1172. 	PREFETCHW	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(Y1)

  1173. #endif

  1174. 

  1175. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1176. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1177. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1178. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1179. 

  1180. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1181. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1182. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1183. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1184. 

  1185. 	subq	 $-8 * SIZE, A1

  1186. 	subq	 $-8 * SIZE, Y1

  1187. 

  1188. 	subq	 $1, I

  1189. 	BRANCH

  1190. 	jg	.L33

  1191. 	ALIGN_3

  1192. 

  1193. .L34:

  1194. 	pshufd	 $0x4e, %xmm4,  %xmm5

  1195. 	mulpd	 %xmm12, %xmm4

  1196. 	addpd	 %xmm4,  %xmm0

  1197. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1198. 	mulpd	 %xmm12, %xmm6

  1199. 	addpd	 %xmm6,  %xmm1

  1200. 

  1201. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1202. 	mulpd	 %xmm12, %xmm8

  1203. 	addpd	 %xmm8,  %xmm2

  1204. 	pshufd	 $0x4e, %xmm10, %xmm11

  1205. 	mulpd	 %xmm12, %xmm10

  1206. 	addpd	 %xmm10, %xmm3

  1207. 

  1208. 	mulpd	 %xmm13, %xmm5

  1209. 	SUBPD	 %xmm5,  %xmm0

  1210. 	mulpd	 %xmm13, %xmm7

  1211. 	SUBPD	 %xmm7,  %xmm1

  1212. 

  1213. 	mulpd	 %xmm13, %xmm9

  1214. 	SUBPD	 %xmm9,  %xmm2

  1215. 	mulpd	 %xmm13, %xmm11

  1216. 	SUBPD	 %xmm11, %xmm3

  1217. 

  1218. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1219. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1220. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1221. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1222. 

  1223. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1224. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1225. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1226. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1227. 

  1228. 	subq	 $-8 * SIZE, A1

  1229. 	subq	 $-8 * SIZE, Y1

  1230. 	ALIGN_3

  1231. 

  1232. .L35:

  1233. 	testq	$2, M

  1234. 	je	.L37

  1235. 

  1236. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  1237. 	MOVUPS_A1(-14 * SIZE, A1, %xmm6)

  1238. 

  1239. 	pshufd	 $0x4e, %xmm4,  %xmm5

  1240. 	mulpd	 %xmm12, %xmm4

  1241. 	addpd	 %xmm4,  %xmm0

  1242. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1243. 	mulpd	 %xmm12, %xmm6

  1244. 	addpd	 %xmm6,  %xmm1

  1245. 

  1246. 	mulpd	 %xmm13, %xmm5

  1247. 	SUBPD	 %xmm5,  %xmm0

  1248. 	mulpd	 %xmm13, %xmm7

  1249. 	SUBPD	 %xmm7,  %xmm1

  1250. 

  1251. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1252. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1253. 	movapd	 %xmm2, %xmm0

  1254. 

  1255. 	addq	 $4 * SIZE, A1

  1256. 	addq	 $4 * SIZE, Y1

  1257. 	ALIGN_3

  1258. 

  1259. .L37:

  1260. 	testq	$1, M

  1261. #if GEMV_UNROLL == 1

  1262. 	je	.L39

  1263. #else

  1264. 	je	.L980

  1265. #endif

  1266. 

  1267. 	MOVUPS_A1(-16 * SIZE, A1, %xmm4)

  1268. 

  1269. 	pshufd	 $0x4e, %xmm4, %xmm5

  1270. 	mulpd	 %xmm12, %xmm4

  1271. 	addpd	 %xmm4,  %xmm0

  1272. 	mulpd	 %xmm13, %xmm5

  1273. 	SUBPD	 %xmm5,  %xmm0

  1274. 

  1275. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1276. 

  1277. #if GEMV_UNROLL == 1

  1278. 	ALIGN_3

  1279. .L39:

  1280. 	cmpq	$1, N

  1281. 	jge	.L31

  1282. #endif

  1283. 

  1284. #ifdef ALIGNED_ACCESS

  1285. 

  1286. 	jmp	.L980

  1287. 	ALIGN_3

  1288. 

  1289. .L100:

  1290. #if GEMV_UNROLL >= 4

  1291. 

  1292. 	cmpq	$4, N

  1293. 	jl	.L110

  1294. 	ALIGN_3

  1295. 

  1296. .L101:

  1297. 	subq	$4, N

  1298. 

  1299. 	leaq	16 * SIZE(BUFFER), Y1

  1300. 	movq	A,  A1

  1301. 	leaq	(A,  LDA, 2), A2

  1302. 	leaq	(A,  LDA, 4), A

  1303. 

  1304. 	movsd	0 * SIZE(X), %xmm8

  1305. 	movhpd	1 * SIZE(X), %xmm8

  1306. 	addq	INCX, X

  1307. 	movsd	0 * SIZE(X), %xmm10

  1308. 	movhpd	1 * SIZE(X), %xmm10

  1309. 	addq	INCX, X

  1310. 	movsd	0 * SIZE(X), %xmm12

  1311. 	movhpd	1 * SIZE(X), %xmm12

  1312. 	addq	INCX, X

  1313. 	movsd	0 * SIZE(X), %xmm14

  1314. 	movhpd	1 * SIZE(X), %xmm14

  1315. 	addq	INCX, X

  1316. 

  1317. 	pcmpeqb	%xmm5, %xmm5

  1318. 	psllq	$63,   %xmm5

  1319. 	shufps	$0xc0, %xmm5, %xmm5

  1320. 

  1321. 	pshufd	$0x4e, %xmm8,  %xmm9

  1322. 	pshufd	$0x4e, %xmm10, %xmm11

  1323. 	pshufd	$0x4e, %xmm12, %xmm13

  1324. 	pshufd	$0x4e, %xmm14, %xmm15

  1325. 

  1326. #ifdef HAVE_SSE3

  1327. 	movddup	ALPHA_R, %xmm6

  1328. 	movddup	ALPHA_I, %xmm7

  1329. #else

  1330. 	movsd	ALPHA_R, %xmm6

  1331. 	unpcklpd %xmm6, %xmm6

  1332. 	movsd	ALPHA_I, %xmm7

  1333. 	unpcklpd %xmm7, %xmm7

  1334. #endif

  1335. 

  1336. 	xorpd	 %xmm5, %xmm9

  1337. 	xorpd	 %xmm5, %xmm11

  1338. 	xorpd	 %xmm5, %xmm13

  1339. 	xorpd	 %xmm5, %xmm15

  1340. 

  1341. 	mulpd	 %xmm6, %xmm8

  1342. 	mulpd	 %xmm7, %xmm9

  1343. 	mulpd	 %xmm6, %xmm10

  1344. 	mulpd	 %xmm7, %xmm11

  1345. 

  1346. 	mulpd	 %xmm6, %xmm12

  1347. 	mulpd	 %xmm7, %xmm13

  1348. 	mulpd	 %xmm6, %xmm14

  1349. 	mulpd	 %xmm7, %xmm15

  1350. 

  1351. #ifndef XCONJ

  1352. 	subpd	 %xmm9,  %xmm8

  1353. 	subpd	 %xmm11, %xmm10

  1354. 	subpd	 %xmm13, %xmm12

  1355. 	subpd	 %xmm15, %xmm14

  1356. #else

  1357. 	addpd	 %xmm9,  %xmm8

  1358. 	addpd	 %xmm11, %xmm10

  1359. 	addpd	 %xmm13, %xmm12

  1360. 	addpd	 %xmm15, %xmm14

  1361. #endif

  1362. 

  1363. 	pshufd	 $0xee, %xmm8,  %xmm9

  1364. 	pshufd	 $0x44, %xmm8,  %xmm8

  1365. 

  1366. 	pshufd	 $0xee, %xmm10, %xmm11

  1367. 	pshufd	 $0x44, %xmm10, %xmm10

  1368. 

  1369. 	pshufd	 $0xee, %xmm12, %xmm13

  1370. 	pshufd	 $0x44, %xmm12, %xmm12

  1371. 

  1372. 	pshufd	 $0xee, %xmm14, %xmm15

  1373. 	pshufd	 $0x44, %xmm14, %xmm14

  1374. 

  1375. #ifndef CONJ

  1376. 	xorpd	 %xmm5, %xmm9

  1377. 	xorpd	 %xmm5, %xmm11

  1378. 	xorpd	 %xmm5, %xmm13

  1379. 	xorpd	 %xmm5, %xmm15

  1380. #else

  1381. 	xorpd	 %xmm5, %xmm8

  1382. 	xorpd	 %xmm5, %xmm10

  1383. 	xorpd	 %xmm5, %xmm12

  1384. 	xorpd	 %xmm5, %xmm14

  1385. #endif

  1386. 

  1387. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  1388. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  1389. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  1390. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  1391. 	ALIGN_3

  1392. 

  1393. 	movq	M,   I

  1394. 	sarq	$2,  I

  1395. 	jle	.L105

  1396. 

  1397. 	movsd	-16 * SIZE(A1), %xmm4

  1398. 	movhpd	-15 * SIZE(A1), %xmm4

  1399. 	movsd	-14 * SIZE(A1), %xmm6

  1400. 	movhpd	-13 * SIZE(A1), %xmm6

  1401. 

  1402. 	decq	 I

  1403. 	jle	 .L104

  1404. 	ALIGN_3

  1405. 

  1406. .L103:

  1407. #ifdef PREFETCH

  1408. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1)

  1409. #endif

  1410. 

  1411. 	pshufd	 $0x4e, %xmm4, %xmm5

  1412. 	mulpd	 %xmm8,  %xmm4

  1413. 	addpd	 %xmm4,  %xmm0

  1414. 	movsd	-12 * SIZE(A1), %xmm4

  1415. 	movhpd	-11 * SIZE(A1), %xmm4

  1416. 	pshufd	 $0x4e, %xmm6, %xmm7

  1417. 	mulpd	 %xmm8,  %xmm6

  1418. 	addpd	 %xmm6,  %xmm1

  1419. 	movsd	-10 * SIZE(A1), %xmm6

  1420. 	movhpd	 -9 * SIZE(A1), %xmm6

  1421. 

  1422. 	mulpd	 %xmm9,  %xmm5

  1423. 	SUBPD	 %xmm5,  %xmm0

  1424. 	mulpd	 %xmm9,  %xmm7

  1425. 	SUBPD	 %xmm7,  %xmm1

  1426. 

  1427. 	pshufd	 $0x4e, %xmm4, %xmm5

  1428. 	mulpd	 %xmm8,  %xmm4

  1429. 	addpd	 %xmm4,  %xmm2

  1430. 	movsd	-16 * SIZE(A1, LDA), %xmm4

  1431. 	movhpd	-15 * SIZE(A1, LDA), %xmm4

  1432. 	pshufd	 $0x4e, %xmm6, %xmm7

  1433. 	mulpd	 %xmm8,  %xmm6

  1434. 	addpd	 %xmm6,  %xmm3

  1435. 	movsd	-14 * SIZE(A1, LDA), %xmm6

  1436. 	movhpd	-13 * SIZE(A1, LDA), %xmm6

  1437. 

  1438. 	mulpd	 %xmm9,  %xmm5

  1439. 	SUBPD	 %xmm5,  %xmm2

  1440. 	mulpd	 %xmm9,  %xmm7

  1441. 	SUBPD	 %xmm7,  %xmm3

  1442. 

  1443. #ifdef PREFETCH

  1444. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1, LDA)

  1445. #endif

  1446. 

  1447. 	pshufd	 $0x4e, %xmm4, %xmm5

  1448. 	mulpd	 %xmm10, %xmm4

  1449. 	addpd	 %xmm4,  %xmm0

  1450. 	movsd	-12 * SIZE(A1, LDA), %xmm4

  1451. 	movhpd	-11 * SIZE(A1, LDA), %xmm4

  1452. 	pshufd	 $0x4e, %xmm6, %xmm7

  1453. 	mulpd	 %xmm10, %xmm6

  1454. 	addpd	 %xmm6,  %xmm1

  1455. 	movsd	-10 * SIZE(A1, LDA), %xmm6

  1456. 	movhpd	 -9 * SIZE(A1, LDA), %xmm6

  1457. 

  1458. 	mulpd	 %xmm11, %xmm5

  1459. 	SUBPD	 %xmm5,  %xmm0

  1460. 	mulpd	 %xmm11, %xmm7

  1461. 	SUBPD	 %xmm7,  %xmm1

  1462. 

  1463. 	pshufd	 $0x4e, %xmm4, %xmm5

  1464. 	mulpd	 %xmm10, %xmm4

  1465. 	addpd	 %xmm4,  %xmm2

  1466. 	movsd	-16 * SIZE(A2), %xmm4

  1467. 	movhpd	-15 * SIZE(A2), %xmm4

  1468. 	pshufd	 $0x4e, %xmm6, %xmm7

  1469. 	mulpd	 %xmm10, %xmm6

  1470. 	addpd	 %xmm6,  %xmm3

  1471. 	movsd	-14 * SIZE(A2), %xmm6

  1472. 	movhpd	-13 * SIZE(A2), %xmm6

  1473. 

  1474. 	mulpd	 %xmm11, %xmm5

  1475. 	SUBPD	 %xmm5,  %xmm2

  1476. 	mulpd	 %xmm11, %xmm7

  1477. 	SUBPD	 %xmm7,  %xmm3

  1478. 

  1479. #ifdef PREFETCH

  1480. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2)

  1481. #endif

  1482. 

  1483. 	pshufd	 $0x4e, %xmm4, %xmm5

  1484. 	mulpd	 %xmm12, %xmm4

  1485. 	addpd	 %xmm4,  %xmm0

  1486. 	movsd	-12 * SIZE(A2), %xmm4

  1487. 	movhpd	-11 * SIZE(A2), %xmm4

  1488. 	pshufd	 $0x4e, %xmm6, %xmm7

  1489. 	mulpd	 %xmm12, %xmm6

  1490. 	addpd	 %xmm6,  %xmm1

  1491. 	movsd	-10 * SIZE(A2), %xmm6

  1492. 	movhpd	 -9 * SIZE(A2), %xmm6

  1493. 

  1494. 	mulpd	 %xmm13, %xmm5

  1495. 	SUBPD	 %xmm5,  %xmm0

  1496. 	mulpd	 %xmm13, %xmm7

  1497. 	SUBPD	 %xmm7,  %xmm1

  1498. 

  1499. 	pshufd	 $0x4e, %xmm4, %xmm5

  1500. 	mulpd	 %xmm12, %xmm4

  1501. 	addpd	 %xmm4,  %xmm2

  1502. 	movsd	-16 * SIZE(A2, LDA), %xmm4

  1503. 	movhpd	-15 * SIZE(A2, LDA), %xmm4

  1504. 	pshufd	 $0x4e, %xmm6, %xmm7

  1505. 	mulpd	 %xmm12, %xmm6

  1506. 	addpd	 %xmm6,  %xmm3

  1507. 	movsd	-14 * SIZE(A2, LDA), %xmm6

  1508. 	movhpd	-13 * SIZE(A2, LDA), %xmm6

  1509. 

  1510. 	mulpd	 %xmm13, %xmm5

  1511. 	SUBPD	 %xmm5,  %xmm2

  1512. 	mulpd	 %xmm13, %xmm7

  1513. 	SUBPD	 %xmm7,  %xmm3

  1514. 

  1515. #ifdef PREFETCH

  1516. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2, LDA)

  1517. #endif

  1518. 

  1519. 	pshufd	 $0x4e, %xmm4, %xmm5

  1520. 	mulpd	 %xmm14, %xmm4

  1521. 	addpd	 %xmm4,  %xmm0

  1522. 	movsd	-12 * SIZE(A2, LDA), %xmm4

  1523. 	movhpd	-11 * SIZE(A2, LDA), %xmm4

  1524. 	pshufd	 $0x4e, %xmm6, %xmm7

  1525. 	mulpd	 %xmm14, %xmm6

  1526. 	addpd	 %xmm6,  %xmm1

  1527. 	movsd	-10 * SIZE(A2, LDA), %xmm6

  1528. 	movhpd	 -9 * SIZE(A2, LDA), %xmm6

  1529. 

  1530. 	mulpd	 %xmm15, %xmm5

  1531. 	SUBPD	 %xmm5,  %xmm0

  1532. 	mulpd	 %xmm15, %xmm7

  1533. 	SUBPD	 %xmm7,  %xmm1

  1534. 

  1535. 	pshufd	 $0x4e, %xmm4, %xmm5

  1536. 	mulpd	 %xmm14, %xmm4

  1537. 	addpd	 %xmm4,  %xmm2

  1538. 	movsd	 -8 * SIZE(A1), %xmm4

  1539. 	movhpd	 -7 * SIZE(A1), %xmm4

  1540. 	pshufd	 $0x4e, %xmm6, %xmm7

  1541. 	mulpd	 %xmm14, %xmm6

  1542. 	addpd	 %xmm6,  %xmm3

  1543. 	movsd	 -6 * SIZE(A1), %xmm6

  1544. 	movhpd	 -5 * SIZE(A1), %xmm6

  1545. 

  1546. 	mulpd	 %xmm15, %xmm5

  1547. 	SUBPD	 %xmm5,  %xmm2

  1548. 	mulpd	 %xmm15, %xmm7

  1549. 	SUBPD	 %xmm7,  %xmm3

  1550. 

  1551. #ifdef PREFETCHW

  1552. 	PREFETCHW	(PREFETCHSIZE) - 128 + PREOFFSET(Y1)

  1553. #endif

  1554. 

  1555. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1556. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1557. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1558. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1559. 

  1560. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1561. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1562. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1563. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1564. 

  1565. 	subq	 $-8 * SIZE, A1

  1566. 	subq	 $-8 * SIZE, A2

  1567. 	subq	 $-8 * SIZE, Y1

  1568. 

  1569. 	subq	 $1, I

  1570. 	BRANCH

  1571. 	jg	.L103

  1572. 	ALIGN_3

  1573. 

  1574. .L104:

  1575. 	pshufd	 $0x4e, %xmm4, %xmm5

  1576. 	mulpd	 %xmm8,  %xmm4

  1577. 	addpd	 %xmm4,  %xmm0

  1578. 	movsd	-12 * SIZE(A1), %xmm4

  1579. 	movhpd	-11 * SIZE(A1), %xmm4

  1580. 	pshufd	 $0x4e, %xmm6, %xmm7

  1581. 	mulpd	 %xmm8,  %xmm6

  1582. 	addpd	 %xmm6,  %xmm1

  1583. 	movsd	-10 * SIZE(A1), %xmm6

  1584. 	movhpd	 -9 * SIZE(A1), %xmm6

  1585. 

  1586. 	mulpd	 %xmm9,  %xmm5

  1587. 	SUBPD	 %xmm5,  %xmm0

  1588. 	mulpd	 %xmm9,  %xmm7

  1589. 	SUBPD	 %xmm7,  %xmm1

  1590. 

  1591. 	pshufd	 $0x4e, %xmm4, %xmm5

  1592. 	mulpd	 %xmm8,  %xmm4

  1593. 	addpd	 %xmm4,  %xmm2

  1594. 	movsd	-16 * SIZE(A1, LDA), %xmm4

  1595. 	movhpd	-15 * SIZE(A1, LDA), %xmm4

  1596. 	pshufd	 $0x4e, %xmm6, %xmm7

  1597. 	mulpd	 %xmm8,  %xmm6

  1598. 	addpd	 %xmm6,  %xmm3

  1599. 	movsd	-14 * SIZE(A1, LDA), %xmm6

  1600. 	movhpd	-13 * SIZE(A1, LDA), %xmm6

  1601. 

  1602. 	mulpd	 %xmm9,  %xmm5

  1603. 	SUBPD	 %xmm5,  %xmm2

  1604. 	mulpd	 %xmm9,  %xmm7

  1605. 	SUBPD	 %xmm7,  %xmm3

  1606. 

  1607. 	pshufd	 $0x4e, %xmm4, %xmm5

  1608. 	mulpd	 %xmm10, %xmm4

  1609. 	addpd	 %xmm4,  %xmm0

  1610. 	movsd	-12 * SIZE(A1, LDA), %xmm4

  1611. 	movhpd	-11 * SIZE(A1, LDA), %xmm4

  1612. 	pshufd	 $0x4e, %xmm6, %xmm7

  1613. 	mulpd	 %xmm10, %xmm6

  1614. 	addpd	 %xmm6,  %xmm1

  1615. 	movsd	-10 * SIZE(A1, LDA), %xmm6

  1616. 	movhpd	 -9 * SIZE(A1, LDA), %xmm6

  1617. 

  1618. 	mulpd	 %xmm11, %xmm5

  1619. 	SUBPD	 %xmm5,  %xmm0

  1620. 	mulpd	 %xmm11, %xmm7

  1621. 	SUBPD	 %xmm7,  %xmm1

  1622. 

  1623. 	pshufd	 $0x4e, %xmm4, %xmm5

  1624. 	mulpd	 %xmm10, %xmm4

  1625. 	addpd	 %xmm4,  %xmm2

  1626. 	movsd	-16 * SIZE(A2), %xmm4

  1627. 	movhpd	-15 * SIZE(A2), %xmm4

  1628. 	pshufd	 $0x4e, %xmm6, %xmm7

  1629. 	mulpd	 %xmm10, %xmm6

  1630. 	addpd	 %xmm6,  %xmm3

  1631. 	movsd	-14 * SIZE(A2), %xmm6

  1632. 	movhpd	-13 * SIZE(A2), %xmm6

  1633. 

  1634. 	mulpd	 %xmm11, %xmm5

  1635. 	SUBPD	 %xmm5,  %xmm2

  1636. 	mulpd	 %xmm11, %xmm7

  1637. 	SUBPD	 %xmm7,  %xmm3

  1638. 

  1639. 	pshufd	 $0x4e, %xmm4, %xmm5

  1640. 	mulpd	 %xmm12, %xmm4

  1641. 	addpd	 %xmm4,  %xmm0

  1642. 	movsd	-12 * SIZE(A2), %xmm4

  1643. 	movhpd	-11 * SIZE(A2), %xmm4

  1644. 	pshufd	 $0x4e, %xmm6, %xmm7

  1645. 	mulpd	 %xmm12, %xmm6

  1646. 	addpd	 %xmm6,  %xmm1

  1647. 	movsd	-10 * SIZE(A2), %xmm6

  1648. 	movhpd	 -9 * SIZE(A2), %xmm6

  1649. 

  1650. 	mulpd	 %xmm13, %xmm5

  1651. 	SUBPD	 %xmm5,  %xmm0

  1652. 	mulpd	 %xmm13, %xmm7

  1653. 	SUBPD	 %xmm7,  %xmm1

  1654. 

  1655. 	pshufd	 $0x4e, %xmm4, %xmm5

  1656. 	mulpd	 %xmm12, %xmm4

  1657. 	addpd	 %xmm4,  %xmm2

  1658. 	movsd	-16 * SIZE(A2, LDA), %xmm4

  1659. 	movhpd	-15 * SIZE(A2, LDA), %xmm4

  1660. 	pshufd	 $0x4e, %xmm6, %xmm7

  1661. 	mulpd	 %xmm12, %xmm6

  1662. 	addpd	 %xmm6,  %xmm3

  1663. 	movsd	-14 * SIZE(A2, LDA), %xmm6

  1664. 	movhpd	-13 * SIZE(A2, LDA), %xmm6

  1665. 

  1666. 	mulpd	 %xmm13, %xmm5

  1667. 	SUBPD	 %xmm5,  %xmm2

  1668. 	mulpd	 %xmm13, %xmm7

  1669. 	SUBPD	 %xmm7,  %xmm3

  1670. 

  1671. 	pshufd	 $0x4e, %xmm4, %xmm5

  1672. 	mulpd	 %xmm14, %xmm4

  1673. 	addpd	 %xmm4,  %xmm0

  1674. 	movsd	-12 * SIZE(A2, LDA), %xmm4

  1675. 	movhpd	-11 * SIZE(A2, LDA), %xmm4

  1676. 	pshufd	 $0x4e, %xmm6, %xmm7

  1677. 	mulpd	 %xmm14, %xmm6

  1678. 	addpd	 %xmm6,  %xmm1

  1679. 	movsd	-10 * SIZE(A2, LDA), %xmm6

  1680. 	movhpd	 -9 * SIZE(A2, LDA), %xmm6

  1681. 

  1682. 	mulpd	 %xmm15, %xmm5

  1683. 	SUBPD	 %xmm5,  %xmm0

  1684. 	mulpd	 %xmm15, %xmm7

  1685. 	SUBPD	 %xmm7,  %xmm1

  1686. 

  1687. 	pshufd	 $0x4e, %xmm4, %xmm5

  1688. 	mulpd	 %xmm14, %xmm4

  1689. 	addpd	 %xmm4,  %xmm2

  1690. 	pshufd	 $0x4e, %xmm6, %xmm7

  1691. 	mulpd	 %xmm14, %xmm6

  1692. 	addpd	 %xmm6,  %xmm3

  1693. 

  1694. 	mulpd	 %xmm15, %xmm5

  1695. 	SUBPD	 %xmm5,  %xmm2

  1696. 	mulpd	 %xmm15, %xmm7

  1697. 	SUBPD	 %xmm7,  %xmm3

  1698. 

  1699. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1700. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1701. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1702. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1703. 

  1704. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1705. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1706. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1707. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1708. 

  1709. 	subq	 $-8 * SIZE, A1

  1710. 	subq	 $-8 * SIZE, A2

  1711. 	subq	 $-8 * SIZE, Y1

  1712. 	ALIGN_3

  1713. 

  1714. .L105:

  1715. 	testq	$2, M

  1716. 	je	.L107

  1717. 

  1718. 	movsd	-16 * SIZE(A1), %xmm4

  1719. 	movhpd	-15 * SIZE(A1), %xmm4

  1720. 	movsd	-14 * SIZE(A1), %xmm6

  1721. 	movhpd	-13 * SIZE(A1), %xmm6

  1722. 

  1723. 	pshufd	 $0x4e, %xmm4, %xmm5

  1724. 	mulpd	 %xmm8,  %xmm4

  1725. 	addpd	 %xmm4,  %xmm0

  1726. 	movsd	-16 * SIZE(A1, LDA), %xmm4

  1727. 	movhpd	-15 * SIZE(A1, LDA), %xmm4

  1728. 	pshufd	 $0x4e, %xmm6, %xmm7

  1729. 	mulpd	 %xmm8,  %xmm6

  1730. 	addpd	 %xmm6,  %xmm1

  1731. 	movsd	-14 * SIZE(A1, LDA), %xmm6

  1732. 	movhpd	-13 * SIZE(A1, LDA), %xmm6

  1733. 

  1734. 	mulpd	 %xmm9,  %xmm5

  1735. 	SUBPD	 %xmm5,  %xmm0

  1736. 	mulpd	 %xmm9,  %xmm7

  1737. 	SUBPD	 %xmm7,  %xmm1

  1738. 

  1739. 	pshufd	 $0x4e, %xmm4, %xmm5

  1740. 	mulpd	 %xmm10, %xmm4

  1741. 	addpd	 %xmm4,  %xmm0

  1742. 	movsd	-16 * SIZE(A2), %xmm4

  1743. 	movhpd	-15 * SIZE(A2), %xmm4

  1744. 	pshufd	 $0x4e, %xmm6, %xmm7

  1745. 	mulpd	 %xmm10, %xmm6

  1746. 	addpd	 %xmm6,  %xmm1

  1747. 	movsd	-14 * SIZE(A2), %xmm6

  1748. 	movhpd	-13 * SIZE(A2), %xmm6

  1749. 

  1750. 	mulpd	 %xmm11, %xmm5

  1751. 	SUBPD	 %xmm5,  %xmm0

  1752. 	mulpd	 %xmm11, %xmm7

  1753. 	SUBPD	 %xmm7,  %xmm1

  1754. 

  1755. 	pshufd	 $0x4e, %xmm4, %xmm5

  1756. 	mulpd	 %xmm12, %xmm4

  1757. 	addpd	 %xmm4,  %xmm0

  1758. 	movsd	-16 * SIZE(A2, LDA), %xmm4

  1759. 	movhpd	-15 * SIZE(A2, LDA), %xmm4

  1760. 	pshufd	 $0x4e, %xmm6, %xmm7

  1761. 	mulpd	 %xmm12, %xmm6

  1762. 	addpd	 %xmm6,  %xmm1

  1763. 	movsd	-14 * SIZE(A2, LDA), %xmm6

  1764. 	movhpd	-13 * SIZE(A2, LDA), %xmm6

  1765. 

  1766. 	mulpd	 %xmm13, %xmm5

  1767. 	SUBPD	 %xmm5,  %xmm0

  1768. 	mulpd	 %xmm13, %xmm7

  1769. 	SUBPD	 %xmm7,  %xmm1

  1770. 

  1771. 	pshufd	 $0x4e, %xmm4, %xmm5

  1772. 	mulpd	 %xmm14, %xmm4

  1773. 	addpd	 %xmm4,  %xmm0

  1774. 	mulpd	 %xmm15, %xmm5

  1775. 	SUBPD	 %xmm5,  %xmm0

  1776. 

  1777. 	pshufd	 $0x4e, %xmm6, %xmm7

  1778. 	mulpd	 %xmm14, %xmm6

  1779. 	addpd	 %xmm6,  %xmm1

  1780. 	mulpd	 %xmm15, %xmm7

  1781. 	SUBPD	 %xmm7,  %xmm1

  1782. 

  1783. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1784. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1785. 	movapd	 %xmm2, %xmm0

  1786. 

  1787. 	addq	 $4 * SIZE, A1

  1788. 	addq	 $4 * SIZE, A2

  1789. 	addq	 $4 * SIZE, Y1

  1790. 	ALIGN_3

  1791. 

  1792. .L107:

  1793. 	testq	$1, M

  1794. 	je	.L109

  1795. 

  1796. 	movsd	-16 * SIZE(A1), %xmm4

  1797. 	movhpd	-15 * SIZE(A1), %xmm4

  1798. 	movsd	-16 * SIZE(A1, LDA), %xmm6

  1799. 	movhpd	-15 * SIZE(A1, LDA), %xmm6

  1800. 

  1801. 	pshufd	 $0x4e, %xmm4, %xmm5

  1802. 	mulpd	 %xmm8,  %xmm4

  1803. 	addpd	 %xmm4,  %xmm0

  1804. 	movsd	-16 * SIZE(A2), %xmm4

  1805. 	movhpd	-15 * SIZE(A2), %xmm4

  1806. 	mulpd	 %xmm9,  %xmm5

  1807. 	SUBPD	 %xmm5,  %xmm0

  1808. 

  1809. 	pshufd	 $0x4e, %xmm6, %xmm7

  1810. 	mulpd	 %xmm10, %xmm6

  1811. 	addpd	 %xmm6,  %xmm0

  1812. 	movsd	-16 * SIZE(A2, LDA), %xmm6

  1813. 	movhpd	-15 * SIZE(A2, LDA), %xmm6

  1814. 	mulpd	 %xmm11, %xmm7

  1815. 	SUBPD	 %xmm7,  %xmm0

  1816. 

  1817. 	pshufd	 $0x4e, %xmm4, %xmm5

  1818. 	mulpd	 %xmm12, %xmm4

  1819. 	addpd	 %xmm4,  %xmm0

  1820. 	mulpd	 %xmm13, %xmm5

  1821. 	SUBPD	 %xmm5,  %xmm0

  1822. 

  1823. 	pshufd	 $0x4e, %xmm6, %xmm7

  1824. 	mulpd	 %xmm14, %xmm6

  1825. 	addpd	 %xmm6,  %xmm0

  1826. 	mulpd	 %xmm15, %xmm7

  1827. 	SUBPD	 %xmm7,  %xmm0

  1828. 

  1829. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1830. 	ALIGN_3

  1831. 

  1832. .L109:

  1833. 	cmpq	$4, N

  1834. 	jge	.L101

  1835. 	ALIGN_3

  1836. 

  1837. .L110:

  1838. #endif

  1839. 

  1840. #if GEMV_UNROLL >= 2

  1841. 

  1842. 	cmpq	$2, N

  1843. 	jl	.L120

  1844. 

  1845. #if GEMV_UNROLL == 2

  1846. 	ALIGN_3

  1847. 

  1848. .L111:

  1849. #endif

  1850. 

  1851. 	subq	$2, N

  1852. 

  1853. 	leaq	16 * SIZE(BUFFER), Y1

  1854. 	movq	A,  A1

  1855. 	leaq	(A,  LDA, 1), A2

  1856. 	leaq	(A,  LDA, 2), A

  1857. 

  1858. 	movsd	0 * SIZE(X), %xmm12

  1859. 	movhpd	1 * SIZE(X), %xmm12

  1860. 	addq	INCX, X

  1861. 	movsd	0 * SIZE(X), %xmm14

  1862. 	movhpd	1 * SIZE(X), %xmm14

  1863. 	addq	INCX, X

  1864. 

  1865. 	pcmpeqb	%xmm11, %xmm11

  1866. 	psllq	$63,    %xmm11

  1867. 	shufps	$0xc0, %xmm11, %xmm11

  1868. 

  1869. 	pshufd	$0x4e, %xmm12, %xmm13

  1870. 	pshufd	$0x4e, %xmm14, %xmm15

  1871. 

  1872. #ifdef HAVE_SSE3

  1873. 	movddup	ALPHA_R, %xmm8

  1874. 	movddup	ALPHA_I, %xmm9

  1875. #else

  1876. 	movsd	ALPHA_R, %xmm8

  1877. 	unpcklpd %xmm8, %xmm8

  1878. 	movsd	ALPHA_I, %xmm9

  1879. 	unpcklpd %xmm9, %xmm9

  1880. #endif

  1881. 

  1882. 	xorpd	 %xmm11, %xmm13

  1883. 	xorpd	 %xmm11, %xmm15

  1884. 

  1885. 	mulpd	 %xmm8, %xmm12

  1886. 	mulpd	 %xmm9, %xmm13

  1887. 	mulpd	 %xmm8, %xmm14

  1888. 	mulpd	 %xmm9, %xmm15

  1889. 

  1890. #ifndef XCONJ

  1891. 	subpd	 %xmm13, %xmm12

  1892. 	subpd	 %xmm15, %xmm14

  1893. #else

  1894. 	addpd	 %xmm13, %xmm12

  1895. 	addpd	 %xmm15, %xmm14

  1896. #endif

  1897. 

  1898. 	pshufd	 $0xee, %xmm12, %xmm13

  1899. 	pshufd	 $0x44, %xmm12, %xmm12

  1900. 

  1901. 	pshufd	 $0xee, %xmm14, %xmm15

  1902. 	pshufd	 $0x44, %xmm14, %xmm14

  1903. 

  1904. #ifndef CONJ

  1905. 	xorpd	 %xmm11, %xmm13

  1906. 	xorpd	 %xmm11, %xmm15

  1907. #else

  1908. 	xorpd	 %xmm11, %xmm12

  1909. 	xorpd	 %xmm11, %xmm14

  1910. #endif

  1911. 

  1912. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  1913. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  1914. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  1915. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  1916. 	ALIGN_3

  1917. 

  1918. 	movq	M,   I

  1919. 	sarq	$2,  I

  1920. 	jle	.L115

  1921. 

  1922. 	movsd	-16 * SIZE(A1), %xmm4

  1923. 	movhpd	-15 * SIZE(A1), %xmm4

  1924. 	movsd	-14 * SIZE(A1), %xmm6

  1925. 	movhpd	-13 * SIZE(A1), %xmm6

  1926. 	movsd	-12 * SIZE(A1), %xmm8

  1927. 	movhpd	-11 * SIZE(A1), %xmm8

  1928. 	movsd	-10 * SIZE(A1), %xmm10

  1929. 	movhpd	 -9 * SIZE(A1), %xmm10

  1930. 

  1931. 	decq	 I

  1932. 	jle	 .L114

  1933. 	ALIGN_3

  1934. 

  1935. .L113:

  1936. #ifdef PREFETCH

  1937. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2)

  1938. #endif

  1939. 

  1940. 	pshufd	 $0x4e, %xmm4,  %xmm5

  1941. 	mulpd	 %xmm12, %xmm4

  1942. 	addpd	 %xmm4,  %xmm0

  1943. 	movsd	-16 * SIZE(A2), %xmm4

  1944. 	movhpd	-15 * SIZE(A2), %xmm4

  1945. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1946. 	mulpd	 %xmm12, %xmm6

  1947. 	addpd	 %xmm6,  %xmm1

  1948. 	movsd	-14 * SIZE(A2), %xmm6

  1949. 	movhpd	-13 * SIZE(A2), %xmm6

  1950. 

  1951. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1952. 	mulpd	 %xmm12, %xmm8

  1953. 	addpd	 %xmm8,  %xmm2

  1954. 	movsd	-12 * SIZE(A2), %xmm8

  1955. 	movhpd	-11 * SIZE(A2), %xmm8

  1956. 	pshufd	 $0x4e, %xmm10, %xmm11

  1957. 	mulpd	 %xmm12, %xmm10

  1958. 	addpd	 %xmm10, %xmm3

  1959. 	movsd	-10 * SIZE(A2), %xmm10

  1960. 	movhpd	 -9 * SIZE(A2), %xmm10

  1961. 

  1962. 	mulpd	 %xmm13, %xmm5

  1963. 	SUBPD	 %xmm5,  %xmm0

  1964. 	mulpd	 %xmm13, %xmm7

  1965. 	SUBPD	 %xmm7,  %xmm1

  1966. 

  1967. 	mulpd	 %xmm13, %xmm9

  1968. 	SUBPD	 %xmm9,  %xmm2

  1969. 	mulpd	 %xmm13, %xmm11

  1970. 	SUBPD	 %xmm11, %xmm3

  1971. 

  1972. #ifdef PREFETCH

  1973. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1)

  1974. #endif

  1975. 

  1976. 	pshufd	 $0x4e, %xmm4,  %xmm5

  1977. 	mulpd	 %xmm14, %xmm4

  1978. 	addpd	 %xmm4,  %xmm0

  1979. 	movsd	 -8 * SIZE(A1), %xmm4

  1980. 	movhpd	 -7 * SIZE(A1), %xmm4

  1981. 	pshufd	 $0x4e, %xmm6,  %xmm7

  1982. 	mulpd	 %xmm14, %xmm6

  1983. 	addpd	 %xmm6,  %xmm1

  1984. 	movsd	 -6 * SIZE(A1), %xmm6

  1985. 	movhpd	 -5 * SIZE(A1), %xmm6

  1986. 

  1987. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1988. 	mulpd	 %xmm14, %xmm8

  1989. 	addpd	 %xmm8,  %xmm2

  1990. 	movsd	 -4 * SIZE(A1), %xmm8

  1991. 	movhpd	 -3 * SIZE(A1), %xmm8

  1992. 	pshufd	 $0x4e, %xmm10, %xmm11

  1993. 	mulpd	 %xmm14, %xmm10

  1994. 	addpd	 %xmm10, %xmm3

  1995. 	movsd	 -2 * SIZE(A1), %xmm10

  1996. 	movhpd	 -1 * SIZE(A1), %xmm10

  1997. 

  1998. 	mulpd	 %xmm15, %xmm5

  1999. 	SUBPD	 %xmm5,  %xmm0

  2000. 	mulpd	 %xmm15, %xmm7

  2001. 	SUBPD	 %xmm7,  %xmm1

  2002. 

  2003. 	mulpd	 %xmm15, %xmm9

  2004. 	SUBPD	 %xmm9,  %xmm2

  2005. 	mulpd	 %xmm15, %xmm11

  2006. 	SUBPD	 %xmm11, %xmm3

  2007. 

  2008. #ifdef PREFETCHW

  2009. 	PREFETCHW	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(Y1)

  2010. #endif

  2011. 

  2012. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2013. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2014. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  2015. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  2016. 

  2017. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  2018. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  2019. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  2020. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  2021. 

  2022. 	subq	 $-8 * SIZE, A1

  2023. 	subq	 $-8 * SIZE, A2

  2024. 	subq	 $-8 * SIZE, Y1

  2025. 

  2026. 	subq	 $1, I

  2027. 	BRANCH

  2028. 	jg	.L113

  2029. 	ALIGN_3

  2030. 

  2031. .L114:

  2032. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2033. 	mulpd	 %xmm12, %xmm4

  2034. 	addpd	 %xmm4,  %xmm0

  2035. 	movsd	-16 * SIZE(A2), %xmm4

  2036. 	movhpd	-15 * SIZE(A2), %xmm4

  2037. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2038. 	mulpd	 %xmm12, %xmm6

  2039. 	addpd	 %xmm6,  %xmm1

  2040. 	movsd	-14 * SIZE(A2), %xmm6

  2041. 	movhpd	-13 * SIZE(A2), %xmm6

  2042. 

  2043. 	pshufd	 $0x4e, %xmm8,  %xmm9

  2044. 	mulpd	 %xmm12, %xmm8

  2045. 	addpd	 %xmm8,  %xmm2

  2046. 	movsd	-12 * SIZE(A2), %xmm8

  2047. 	movhpd	-11 * SIZE(A2), %xmm8

  2048. 	pshufd	 $0x4e, %xmm10, %xmm11

  2049. 	mulpd	 %xmm12, %xmm10

  2050. 	addpd	 %xmm10, %xmm3

  2051. 	movsd	-10 * SIZE(A2), %xmm10

  2052. 	movhpd	 -9 * SIZE(A2), %xmm10

  2053. 

  2054. 	mulpd	 %xmm13, %xmm5

  2055. 	SUBPD	 %xmm5,  %xmm0

  2056. 	mulpd	 %xmm13, %xmm7

  2057. 	SUBPD	 %xmm7,  %xmm1

  2058. 

  2059. 	mulpd	 %xmm13, %xmm9

  2060. 	SUBPD	 %xmm9,  %xmm2

  2061. 	mulpd	 %xmm13, %xmm11

  2062. 	SUBPD	 %xmm11, %xmm3

  2063. 

  2064. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2065. 	mulpd	 %xmm14, %xmm4

  2066. 	addpd	 %xmm4,  %xmm0

  2067. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2068. 	mulpd	 %xmm14, %xmm6

  2069. 	addpd	 %xmm6,  %xmm1

  2070. 

  2071. 	pshufd	 $0x4e, %xmm8,  %xmm9

  2072. 	mulpd	 %xmm14, %xmm8

  2073. 	addpd	 %xmm8,  %xmm2

  2074. 	pshufd	 $0x4e, %xmm10, %xmm11

  2075. 	mulpd	 %xmm14, %xmm10

  2076. 	addpd	 %xmm10, %xmm3

  2077. 

  2078. 	mulpd	 %xmm15, %xmm5

  2079. 	SUBPD	 %xmm5,  %xmm0

  2080. 	mulpd	 %xmm15, %xmm7

  2081. 	SUBPD	 %xmm7,  %xmm1

  2082. 

  2083. 	mulpd	 %xmm15, %xmm9

  2084. 	SUBPD	 %xmm9,  %xmm2

  2085. 	mulpd	 %xmm15, %xmm11

  2086. 	SUBPD	 %xmm11, %xmm3

  2087. 

  2088. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2089. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2090. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  2091. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  2092. 

  2093. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  2094. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  2095. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  2096. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  2097. 

  2098. 	subq	 $-8 * SIZE, A1

  2099. 	subq	 $-8 * SIZE, A2

  2100. 	subq	 $-8 * SIZE, Y1

  2101. 	ALIGN_3

  2102. 

  2103. .L115:

  2104. 	testq	$2, M

  2105. 	je	.L117

  2106. 

  2107. 	movsd	-16 * SIZE(A1), %xmm4

  2108. 	movhpd	-15 * SIZE(A1), %xmm4

  2109. 	movsd	-14 * SIZE(A1), %xmm6

  2110. 	movhpd	-13 * SIZE(A1), %xmm6

  2111. 

  2112. 	movsd	-16 * SIZE(A2), %xmm8

  2113. 	movhpd	-15 * SIZE(A2), %xmm8

  2114. 	movsd	-14 * SIZE(A2), %xmm10

  2115. 	movhpd	-13 * SIZE(A2), %xmm10

  2116. 

  2117. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2118. 	mulpd	 %xmm12, %xmm4

  2119. 	addpd	 %xmm4,  %xmm0

  2120. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2121. 	mulpd	 %xmm12, %xmm6

  2122. 	addpd	 %xmm6,  %xmm1

  2123. 

  2124. 	mulpd	 %xmm13, %xmm5

  2125. 	SUBPD	 %xmm5,  %xmm0

  2126. 	mulpd	 %xmm13, %xmm7

  2127. 	SUBPD	 %xmm7,  %xmm1

  2128. 

  2129. 	pshufd	 $0x4e, %xmm8,  %xmm9

  2130. 	mulpd	 %xmm14, %xmm8

  2131. 	addpd	 %xmm8,  %xmm0

  2132. 	pshufd	 $0x4e, %xmm10, %xmm11

  2133. 	mulpd	 %xmm14, %xmm10

  2134. 	addpd	 %xmm10, %xmm1

  2135. 

  2136. 	mulpd	 %xmm15, %xmm9

  2137. 	SUBPD	 %xmm9,  %xmm0

  2138. 	mulpd	 %xmm15, %xmm11

  2139. 	SUBPD	 %xmm11, %xmm1

  2140. 

  2141. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2142. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2143. 	movapd	 %xmm2, %xmm0

  2144. 

  2145. 	addq	 $4 * SIZE, A1

  2146. 	addq	 $4 * SIZE, A2

  2147. 	addq	 $4 * SIZE, Y1

  2148. 	ALIGN_3

  2149. 

  2150. .L117:

  2151. 	testq	$1, M

  2152. #if GEMV_UNROLL == 2

  2153. 	je	.L119

  2154. #else

  2155. 	je	.L120

  2156. #endif

  2157. 

  2158. 	movsd	-16 * SIZE(A1), %xmm4

  2159. 	movhpd	-15 * SIZE(A1), %xmm4

  2160. 	movsd	-16 * SIZE(A2), %xmm6

  2161. 	movhpd	-15 * SIZE(A2), %xmm6

  2162. 

  2163. 	pshufd	 $0x4e, %xmm4, %xmm5

  2164. 	mulpd	 %xmm12, %xmm4

  2165. 	addpd	 %xmm4,  %xmm0

  2166. 	mulpd	 %xmm13, %xmm5

  2167. 	SUBPD	 %xmm5,  %xmm0

  2168. 

  2169. 	pshufd	 $0x4e, %xmm6, %xmm7

  2170. 	mulpd	 %xmm14, %xmm6

  2171. 	addpd	 %xmm6,  %xmm0

  2172. 	mulpd	 %xmm15, %xmm7

  2173. 	SUBPD	 %xmm7,  %xmm0

  2174. 

  2175. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2176. 

  2177. #if GEMV_UNROLL == 2

  2178. 	ALIGN_3

  2179. 

  2180. .L119:

  2181. 	cmpq	$2, N

  2182. 	jge	.L111

  2183. #endif

  2184. 	ALIGN_3

  2185. 

  2186. .L120:

  2187. #endif

  2188. 

  2189. 	cmpq	$1, N

  2190. 	jl	.L980

  2191. 

  2192. #if GEMV_UNROLL == 1

  2193. .L121:

  2194. 	decq	N

  2195. #endif

  2196. 

  2197. 	leaq	16 * SIZE(BUFFER), Y1

  2198. 	movq	A,  A1

  2199. #if GEMV_UNROLL == 1

  2200. 	addq	LDA, A

  2201. #endif

  2202. 

  2203. 	movsd	0 * SIZE(X), %xmm12

  2204. 	movhpd	1 * SIZE(X), %xmm12

  2205. 	addq	INCX, X

  2206. 

  2207. 	pcmpeqb	%xmm11, %xmm11

  2208. 	psllq	$63,    %xmm11

  2209. 	shufps	$0xc0, %xmm11, %xmm11

  2210. 

  2211. 	pshufd	$0x4e, %xmm12, %xmm13

  2212. 

  2213. #ifdef HAVE_SSE3

  2214. 	movddup	ALPHA_R, %xmm8

  2215. 	movddup	ALPHA_I, %xmm9

  2216. #else

  2217. 	movsd	ALPHA_R, %xmm8

  2218. 	unpcklpd %xmm8, %xmm8

  2219. 	movsd	ALPHA_I, %xmm9

  2220. 	unpcklpd %xmm9, %xmm9

  2221. #endif

  2222. 

  2223. 	xorpd	 %xmm11, %xmm13

  2224. 

  2225. 	mulpd	 %xmm8, %xmm12

  2226. 	mulpd	 %xmm9, %xmm13

  2227. 

  2228. #ifndef XCONJ

  2229. 	subpd	 %xmm13, %xmm12

  2230. #else

  2231. 	addpd	 %xmm13, %xmm12

  2232. #endif

  2233. 

  2234. 	pshufd	 $0xee, %xmm12, %xmm13

  2235. 	pshufd	 $0x44, %xmm12, %xmm12

  2236. 

  2237. #ifndef CONJ

  2238. 	xorpd	 %xmm11, %xmm13

  2239. #else

  2240. 	xorpd	 %xmm11, %xmm12

  2241. #endif

  2242. 

  2243. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  2244. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  2245. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  2246. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  2247. 

  2248. 	movq	M,   I

  2249. 	sarq	$2,  I

  2250. 	jle	.L125

  2251. 

  2252. 	movsd	-16 * SIZE(A1), %xmm4

  2253. 	movhpd	-15 * SIZE(A1), %xmm4

  2254. 	movsd	-14 * SIZE(A1), %xmm6

  2255. 	movhpd	-13 * SIZE(A1), %xmm6

  2256. 	movsd	-12 * SIZE(A1), %xmm8

  2257. 	movhpd	-11 * SIZE(A1), %xmm8

  2258. 	movsd	-10 * SIZE(A1), %xmm10

  2259. 	movhpd	 -9 * SIZE(A1), %xmm10

  2260. 

  2261. 	decq	 I

  2262. 	jle	 .L124

  2263. 	ALIGN_3

  2264. 

  2265. .L123:

  2266. #ifdef PREFETCH

  2267. 	PREFETCH	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1)

  2268. #endif

  2269. 

  2270. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2271. 	mulpd	 %xmm12, %xmm4

  2272. 	addpd	 %xmm4,  %xmm0

  2273. 	movsd	 -8 * SIZE(A1), %xmm4

  2274. 	movhpd	 -7 * SIZE(A1), %xmm4

  2275. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2276. 	mulpd	 %xmm12, %xmm6

  2277. 	addpd	 %xmm6,  %xmm1

  2278. 	movsd	 -6 * SIZE(A1), %xmm6

  2279. 	movhpd	 -5 * SIZE(A1), %xmm6

  2280. 

  2281. 	pshufd	 $0x4e, %xmm8,  %xmm9

  2282. 	mulpd	 %xmm12, %xmm8

  2283. 	addpd	 %xmm8,  %xmm2

  2284. 	movsd	 -4 * SIZE(A1), %xmm8

  2285. 	movhpd	 -3 * SIZE(A1), %xmm8

  2286. 	pshufd	 $0x4e, %xmm10, %xmm11

  2287. 	mulpd	 %xmm12, %xmm10

  2288. 	addpd	 %xmm10, %xmm3

  2289. 	movsd	 -2 * SIZE(A1), %xmm10

  2290. 	movhpd	 -1 * SIZE(A1), %xmm10

  2291. 

  2292. 	mulpd	 %xmm13, %xmm5

  2293. 	SUBPD	 %xmm5,  %xmm0

  2294. 	mulpd	 %xmm13, %xmm7

  2295. 	SUBPD	 %xmm7,  %xmm1

  2296. 

  2297. 	mulpd	 %xmm13, %xmm9

  2298. 	SUBPD	 %xmm9,  %xmm2

  2299. 	mulpd	 %xmm13, %xmm11

  2300. 	SUBPD	 %xmm11, %xmm3

  2301. 

  2302. #ifdef PREFETCHW

  2303. 	PREFETCHW	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(Y1)

  2304. #endif

  2305. 

  2306. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2307. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2308. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  2309. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  2310. 

  2311. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  2312. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  2313. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  2314. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  2315. 

  2316. 	subq	 $-8 * SIZE, A1

  2317. 	subq	 $-8 * SIZE, Y1

  2318. 

  2319. 	subq	 $1, I

  2320. 	BRANCH

  2321. 	jg	.L123

  2322. 	ALIGN_3

  2323. 

  2324. .L124:

  2325. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2326. 	mulpd	 %xmm12, %xmm4

  2327. 	addpd	 %xmm4,  %xmm0

  2328. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2329. 	mulpd	 %xmm12, %xmm6

  2330. 	addpd	 %xmm6,  %xmm1

  2331. 

  2332. 	pshufd	 $0x4e, %xmm8,  %xmm9

  2333. 	mulpd	 %xmm12, %xmm8

  2334. 	addpd	 %xmm8,  %xmm2

  2335. 	pshufd	 $0x4e, %xmm10, %xmm11

  2336. 	mulpd	 %xmm12, %xmm10

  2337. 	addpd	 %xmm10, %xmm3

  2338. 

  2339. 	mulpd	 %xmm13, %xmm5

  2340. 	SUBPD	 %xmm5,  %xmm0

  2341. 	mulpd	 %xmm13, %xmm7

  2342. 	SUBPD	 %xmm7,  %xmm1

  2343. 

  2344. 	mulpd	 %xmm13, %xmm9

  2345. 	SUBPD	 %xmm9,  %xmm2

  2346. 	mulpd	 %xmm13, %xmm11

  2347. 	SUBPD	 %xmm11, %xmm3

  2348. 

  2349. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2350. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2351. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  2352. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  2353. 

  2354. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  2355. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  2356. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  2357. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  2358. 

  2359. 	subq	 $-8 * SIZE, A1

  2360. 	subq	 $-8 * SIZE, Y1

  2361. 	ALIGN_3

  2362. 

  2363. .L125:

  2364. 	testq	$2, M

  2365. 	je	.L127

  2366. 

  2367. 

  2368. 	movsd	-16 * SIZE(A1), %xmm4

  2369. 	movhpd	-15 * SIZE(A1), %xmm4

  2370. 	movsd	-14 * SIZE(A1), %xmm6

  2371. 	movhpd	-13 * SIZE(A1), %xmm6

  2372. 

  2373. 	pshufd	 $0x4e, %xmm4,  %xmm5

  2374. 	mulpd	 %xmm12, %xmm4

  2375. 	addpd	 %xmm4,  %xmm0

  2376. 	pshufd	 $0x4e, %xmm6,  %xmm7

  2377. 	mulpd	 %xmm12, %xmm6

  2378. 	addpd	 %xmm6,  %xmm1

  2379. 

  2380. 	mulpd	 %xmm13, %xmm5

  2381. 	SUBPD	 %xmm5,  %xmm0

  2382. 	mulpd	 %xmm13, %xmm7

  2383. 	SUBPD	 %xmm7,  %xmm1

  2384. 

  2385. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2386. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  2387. 	movapd	 %xmm2, %xmm0

  2388. 

  2389. 	addq	 $4 * SIZE, A1

  2390. 	addq	 $4 * SIZE, Y1

  2391. 	ALIGN_3

  2392. 

  2393. .L127:

  2394. 	testq	$1, M

  2395. #if GEMV_UNROLL == 1

  2396. 	je	.L129

  2397. #else

  2398. 	je	.L980

  2399. #endif

  2400. 

  2401. 	movsd	-16 * SIZE(A1), %xmm4

  2402. 	movhpd	-15 * SIZE(A1), %xmm4

  2403. 

  2404. 	pshufd	 $0x4e, %xmm4, %xmm5

  2405. 	mulpd	 %xmm12, %xmm4

  2406. 	addpd	 %xmm4,  %xmm0

  2407. 	mulpd	 %xmm13, %xmm5

  2408. 	SUBPD	 %xmm5,  %xmm0

  2409. 

  2410. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  2411. 

  2412. #if GEMV_UNROLL == 1

  2413. 	ALIGN_3

  2414. .L129:

  2415. 	cmpq	$1, N

  2416. 	jge	.L121

  2417. #endif

  2418. 

  2419. 

  2420. #endif

  2421. 	ALIGN_3

  2422. 

  2423. .L980:

  2424. 	testq	$SIZE, Y

  2425. 	jne	.L990

  2426. 

  2427. 	movq	Y,  Y1

  2428. 

  2429. 	movq	M,  %rax

  2430. 	sarq	$3, %rax

  2431. 	jle	.L184

  2432. 	ALIGN_3

  2433. 

  2434. .L182:

  2435.  	movapd	 (Y), %xmm0

  2436. 	addq	INCY, Y

  2437. 	movapd	 (Y), %xmm1

  2438. 	addq	INCY, Y

  2439. 	movapd	 (Y), %xmm2

  2440. 	addq	INCY, Y

  2441. 	movapd	 (Y), %xmm3

  2442. 	addq	INCY, Y

  2443.  	movapd	 (Y), %xmm4

  2444. 	addq	INCY, Y

  2445. 	movapd	 (Y), %xmm5

  2446. 	addq	INCY, Y

  2447. 	movapd	 (Y), %xmm6

  2448. 	addq	INCY, Y

  2449. 	movapd	 (Y), %xmm7

  2450. 	addq	INCY, Y

  2451. 

  2452. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2453. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2454. 	addpd	 4 * SIZE(BUFFER), %xmm2

  2455. 	addpd	 6 * SIZE(BUFFER), %xmm3

  2456. 	addpd	 8 * SIZE(BUFFER), %xmm4

  2457. 	addpd	10 * SIZE(BUFFER), %xmm5

  2458. 	addpd	12 * SIZE(BUFFER), %xmm6

  2459. 	addpd	14 * SIZE(BUFFER), %xmm7

  2460. 

  2461. 	movapd	%xmm0,  (Y1)

  2462. 	addq	INCY, Y1

  2463. 	movapd	%xmm1,  (Y1)

  2464. 	addq	INCY, Y1

  2465. 	movapd	%xmm2,  (Y1)

  2466. 	addq	INCY, Y1

  2467. 	movapd	%xmm3,  (Y1)

  2468. 	addq	INCY, Y1

  2469. 	movapd	%xmm4,  (Y1)

  2470. 	addq	INCY, Y1

  2471. 	movapd	%xmm5,  (Y1)

  2472. 	addq	INCY, Y1

  2473. 	movapd	%xmm6,  (Y1)

  2474. 	addq	INCY, Y1

  2475. 	movapd	%xmm7,  (Y1)

  2476. 	addq	INCY, Y1

  2477. 

  2478. 	subq	$-16 * SIZE, BUFFER

  2479. 	decq	%rax

  2480. 	jg	.L182

  2481. 	ALIGN_3

  2482. 

  2483. .L184:

  2484. 	testq	$7, M

  2485. 	jle	.L999

  2486. 

  2487. 	testq	$4, M

  2488. 	jle	.L185

  2489. 

  2490.  	movapd	 (Y), %xmm0

  2491. 	addq	INCY, Y

  2492. 	movapd	 (Y), %xmm1

  2493. 	addq	INCY, Y

  2494. 	movapd	 (Y), %xmm2

  2495. 	addq	INCY, Y

  2496. 	movapd	 (Y), %xmm3

  2497. 	addq	INCY, Y

  2498. 

  2499. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2500. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2501. 	addpd	 4 * SIZE(BUFFER), %xmm2

  2502. 	addpd	 6 * SIZE(BUFFER), %xmm3

  2503. 

  2504. 	movapd	%xmm0,  (Y1)

  2505. 	addq	INCY, Y1

  2506. 	movapd	%xmm1,  (Y1)

  2507. 	addq	INCY, Y1

  2508. 	movapd	%xmm2,  (Y1)

  2509. 	addq	INCY, Y1

  2510. 	movapd	%xmm3,  (Y1)

  2511. 	addq	INCY, Y1

  2512. 

  2513. 	addq	$8 * SIZE, BUFFER

  2514. 	ALIGN_3

  2515. 

  2516. .L185:

  2517. 	testq	$2, M

  2518. 	jle	.L186

  2519. 

  2520.  	movapd	 (Y), %xmm0

  2521. 	addq	INCY, Y

  2522. 	movapd	 (Y), %xmm1

  2523. 	addq	INCY, Y

  2524. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2525. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2526. 

  2527. 	movapd	%xmm0,  (Y1)

  2528. 	addq	INCY, Y1

  2529. 	movapd	%xmm1,  (Y1)

  2530. 	addq	INCY, Y1

  2531. 

  2532. 	addq	$4 * SIZE, BUFFER

  2533. 	ALIGN_3

  2534. 

  2535. .L186:

  2536. 	testq	$1, M

  2537. 	jle	.L999

  2538. 

  2539.  	movapd	 (Y), %xmm0

  2540. 

  2541. 	addpd	 (BUFFER), %xmm0

  2542. 

  2543. 	movapd	%xmm0,  (Y1)

  2544. 	jmp	.L999

  2545. 	ALIGN_3

  2546. 

  2547. .L990:

  2548. 	movq	Y,  Y1

  2549. 

  2550. 	movq	M,  %rax

  2551. 	sarq	$3, %rax

  2552. 	jle	.L994

  2553. 	ALIGN_3

  2554. 

  2555. .L992:

  2556.  	movsd	 0 * SIZE(Y), %xmm0

  2557. 	movhpd	 1 * SIZE(Y), %xmm0

  2558. 	addq	INCY, Y

  2559. 

  2560. 	movsd	 0 * SIZE(Y), %xmm1

  2561. 	movhpd	 1 * SIZE(Y), %xmm1

  2562. 	addq	INCY, Y

  2563. 

  2564. 	movsd	 0 * SIZE(Y), %xmm2

  2565. 	movhpd	 1 * SIZE(Y), %xmm2

  2566. 	addq	INCY, Y

  2567. 

  2568. 	movsd	 0 * SIZE(Y), %xmm3

  2569. 	movhpd	 1 * SIZE(Y), %xmm3

  2570. 	addq	INCY, Y

  2571. 

  2572.  	movsd	 0 * SIZE(Y), %xmm4

  2573. 	movhpd	 1 * SIZE(Y), %xmm4

  2574. 	addq	INCY, Y

  2575. 

  2576. 	movsd	 0 * SIZE(Y), %xmm5

  2577. 	movhpd	 1 * SIZE(Y), %xmm5

  2578. 	addq	INCY, Y

  2579. 

  2580. 	movsd	 0 * SIZE(Y), %xmm6

  2581. 	movhpd	 1 * SIZE(Y), %xmm6

  2582. 	addq	INCY, Y

  2583. 

  2584. 	movsd	 0 * SIZE(Y), %xmm7

  2585. 	movhpd	 1 * SIZE(Y), %xmm7

  2586. 	addq	INCY, Y

  2587. 

  2588. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2589. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2590. 	addpd	 4 * SIZE(BUFFER), %xmm2

  2591. 	addpd	 6 * SIZE(BUFFER), %xmm3

  2592. 	addpd	 8 * SIZE(BUFFER), %xmm4

  2593. 	addpd	10 * SIZE(BUFFER), %xmm5

  2594. 	addpd	12 * SIZE(BUFFER), %xmm6

  2595. 	addpd	14 * SIZE(BUFFER), %xmm7

  2596. 

  2597. 	movlpd	%xmm0,  0 * SIZE(Y1)

  2598. 	movhpd	%xmm0,  1 * SIZE(Y1)

  2599. 	addq	INCY, Y1

  2600. 

  2601. 	movlpd	%xmm1,  0 * SIZE(Y1)

  2602. 	movhpd	%xmm1,  1 * SIZE(Y1)

  2603. 	addq	INCY, Y1

  2604. 

  2605. 	movlpd	%xmm2,  0 * SIZE(Y1)

  2606. 	movhpd	%xmm2,  1 * SIZE(Y1)

  2607. 	addq	INCY, Y1

  2608. 

  2609. 	movlpd	%xmm3,  0 * SIZE(Y1)

  2610. 	movhpd	%xmm3,  1 * SIZE(Y1)

  2611. 	addq	INCY, Y1

  2612. 

  2613. 	movlpd	%xmm4,  0 * SIZE(Y1)

  2614. 	movhpd	%xmm4,  1 * SIZE(Y1)

  2615. 	addq	INCY, Y1

  2616. 

  2617. 	movlpd	%xmm5,  0 * SIZE(Y1)

  2618. 	movhpd	%xmm5,  1 * SIZE(Y1)

  2619. 	addq	INCY, Y1

  2620. 

  2621. 	movlpd	%xmm6,  0 * SIZE(Y1)

  2622. 	movhpd	%xmm6,  1 * SIZE(Y1)

  2623. 	addq	INCY, Y1

  2624. 

  2625. 	movlpd	%xmm7,  0 * SIZE(Y1)

  2626. 	movhpd	%xmm7,  1 * SIZE(Y1)

  2627. 	addq	INCY, Y1

  2628. 

  2629. 	subq	$-16 * SIZE, BUFFER

  2630. 	decq	%rax

  2631. 	jg	.L992

  2632. 	ALIGN_3

  2633. 

  2634. .L994:

  2635. 	testq	$7, M

  2636. 	jle	.L999

  2637. 

  2638. 	testq	$4, M

  2639. 	jle	.L995

  2640. 

  2641.  	movsd	 0 * SIZE(Y), %xmm0

  2642. 	movhpd	 1 * SIZE(Y), %xmm0

  2643. 	addq	INCY, Y

  2644. 

  2645. 	movsd	 0 * SIZE(Y), %xmm1

  2646. 	movhpd	 1 * SIZE(Y), %xmm1

  2647. 	addq	INCY, Y

  2648. 

  2649. 	movsd	 0 * SIZE(Y), %xmm2

  2650. 	movhpd	 1 * SIZE(Y), %xmm2

  2651. 	addq	INCY, Y

  2652. 

  2653. 	movsd	 0 * SIZE(Y), %xmm3

  2654. 	movhpd	 1 * SIZE(Y), %xmm3

  2655. 	addq	INCY, Y

  2656. 

  2657. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2658. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2659. 	addpd	 4 * SIZE(BUFFER), %xmm2

  2660. 	addpd	 6 * SIZE(BUFFER), %xmm3

  2661. 

  2662. 	movlpd	%xmm0,  0 * SIZE(Y1)

  2663. 	movhpd	%xmm0,  1 * SIZE(Y1)

  2664. 	addq	INCY, Y1

  2665. 

  2666. 	movlpd	%xmm1,  0 * SIZE(Y1)

  2667. 	movhpd	%xmm1,  1 * SIZE(Y1)

  2668. 	addq	INCY, Y1

  2669. 

  2670. 	movlpd	%xmm2,  0 * SIZE(Y1)

  2671. 	movhpd	%xmm2,  1 * SIZE(Y1)

  2672. 	addq	INCY, Y1

  2673. 

  2674. 	movlpd	%xmm3,  0 * SIZE(Y1)

  2675. 	movhpd	%xmm3,  1 * SIZE(Y1)

  2676. 	addq	INCY, Y1

  2677. 

  2678. 	addq	$8 * SIZE, BUFFER

  2679. 	ALIGN_3

  2680. 

  2681. .L995:

  2682. 	testq	$2, M

  2683. 	jle	.L996

  2684. 

  2685.  	movsd	 0 * SIZE(Y), %xmm0

  2686. 	movhpd	 1 * SIZE(Y), %xmm0

  2687. 	addq	INCY, Y

  2688. 

  2689. 	movsd	 0 * SIZE(Y), %xmm1

  2690. 	movhpd	 1 * SIZE(Y), %xmm1

  2691. 	addq	INCY, Y

  2692. 

  2693. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2694. 	addpd	 2 * SIZE(BUFFER), %xmm1

  2695. 

  2696. 	movlpd	%xmm0,  0 * SIZE(Y1)

  2697. 	movhpd	%xmm0,  1 * SIZE(Y1)

  2698. 	addq	INCY, Y1

  2699. 

  2700. 	movlpd	%xmm1,  0 * SIZE(Y1)

  2701. 	movhpd	%xmm1,  1 * SIZE(Y1)

  2702. 	addq	INCY, Y1

  2703. 

  2704. 	addq	$4 * SIZE, BUFFER

  2705. 	ALIGN_3

  2706. 

  2707. .L996:

  2708. 	testq	$1, M

  2709. 	jle	.L999

  2710. 

  2711.  	movsd	 0 * SIZE(Y), %xmm0

  2712. 	movhpd	 1 * SIZE(Y), %xmm0

  2713. 

  2714. 	addpd	 0 * SIZE(BUFFER), %xmm0

  2715. 

  2716. 	movlpd	%xmm0,  0 * SIZE(Y1)

  2717. 	movhpd	%xmm0,  1 * SIZE(Y1)

  2718. 	ALIGN_3

  2719. 

  2720. .L999:

  2721. 	movq	M, I

  2722. 	salq	$ZBASE_SHIFT,I

  2723. 	addq	I,AA

  2724. 	jmp	.L0t

  2725. .L999x:

  2726. 

  2727. 	movq	  0(%rsp), %rbx

  2728. 	movq	  8(%rsp), %rbp

  2729. 	movq	 16(%rsp), %r12

  2730. 	movq	 24(%rsp), %r13

  2731. 	movq	 32(%rsp), %r14

  2732. 	movq	 40(%rsp), %r15

  2733. 

  2734. #ifdef WINDOWS_ABI

  2735. 	movq	 48(%rsp), %rdi

  2736. 	movq	 56(%rsp), %rsi

  2737. 	movups	 64(%rsp), %xmm6

  2738. 	movups	 80(%rsp), %xmm7

  2739. 	movups	 96(%rsp), %xmm8

  2740. 	movups	112(%rsp), %xmm9

  2741. 	movups	128(%rsp), %xmm10

  2742. 	movups	144(%rsp), %xmm11

  2743. 	movups	160(%rsp), %xmm12

  2744. 	movups	176(%rsp), %xmm13

  2745. 	movups	192(%rsp), %xmm14

  2746. 	movups	208(%rsp), %xmm15

  2747. #endif

  2748. 

  2749. 	addq	$STACKSIZE, %rsp

  2750. 	ret

  2751. 

  2752. 	EPILOGUE