OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 925d4e1dc6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '925d4e1dc6'
${ noResults }
OpenBLAS/kernel/power/zgemv_n_ppc440.S

1387 lines
29 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef linux

  43. #ifndef __64BIT__

  44. #define M	r3

  45. #define	N	r4

  46. #define A	r6

  47. #define LDA	r7

  48. #define X	r8

  49. #define	INCX	r9

  50. #define	Y	r10

  51. #define	INCY	r5

  52. #else

  53. #define M	r3

  54. #define	N	r4

  55. #define A	r8

  56. #define LDA	r9

  57. #define X	r10

  58. #define	INCX	r5

  59. #define	Y	r6

  60. #define	INCY	r7

  61. #endif

  62. #endif

  63. 

  64. #if defined(_AIX) || defined(__APPLE__)

  65. #if !defined(__64BIT__) && defined(DOUBLE)

  66. #define M	r3

  67. #define	N	r4

  68. #define A	r10

  69. #define LDA	r5

  70. #define X	r6

  71. #define	INCX	r7

  72. #define	Y	r8

  73. #define	INCY	r9

  74. #else

  75. #define M	r3

  76. #define	N	r4

  77. #define A	r8

  78. #define LDA	r9

  79. #define X	r10

  80. #define	INCX	r5

  81. #define	Y	r6

  82. #define	INCY	r7

  83. #endif

  84. #endif

  85. 

  86. #define I	r11

  87. #define	J	r12

  88. 

  89. #define AO1	r14

  90. #define AO2	r15

  91. #define AO3	r16

  92. #define AO4	r17

  93. 

  94. #define	Y1	r18

  95. #define Y2	r19

  96. #define	PREA	r20

  97. #define YY	r21

  98. #define BUFFER	r22

  99. 

  100. #define y01 f0

  101. #define y02 f1

  102. #define y03 f2

  103. #define y04 f3

  104. #define y05 f4

  105. #define y06 f5

  106. #define y07 f6

  107. #define y08 f7

  108. #define y09 f8

  109. #define y10 f9

  110. #define y11 f10

  111. #define y12 f11

  112. #define y13 f12

  113. #define y14 f13

  114. #define y15 f14

  115. #define y16 f15

  116. 

  117. #define alpha1r f16

  118. #define alpha1i f17

  119. #define alpha2r f18

  120. #define alpha2i f19

  121. #define alpha3r f20

  122. #define alpha3i f21

  123. #define alpha4r f22

  124. #define alpha4i f23

  125. 

  126. #define a1     f24

  127. #define a2     f25

  128. #define a3     f26

  129. #define a4     f27

  130. #define a5     f28

  131. #define a6     f29

  132. #define a7     f30

  133. #define a8     f31

  134. 

  135. #define alpha_r f14

  136. #define alpha_i f15

  137. 

  138. #if defined(PPCG4)

  139. #define PREFETCHSIZE_A  (3 * 4)

  140. #endif

  141. 

  142. #if defined(POWER6)

  143. #define PREFETCHSIZE_A  (3 * 4)

  144. #endif

  145. 

  146. #ifndef XCONJ

  147. #define FMADDR FMADD

  148. #define FMSUBR FNMSUB

  149. #else

  150. #define FMADDR FNMSUB

  151. #define FMSUBR FMADD

  152. #endif

  153. 

  154. #ifndef  CONJ

  155. #define FMADDX FMADD

  156. #define FMSUBX FNMSUB

  157. #else

  158. #define FMADDX FNMSUB

  159. #define FMSUBX FMADD

  160. #endif

  161. 

  162. #ifndef NEEDPARAM

  163. 

  164. #ifndef __64BIT__

  165. #define STACKSIZE 232

  166. #define ALPHA_R	  208(SP)

  167. #define ALPHA_I	  216(SP)

  168. #define	FZERO	  224(SP)

  169. #else

  170. #define STACKSIZE 280

  171. #define ALPHA_R   256(SP)

  172. #define	ALPHA_I	  264(SP)

  173. #define	FZERO	  272(SP)

  174. #endif

  175. 

  176. 	PROLOGUE

  177. 	PROFCODE

  178. 

  179. 	addi	SP,   SP, -STACKSIZE

  180. 	li	r0,   0

  181. 

  182. 	stfd	f14,     0(SP)

  183. 	stfd	f15,     8(SP)

  184. 	stfd	f16,    16(SP)

  185. 	stfd	f17,    24(SP)

  186. 	stfd	f18,    32(SP)

  187. 	stfd	f19,    40(SP)

  188. 	stfd	f20,    48(SP)

  189. 	stfd	f21,    56(SP)

  190. 	stfd	f22,    64(SP)

  191. 	stfd	f23,    72(SP)

  192. 	stfd	f24,    80(SP)

  193. 	stfd	f25,    88(SP)

  194. 	stfd	f26,    96(SP)

  195. 	stfd	f27,   104(SP)

  196. 	stfd	f28,   112(SP)

  197. 	stfd	f29,   120(SP)

  198. 	stfd	f30,   128(SP)

  199. 	stfd	f31,   136(SP)

  200. 

  201. #ifdef __64BIT__

  202. 	std	r0,    FZERO

  203. 	std	r14,   144(SP)

  204. 	std	r15,   152(SP)

  205. 	std	r16,   160(SP)

  206. 	std	r17,   168(SP)

  207. 	std	r18,   176(SP)

  208. 	std	r19,   184(SP)

  209. 	std	r20,   192(SP)

  210. 	std	r21,   200(SP)

  211. 	std	r22,   208(SP)

  212. #else

  213. 	stw	r0,    0 + FZERO

  214. 	stw	r0,    4 + FZERO

  215. 	stw	r14,   144(SP)

  216. 	stw	r15,   148(SP)

  217. 	stw	r16,   152(SP)

  218. 	stw	r17,   156(SP)

  219. 	stw	r18,   160(SP)

  220. 	stw	r19,   164(SP)

  221. 	stw	r20,   168(SP)

  222. 	stw	r21,   172(SP)

  223. 	stw	r22,   176(SP)

  224. #endif

  225. 

  226. #ifdef linux

  227. #ifndef __64BIT__

  228. 	lwz	INCY,	 FRAMESLOT(0) + STACKSIZE(SP)

  229. 	lwz	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  230. #else

  231. 	ld	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  232. 	ld	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  233. 	ld	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  234. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  235. #endif

  236. #endif

  237. 

  238. #if defined(_AIX) || defined(__APPLE__)

  239. #ifndef __64BIT__

  240. #ifdef DOUBLE

  241. 	lwz	LDA,     FRAMESLOT(0) + STACKSIZE(SP)

  242. 	lwz	X,       FRAMESLOT(1) + STACKSIZE(SP)

  243. 	lwz	INCX,    FRAMESLOT(2) + STACKSIZE(SP)

  244. 	lwz	Y,       FRAMESLOT(3) + STACKSIZE(SP)

  245. 	lwz	INCY,    FRAMESLOT(4) + STACKSIZE(SP)

  246. 	lwz	BUFFER,  FRAMESLOT(5) + STACKSIZE(SP)

  247. #else

  248. 	lwz	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  249. 	lwz	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  250. 	lwz	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  251. 	lwz	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  252. #endif

  253. #else

  254. 	ld	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  255. 	ld	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  256. 	ld	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  257. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  258. #endif

  259. #endif

  260. 

  261. 	stfd	f1, ALPHA_R

  262. 	stfd	f2, ALPHA_I

  263. 

  264. 	slwi	LDA,  LDA,  ZBASE_SHIFT

  265. 	slwi	INCX, INCX, ZBASE_SHIFT

  266. 	slwi	INCY, INCY, ZBASE_SHIFT

  267. 

  268. 	addi	INCX, INCX, -SIZE

  269. 	addi	INCY, INCY, -SIZE

  270. 	addi	A, A, -SIZE

  271. 

  272. 	cmpwi	cr0, M, 0

  273. 	ble-	LL(999)

  274. 

  275. 	sub	X, X, INCX

  276. 	cmpwi	cr0, N, 0

  277. 	sub	Y, Y, INCY

  278. 	ble-	LL(999)

  279. 

  280. 	li	PREA, PREFETCHSIZE_A * SIZE

  281. 

  282. 	mr	YY, Y

  283. 	lfd	f0, FZERO

  284. 

  285. 	cmpi	cr0, 0, INCY, SIZE

  286. 	beq	LL(10)

  287. 

  288. 	addi	YY, BUFFER, -SIZE

  289. 	addi	Y1, BUFFER, -SIZE

  290. 

  291. 	addi	r0, M,  3

  292. 	srawi.	r0, r0, 2

  293. 	mtspr	CTR, r0

  294. 	.align 4

  295. 

  296. LL(02):

  297. 	STFDU	f0, 1 * SIZE(Y1)

  298. 	STFDU	f0, 1 * SIZE(Y1)

  299. 	STFDU	f0, 1 * SIZE(Y1)

  300. 	STFDU	f0, 1 * SIZE(Y1)

  301. 	STFDU	f0, 1 * SIZE(Y1)

  302. 	STFDU	f0, 1 * SIZE(Y1)

  303. 	STFDU	f0, 1 * SIZE(Y1)

  304. 	STFDU	f0, 1 * SIZE(Y1)

  305. 	bdnz	LL(02)

  306. 	.align 4

  307. 

  308. LL(10):

  309. 	srawi.	J, N, 2

  310. 	ble	LL(20)

  311. 	.align 4

  312. 

  313. LL(11):

  314. 	lfd	alpha_r, ALPHA_R

  315. 	lfd	alpha_i, ALPHA_I

  316. 

  317. 	LFDUX	a1, X, INCX

  318. 	LFDU	a2, 1 * SIZE(X)

  319. 	LFDUX	a3, X, INCX

  320. 	LFDU	a4, 1 * SIZE(X)

  321. 	LFDUX	a5, X, INCX

  322. 	LFDU	a6, 1 * SIZE(X)

  323. 	LFDUX	a7, X, INCX

  324. 	LFDU	a8, 1 * SIZE(X)

  325. 

  326. 	FMUL	alpha1r, alpha_r, a1

  327. 	FMUL	alpha1i, alpha_i, a1

  328. 	FMUL	alpha2r, alpha_r, a3

  329. 	FMUL	alpha2i, alpha_i, a3

  330. 

  331. 	FMUL	alpha3r, alpha_r, a5

  332. 	mr	Y1, YY

  333. 	FMUL	alpha3i, alpha_i, a5

  334. 	mr	Y2, YY

  335. 	FMUL	alpha4r, alpha_r, a7

  336. 	mr	AO1, A

  337. 	FMUL	alpha4i, alpha_i, a7

  338. 	add	AO2, A,   LDA

  339. 

  340. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  341. 	add	AO3, AO2, LDA

  342. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  343. 	add	AO4, AO3, LDA

  344. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  345. 	add	A,   AO4, LDA

  346. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  347. 

  348. 	FMSUBR	alpha3r, alpha_i, a6, alpha3r

  349. 	srawi.	r0,  M, 2

  350. 	FMADDR	alpha3i, alpha_r, a6, alpha3i

  351. 	FMSUBR	alpha4r, alpha_i, a8, alpha4r

  352. 	mtspr	CTR, r0

  353. 	FMADDR	alpha4i, alpha_r, a8, alpha4i

  354. 	ble	LL(15)

  355. 	.align 4

  356. 

  357. 	LFDU	a1,  1 * SIZE(AO1)

  358. 	LFDU	y01, 1 * SIZE(Y1)

  359. 	LFDU	a2,  1 * SIZE(AO1)

  360. 	LFDU	y02, 1 * SIZE(Y1)

  361. 	LFDU	a3,  1 * SIZE(AO1)

  362. 	LFDU	y03, 1 * SIZE(Y1)

  363. 	LFDU	a4,  1 * SIZE(AO1)

  364. 	LFDU	y04, 1 * SIZE(Y1)

  365. 

  366. 	LFDU	a5,  1 * SIZE(AO1)

  367. 	LFDU	y05, 1 * SIZE(Y1)

  368. 	LFDU	a6,  1 * SIZE(AO1)

  369. 	LFDU	y06, 1 * SIZE(Y1)

  370. 	LFDU	a7,  1 * SIZE(AO1)

  371. 	LFDU	y07, 1 * SIZE(Y1)

  372. 	LFDU	a8,  1 * SIZE(AO1)

  373. 	LFDU	y08, 1 * SIZE(Y1)

  374. 

  375. 	FMADD	y09, alpha1r, a1, y01

  376. 	FMADD	y10, alpha1i, a1, y02

  377. 	FMADD	y11, alpha1r, a3, y03

  378. 	FMADD	y12, alpha1i, a3, y04

  379. 

  380. 	FMADD	y13, alpha1r, a5, y05

  381. 	FMADD	y14, alpha1i, a5, y06

  382. 	FMADD	y15, alpha1r, a7, y07

  383. 	FMADD	y16, alpha1i, a7, y08

  384. 

  385. 	bdz	LL(13)

  386. 	.align 4

  387. 

  388. LL(12):

  389. 	FMSUBX	y09, alpha1i, a2, y09

  390. 	LFDU	a1,  1 * SIZE(AO2)

  391. 	FMADDX	y10, alpha1r, a2, y10

  392. 	LFDU	a2,  1 * SIZE(AO2)

  393. 	FMSUBX	y11, alpha1i, a4, y11

  394. 	LFDU	a3,  1 * SIZE(AO2)

  395. 	FMADDX	y12, alpha1r, a4, y12

  396. 	LFDU	a4,  1 * SIZE(AO2)

  397. #ifdef PPCG4

  398. 	dcbt	AO2, PREA

  399. #endif

  400. 

  401. 	FMSUBX	y13, alpha1i, a6, y13

  402. 	LFDU	a5,  1 * SIZE(AO2)

  403. 	FMADDX	y14, alpha1r, a6, y14

  404. 	LFDU	a6,  1 * SIZE(AO2)

  405. 	FMSUBX	y15, alpha1i, a8, y15

  406. 	LFDU	a7,  1 * SIZE(AO2)

  407. 	FMADDX	y16, alpha1r, a8, y16

  408. 	LFDU	a8,  1 * SIZE(AO2)

  409. #if defined(PPCG4) && defined(DOUBLE)

  410. 	dcbt	AO2, PREA

  411. #endif

  412. 

  413. 	FMADD	y09, alpha2r, a1, y09

  414. 	LFDU	y01, 1 * SIZE(Y1)

  415. 	FMADD	y10, alpha2i, a1, y10

  416. 	LFDU	y02, 1 * SIZE(Y1)

  417. 	FMADD	y11, alpha2r, a3, y11

  418. 	LFDU	y03, 1 * SIZE(Y1)

  419. 	FMADD	y12, alpha2i, a3, y12

  420. 	LFDU	y04, 1 * SIZE(Y1)

  421. 

  422. #ifdef PPCG4

  423. 	dcbtst	Y1, PREA

  424. #endif

  425. 

  426. 	FMADD	y13, alpha2r, a5, y13

  427. 	FMADD	y14, alpha2i, a5, y14

  428. 	FMADD	y15, alpha2r, a7, y15

  429. 	FMADD	y16, alpha2i, a7, y16

  430. 

  431. 	FMSUBX	y09, alpha2i, a2, y09

  432. 	LFDU	a1,  1 * SIZE(AO3)

  433. 	FMADDX	y10, alpha2r, a2, y10

  434. 	LFDU	a2,  1 * SIZE(AO3)

  435. 	FMSUBX	y11, alpha2i, a4, y11

  436. 	LFDU	a3,  1 * SIZE(AO3)

  437. 	FMADDX	y12, alpha2r, a4, y12

  438. 	LFDU	a4,  1 * SIZE(AO3)

  439. 

  440. #ifdef PPCG4

  441. 	dcbt	AO3, PREA

  442. #endif

  443. 

  444. 	FMSUBX	y13, alpha2i, a6, y13

  445. 	LFDU	a5,  1 * SIZE(AO3)

  446. 	FMADDX	y14, alpha2r, a6, y14

  447. 	LFDU	a6,  1 * SIZE(AO3)

  448. 	FMSUBX	y15, alpha2i, a8, y15

  449. 	LFDU	a7,  1 * SIZE(AO3)

  450. 	FMADDX	y16, alpha2r, a8, y16

  451. 	LFDU	a8,  1 * SIZE(AO3)

  452. 

  453. #if defined(PPCG4) && defined(DOUBLE)

  454. 	dcbt	AO3, PREA

  455. #endif

  456. 

  457. 	FMADD	y09, alpha3r, a1, y09

  458. 	LFDU	y05, 1 * SIZE(Y1)

  459. 	FMADD	y10, alpha3i, a1, y10

  460. 	LFDU	y06, 1 * SIZE(Y1)

  461. 	FMADD	y11, alpha3r, a3, y11

  462. 	LFDU	y07, 1 * SIZE(Y1)

  463. 	FMADD	y12, alpha3i, a3, y12

  464. 	LFDU	y08, 1 * SIZE(Y1)

  465. 

  466. #if defined(PPCG4) && defined(DOUBLE)

  467. 	dcbtst	Y1, PREA

  468. #endif

  469. 

  470. 	FMADD	y13, alpha3r, a5, y13

  471. 	FMADD	y14, alpha3i, a5, y14

  472. 	FMADD	y15, alpha3r, a7, y15

  473. 	FMADD	y16, alpha3i, a7, y16

  474. 

  475. 	FMSUBX	y09, alpha3i, a2, y09

  476. 	LFDU	a1,  1 * SIZE(AO4)

  477. 	FMADDX	y10, alpha3r, a2, y10

  478. 	LFDU	a2,  1 * SIZE(AO4)

  479. 	FMSUBX	y11, alpha3i, a4, y11

  480. 	LFDU	a3,  1 * SIZE(AO4)

  481. 	FMADDX	y12, alpha3r, a4, y12

  482. 	LFDU	a4,  1 * SIZE(AO4)

  483. 

  484. #ifdef PPCG4

  485. 	dcbt	AO4, PREA

  486. #endif

  487. 

  488. 	FMSUBX	y13, alpha3i, a6, y13

  489. 	LFDU	a5,  1 * SIZE(AO4)

  490. 	FMADDX	y14, alpha3r, a6, y14

  491. 	LFDU	a6,  1 * SIZE(AO4)

  492. 	FMSUBX	y15, alpha3i, a8, y15

  493. 	LFDU	a7,  1 * SIZE(AO4)

  494. 	FMADDX	y16, alpha3r, a8, y16

  495. 	LFDU	a8,  1 * SIZE(AO4)

  496. 

  497. #if defined(PPCG4) && defined(DOUBLE)

  498. 	dcbt	AO4, PREA

  499. #endif

  500. 

  501. 	FMADD	y09, alpha4r, a1, y09

  502. 	FMADD	y10, alpha4i, a1, y10

  503. 	FMADD	y11, alpha4r, a3, y11

  504. 	FMADD	y12, alpha4i, a3, y12

  505. 

  506. 	FMADD	y13, alpha4r, a5, y13

  507. 	FMADD	y14, alpha4i, a5, y14

  508. 	FMADD	y15, alpha4r, a7, y15

  509. 	FMADD	y16, alpha4i, a7, y16

  510. 

  511. 	FMSUBX	y09, alpha4i, a2, y09

  512. 	LFDU	a1,  1 * SIZE(AO1)

  513. 	FMADDX	y10, alpha4r, a2, y10

  514. 	LFDU	a2,  1 * SIZE(AO1)

  515. 	FMSUBX	y11, alpha4i, a4, y11

  516. 	LFDU	a3,  1 * SIZE(AO1)

  517. 	FMADDX	y12, alpha4r, a4, y12

  518. 	LFDU	a4,  1 * SIZE(AO1)

  519. 

  520. #ifdef PPCG4

  521. 	dcbt	AO1, PREA

  522. #endif

  523. 

  524. 	FMSUBX	y13, alpha4i, a6, y13

  525. 	LFDU	a5,  1 * SIZE(AO1)

  526. 	FMADDX	y14, alpha4r, a6, y14

  527. 	LFDU	a6,  1 * SIZE(AO1)

  528. 	FMSUBX	y15, alpha4i, a8, y15

  529. 	LFDU	a7,  1 * SIZE(AO1)

  530. 	FMADDX	y16, alpha4r, a8, y16

  531. 	LFDU	a8,  1 * SIZE(AO1)

  532. 

  533. #if defined(PPCG4) && defined(DOUBLE)

  534. 	dcbt	AO1, PREA

  535. #endif

  536. 

  537. 	STFDU	y09, 1 * SIZE(Y2)

  538. 	FMADD	y09, alpha1r, a1, y01

  539. 	STFDU	y10, 1 * SIZE(Y2)

  540. 	FMADD	y10, alpha1i, a1, y02

  541. 	STFDU	y11, 1 * SIZE(Y2)

  542. 	FMADD	y11, alpha1r, a3, y03

  543. 	STFDU	y12, 1 * SIZE(Y2)

  544. 	FMADD	y12, alpha1i, a3, y04

  545. 

  546. 	STFDU	y13, 1 * SIZE(Y2)

  547. 	FMADD	y13, alpha1r, a5, y05

  548. 	STFDU	y14, 1 * SIZE(Y2)

  549. 	FMADD	y14, alpha1i, a5, y06

  550. 	STFDU	y15, 1 * SIZE(Y2)

  551. 	FMADD	y15, alpha1r, a7, y07

  552. 	STFDU	y16, 1 * SIZE(Y2)

  553. 	FMADD	y16, alpha1i, a7, y08

  554. 	bdnz	LL(12)

  555. 	.align 4

  556. 

  557. LL(13):

  558. 	FMSUBX	y09, alpha1i, a2, y09

  559. 	LFDU	a1,  1 * SIZE(AO2)

  560. 	FMADDX	y10, alpha1r, a2, y10

  561. 	LFDU	a2,  1 * SIZE(AO2)

  562. 	FMSUBX	y11, alpha1i, a4, y11

  563. 	LFDU	a3,  1 * SIZE(AO2)

  564. 	FMADDX	y12, alpha1r, a4, y12

  565. 	LFDU	a4,  1 * SIZE(AO2)

  566. 

  567. 	FMSUBX	y13, alpha1i, a6, y13

  568. 	LFDU	a5,  1 * SIZE(AO2)

  569. 	FMADDX	y14, alpha1r, a6, y14

  570. 	LFDU	a6,  1 * SIZE(AO2)

  571. 	FMSUBX	y15, alpha1i, a8, y15

  572. 	LFDU	a7,  1 * SIZE(AO2)

  573. 	FMADDX	y16, alpha1r, a8, y16

  574. 	LFDU	a8,  1 * SIZE(AO2)

  575. 

  576. 	FMADD	y09, alpha2r, a1, y09

  577. 	FMADD	y10, alpha2i, a1, y10

  578. 	FMADD	y11, alpha2r, a3, y11

  579. 	FMADD	y12, alpha2i, a3, y12

  580. 

  581. 	FMADD	y13, alpha2r, a5, y13

  582. 	FMADD	y14, alpha2i, a5, y14

  583. 	FMADD	y15, alpha2r, a7, y15

  584. 	FMADD	y16, alpha2i, a7, y16

  585. 

  586. 	FMSUBX	y09, alpha2i, a2, y09

  587. 	LFDU	a1,  1 * SIZE(AO3)

  588. 	FMADDX	y10, alpha2r, a2, y10

  589. 	LFDU	a2,  1 * SIZE(AO3)

  590. 	FMSUBX	y11, alpha2i, a4, y11

  591. 	LFDU	a3,  1 * SIZE(AO3)

  592. 	FMADDX	y12, alpha2r, a4, y12

  593. 	LFDU	a4,  1 * SIZE(AO3)

  594. 

  595. 	FMSUBX	y13, alpha2i, a6, y13

  596. 	LFDU	a5,  1 * SIZE(AO3)

  597. 	FMADDX	y14, alpha2r, a6, y14

  598. 	LFDU	a6,  1 * SIZE(AO3)

  599. 	FMSUBX	y15, alpha2i, a8, y15

  600. 	LFDU	a7,  1 * SIZE(AO3)

  601. 	FMADDX	y16, alpha2r, a8, y16

  602. 	LFDU	a8,  1 * SIZE(AO3)

  603. 

  604. 	FMADD	y09, alpha3r, a1, y09

  605. 	FMADD	y10, alpha3i, a1, y10

  606. 	FMADD	y11, alpha3r, a3, y11

  607. 	FMADD	y12, alpha3i, a3, y12

  608. 

  609. 	FMADD	y13, alpha3r, a5, y13

  610. 	FMADD	y14, alpha3i, a5, y14

  611. 	FMADD	y15, alpha3r, a7, y15

  612. 	FMADD	y16, alpha3i, a7, y16

  613. 

  614. 	FMSUBX	y09, alpha3i, a2, y09

  615. 	LFDU	a1,  1 * SIZE(AO4)

  616. 	FMADDX	y10, alpha3r, a2, y10

  617. 	LFDU	a2,  1 * SIZE(AO4)

  618. 	FMSUBX	y11, alpha3i, a4, y11

  619. 	LFDU	a3,  1 * SIZE(AO4)

  620. 	FMADDX	y12, alpha3r, a4, y12

  621. 	LFDU	a4,  1 * SIZE(AO4)

  622. 

  623. 	FMSUBX	y13, alpha3i, a6, y13

  624. 	LFDU	a5,  1 * SIZE(AO4)

  625. 	FMADDX	y14, alpha3r, a6, y14

  626. 	LFDU	a6,  1 * SIZE(AO4)

  627. 	FMSUBX	y15, alpha3i, a8, y15

  628. 	LFDU	a7,  1 * SIZE(AO4)

  629. 	FMADDX	y16, alpha3r, a8, y16

  630. 	LFDU	a8,  1 * SIZE(AO4)

  631. 

  632. 	FMADD	y09, alpha4r, a1, y09

  633. 	FMADD	y10, alpha4i, a1, y10

  634. 	FMADD	y11, alpha4r, a3, y11

  635. 	FMADD	y12, alpha4i, a3, y12

  636. 

  637. 	FMADD	y13, alpha4r, a5, y13

  638. 	FMADD	y14, alpha4i, a5, y14

  639. 	FMADD	y15, alpha4r, a7, y15

  640. 	FMADD	y16, alpha4i, a7, y16

  641. 

  642. 	FMSUBX	y09, alpha4i, a2, y09

  643. 	FMADDX	y10, alpha4r, a2, y10

  644. 	FMSUBX	y11, alpha4i, a4, y11

  645. 	FMADDX	y12, alpha4r, a4, y12

  646. 

  647. 	FMSUBX	y13, alpha4i, a6, y13

  648. 	STFDU	y09, 1 * SIZE(Y2)

  649. 	FMADDX	y14, alpha4r, a6, y14

  650. 	STFDU	y10, 1 * SIZE(Y2)

  651. 	FMSUBX	y15, alpha4i, a8, y15

  652. 	STFDU	y11, 1 * SIZE(Y2)

  653. 	FMADDX	y16, alpha4r, a8, y16

  654. 	STFDU	y12, 1 * SIZE(Y2)

  655. 

  656. 	STFDU	y13, 1 * SIZE(Y2)

  657. 	STFDU	y14, 1 * SIZE(Y2)

  658. 	STFDU	y15, 1 * SIZE(Y2)

  659. 	STFDU	y16, 1 * SIZE(Y2)

  660. 	.align 4

  661. 

  662. LL(15):

  663. 	andi.	r0, M, 2

  664. 	ble	LL(17)

  665. 

  666. 	LFDU	a1,  1 * SIZE(AO1)

  667. 	LFDU	y01, 1 * SIZE(Y1)

  668. 	LFDU	a2,  1 * SIZE(AO1)

  669. 	LFDU	y02, 1 * SIZE(Y1)

  670. 	LFDU	a3,  1 * SIZE(AO1)

  671. 	LFDU	y03, 1 * SIZE(Y1)

  672. 	LFDU	a4,  1 * SIZE(AO1)

  673. 	LFDU	y04, 1 * SIZE(Y1)

  674. 

  675. 	FMADD	y01, alpha1r, a1, y01

  676. 	LFDU	a5,  1 * SIZE(AO2)

  677. 	FMADD	y02, alpha1i, a1, y02

  678. 	LFDU	a6,  1 * SIZE(AO2)

  679. 	FMADD	y03, alpha1r, a3, y03

  680. 	LFDU	a7,  1 * SIZE(AO2)

  681. 	FMADD	y04, alpha1i, a3, y04

  682. 	LFDU	a8,  1 * SIZE(AO2)

  683. 

  684. 	FMSUBX	y01, alpha1i, a2, y01

  685. 	LFDU	a1,  1 * SIZE(AO3)

  686. 	FMADDX	y02, alpha1r, a2, y02

  687. 	LFDU	a2,  1 * SIZE(AO3)

  688. 	FMSUBX	y03, alpha1i, a4, y03

  689. 	LFDU	a3,  1 * SIZE(AO3)

  690. 	FMADDX	y04, alpha1r, a4, y04

  691. 	LFDU	a4,  1 * SIZE(AO3)

  692. 

  693. 	FMADD	y01, alpha2r, a5, y01

  694. 	FMADD	y02, alpha2i, a5, y02

  695. 	FMADD	y03, alpha2r, a7, y03

  696. 	FMADD	y04, alpha2i, a7, y04

  697. 

  698. 	FMSUBX	y01, alpha2i, a6, y01

  699. 	LFDU	a5,  1 * SIZE(AO4)

  700. 	FMADDX	y02, alpha2r, a6, y02

  701. 	LFDU	a6,  1 * SIZE(AO4)

  702. 	FMSUBX	y03, alpha2i, a8, y03

  703. 	LFDU	a7,  1 * SIZE(AO4)

  704. 	FMADDX	y04, alpha2r, a8, y04

  705. 	LFDU	a8,  1 * SIZE(AO4)

  706. 

  707. 	FMADD	y01, alpha3r, a1, y01

  708. 	FMADD	y02, alpha3i, a1, y02

  709. 	FMADD	y03, alpha3r, a3, y03

  710. 	FMADD	y04, alpha3i, a3, y04

  711. 

  712. 	FMSUBX	y01, alpha3i, a2, y01

  713. 	FMADDX	y02, alpha3r, a2, y02

  714. 	FMSUBX	y03, alpha3i, a4, y03

  715. 	FMADDX	y04, alpha3r, a4, y04

  716. 

  717. 	FMADD	y01, alpha4r, a5, y01

  718. 	FMADD	y02, alpha4i, a5, y02

  719. 	FMADD	y03, alpha4r, a7, y03

  720. 	FMADD	y04, alpha4i, a7, y04

  721. 

  722. 	FMSUBX	y01, alpha4i, a6, y01

  723. 	FMADDX	y02, alpha4r, a6, y02

  724. 	FMSUBX	y03, alpha4i, a8, y03

  725. 	FMADDX	y04, alpha4r, a8, y04

  726. 

  727. 	STFDU	y01, 1 * SIZE(Y2)

  728. 	STFDU	y02, 1 * SIZE(Y2)

  729. 	STFDU	y03, 1 * SIZE(Y2)

  730. 	STFDU	y04, 1 * SIZE(Y2)

  731. 	.align 4

  732. 

  733. LL(17):

  734. 	andi.	r0, M, 1

  735. 	ble	LL(19)

  736. 

  737. 	LFDU	y01, 1 * SIZE(Y1)

  738. 	LFDU	y02, 1 * SIZE(Y1)

  739. 

  740. 	LFDU	a1,  1 * SIZE(AO1)

  741. 	LFDU	a2,  1 * SIZE(AO1)

  742. 	LFDU	a3,  1 * SIZE(AO2)

  743. 	LFDU	a4,  1 * SIZE(AO2)

  744. 

  745. 	FMADD	y01, alpha1r, a1, y01

  746. 	LFDU	a5,  1 * SIZE(AO3)

  747. 	FMADD	y02, alpha1i, a1, y02

  748. 	LFDU	a6,  1 * SIZE(AO3)

  749. 	FMSUBX	y01, alpha1i, a2, y01

  750. 	LFDU	a7,  1 * SIZE(AO4)

  751. 	FMADDX	y02, alpha1r, a2, y02

  752. 	LFDU	a8,  1 * SIZE(AO4)

  753. 

  754. 	FMADD	y01, alpha2r, a3, y01

  755. 	FMADD	y02, alpha2i, a3, y02

  756. 	FMSUBX	y01, alpha2i, a4, y01

  757. 	FMADDX	y02, alpha2r, a4, y02

  758. 

  759. 	FMADD	y01, alpha3r, a5, y01

  760. 	FMADD	y02, alpha3i, a5, y02

  761. 	FMSUBX	y01, alpha3i, a6, y01

  762. 	FMADDX	y02, alpha3r, a6, y02

  763. 

  764. 	FMADD	y01, alpha4r, a7, y01

  765. 	FMADD	y02, alpha4i, a7, y02

  766. 	FMSUBX	y01, alpha4i, a8, y01

  767. 	FMADDX	y02, alpha4r, a8, y02

  768. 

  769. 	STFDU	y01, 1 * SIZE(Y2)

  770. 	STFDU	y02, 1 * SIZE(Y2)

  771. 	.align 4

  772. 

  773. LL(19):

  774. 	addi	J, J, -1

  775. 	cmpi	cr0, 0, J, 0

  776. 	bgt	LL(11)

  777. 	.align 4

  778. 

  779. LL(20):

  780. 	andi.	J, N, 2

  781. 	ble	LL(30)

  782. 

  783. 	lfd	alpha_r, ALPHA_R

  784. 	lfd	alpha_i, ALPHA_I

  785. 

  786. 	LFDUX	a1, X, INCX

  787. 	LFDU	a2, 1 * SIZE(X)

  788. 	LFDUX	a3, X, INCX

  789. 	LFDU	a4, 1 * SIZE(X)

  790. 

  791. 	FMUL	alpha1r, alpha_r, a1

  792. 	mr	Y1, YY

  793. 	FMUL	alpha1i, alpha_i, a1

  794. 	mr	Y2, YY

  795. 	FMUL	alpha2r, alpha_r, a3

  796. 	mr	AO1, A

  797. 	FMUL	alpha2i, alpha_i, a3

  798. 	add	AO2, A,   LDA

  799. 

  800. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  801. 	add	A,   AO2, LDA

  802. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  803. 	srawi.	r0,  M, 2

  804. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  805. 	mtspr	CTR, r0

  806. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  807. 	ble	LL(25)

  808. 	.align 4

  809. 

  810. 	LFDU	a1,  1 * SIZE(AO1)

  811. 	LFDU	y01, 1 * SIZE(Y1)

  812. 	LFDU	a2,  1 * SIZE(AO1)

  813. 	LFDU	y02, 1 * SIZE(Y1)

  814. 	LFDU	a3,  1 * SIZE(AO1)

  815. 	LFDU	y03, 1 * SIZE(Y1)

  816. 	LFDU	a4,  1 * SIZE(AO1)

  817. 	LFDU	y04, 1 * SIZE(Y1)

  818. 

  819. 	LFDU	a5,  1 * SIZE(AO1)

  820. 	LFDU	y05, 1 * SIZE(Y1)

  821. 	LFDU	a6,  1 * SIZE(AO1)

  822. 	LFDU	y06, 1 * SIZE(Y1)

  823. 	LFDU	a7,  1 * SIZE(AO1)

  824. 	LFDU	y07, 1 * SIZE(Y1)

  825. 	LFDU	a8,  1 * SIZE(AO1)

  826. 	LFDU	y08, 1 * SIZE(Y1)

  827. 

  828. 	FMADD	y09, alpha1r, a1, y01

  829. 	FMADD	y10, alpha1i, a1, y02

  830. 	FMADD	y11, alpha1r, a3, y03

  831. 	FMADD	y12, alpha1i, a3, y04

  832. 

  833. 	FMADD	y13, alpha1r, a5, y05

  834. 	FMADD	y14, alpha1i, a5, y06

  835. 	FMADD	y15, alpha1r, a7, y07

  836. 	FMADD	y16, alpha1i, a7, y08

  837. 

  838. 	bdz	LL(23)

  839. 	.align 4

  840. 

  841. LL(22):

  842. 	FMSUBX	y09, alpha1i, a2, y09

  843. 	LFDU	a1,  1 * SIZE(AO2)

  844. 	FMADDX	y10, alpha1r, a2, y10

  845. 	LFDU	a2,  1 * SIZE(AO2)

  846. 	FMSUBX	y11, alpha1i, a4, y11

  847. 	LFDU	a3,  1 * SIZE(AO2)

  848. 	FMADDX	y12, alpha1r, a4, y12

  849. 	LFDU	a4,  1 * SIZE(AO2)

  850. #ifdef PPCG4

  851. 	dcbt	AO2, PREA

  852. #endif

  853. 

  854. 	FMSUBX	y13, alpha1i, a6, y13

  855. 	LFDU	a5,  1 * SIZE(AO2)

  856. 	FMADDX	y14, alpha1r, a6, y14

  857. 	LFDU	a6,  1 * SIZE(AO2)

  858. 	FMSUBX	y15, alpha1i, a8, y15

  859. 	LFDU	a7,  1 * SIZE(AO2)

  860. 	FMADDX	y16, alpha1r, a8, y16

  861. 	LFDU	a8,  1 * SIZE(AO2)

  862. #if defined(PPCG4) && defined(DOUBLE)

  863. 	dcbt	AO2, PREA

  864. #endif

  865. 

  866. 	FMADD	y09, alpha2r, a1, y09

  867. 	LFDU	y01, 1 * SIZE(Y1)

  868. 	FMADD	y10, alpha2i, a1, y10

  869. 	LFDU	y02, 1 * SIZE(Y1)

  870. 	FMADD	y11, alpha2r, a3, y11

  871. 	LFDU	y03, 1 * SIZE(Y1)

  872. 	FMADD	y12, alpha2i, a3, y12

  873. 	LFDU	y04, 1 * SIZE(Y1)

  874. 

  875. #ifdef PPCG4

  876. 	dcbtst	Y1, PREA

  877. #endif

  878. 

  879. 	FMADD	y13, alpha2r, a5, y13

  880. 	LFDU	y05, 1 * SIZE(Y1)

  881. 	FMADD	y14, alpha2i, a5, y14

  882. 	LFDU	y06, 1 * SIZE(Y1)

  883. 	FMADD	y15, alpha2r, a7, y15

  884. 	LFDU	y07, 1 * SIZE(Y1)

  885. 	FMADD	y16, alpha2i, a7, y16

  886. 	LFDU	y08, 1 * SIZE(Y1)

  887. 

  888. #if defined(PPCG4) && defined(DOUBLE)

  889. 	dcbtst	Y1, PREA

  890. #endif

  891. 

  892. 	FMSUBX	y09, alpha2i, a2, y09

  893. 	LFDU	a1,  1 * SIZE(AO1)

  894. 	FMADDX	y10, alpha2r, a2, y10

  895. 	LFDU	a2,  1 * SIZE(AO1)

  896. 	FMSUBX	y11, alpha2i, a4, y11

  897. 	LFDU	a3,  1 * SIZE(AO1)

  898. 	FMADDX	y12, alpha2r, a4, y12

  899. 	LFDU	a4,  1 * SIZE(AO1)

  900. 

  901. #ifdef PPCG4

  902. 	dcbt	AO1, PREA

  903. #endif

  904. 

  905. 	FMSUBX	y13, alpha2i, a6, y13

  906. 	LFDU	a5,  1 * SIZE(AO1)

  907. 	FMADDX	y14, alpha2r, a6, y14

  908. 	LFDU	a6,  1 * SIZE(AO1)

  909. 	FMSUBX	y15, alpha2i, a8, y15

  910. 	LFDU	a7,  1 * SIZE(AO1)

  911. 	FMADDX	y16, alpha2r, a8, y16

  912. 	LFDU	a8,  1 * SIZE(AO1)

  913. 

  914. #if defined(PPCG4) && defined(DOUBLE)

  915. 	dcbt	AO1, PREA

  916. #endif

  917. 

  918. 	STFDU	y09, 1 * SIZE(Y2)

  919. 	FMADD	y09, alpha1r, a1, y01

  920. 	STFDU	y10, 1 * SIZE(Y2)

  921. 	FMADD	y10, alpha1i, a1, y02

  922. 	STFDU	y11, 1 * SIZE(Y2)

  923. 	FMADD	y11, alpha1r, a3, y03

  924. 	STFDU	y12, 1 * SIZE(Y2)

  925. 	FMADD	y12, alpha1i, a3, y04

  926. 

  927. 	STFDU	y13, 1 * SIZE(Y2)

  928. 	FMADD	y13, alpha1r, a5, y05

  929. 	STFDU	y14, 1 * SIZE(Y2)

  930. 	FMADD	y14, alpha1i, a5, y06

  931. 	STFDU	y15, 1 * SIZE(Y2)

  932. 	FMADD	y15, alpha1r, a7, y07

  933. 	STFDU	y16, 1 * SIZE(Y2)

  934. 	FMADD	y16, alpha1i, a7, y08

  935. 	bdnz	LL(22)

  936. 	.align 4

  937. 

  938. LL(23):

  939. 	FMSUBX	y09, alpha1i, a2, y09

  940. 	LFDU	a1,  1 * SIZE(AO2)

  941. 	FMADDX	y10, alpha1r, a2, y10

  942. 	LFDU	a2,  1 * SIZE(AO2)

  943. 	FMSUBX	y11, alpha1i, a4, y11

  944. 	LFDU	a3,  1 * SIZE(AO2)

  945. 	FMADDX	y12, alpha1r, a4, y12

  946. 	LFDU	a4,  1 * SIZE(AO2)

  947. 

  948. 	FMSUBX	y13, alpha1i, a6, y13

  949. 	LFDU	a5,  1 * SIZE(AO2)

  950. 	FMADDX	y14, alpha1r, a6, y14

  951. 	LFDU	a6,  1 * SIZE(AO2)

  952. 	FMSUBX	y15, alpha1i, a8, y15

  953. 	LFDU	a7,  1 * SIZE(AO2)

  954. 	FMADDX	y16, alpha1r, a8, y16

  955. 	LFDU	a8,  1 * SIZE(AO2)

  956. 

  957. 	FMADD	y09, alpha2r, a1, y09

  958. 	FMADD	y10, alpha2i, a1, y10

  959. 	FMADD	y11, alpha2r, a3, y11

  960. 	FMADD	y12, alpha2i, a3, y12

  961. 

  962. 	FMADD	y13, alpha2r, a5, y13

  963. 	FMADD	y14, alpha2i, a5, y14

  964. 	FMADD	y15, alpha2r, a7, y15

  965. 	FMADD	y16, alpha2i, a7, y16

  966. 

  967. 	FMSUBX	y09, alpha2i, a2, y09

  968. 	FMADDX	y10, alpha2r, a2, y10

  969. 	FMSUBX	y11, alpha2i, a4, y11

  970. 	FMADDX	y12, alpha2r, a4, y12

  971. 

  972. 	FMSUBX	y13, alpha2i, a6, y13

  973. 	STFDU	y09, 1 * SIZE(Y2)

  974. 	FMADDX	y14, alpha2r, a6, y14

  975. 	STFDU	y10, 1 * SIZE(Y2)

  976. 	FMSUBX	y15, alpha2i, a8, y15

  977. 	STFDU	y11, 1 * SIZE(Y2)

  978. 	FMADDX	y16, alpha2r, a8, y16

  979. 	STFDU	y12, 1 * SIZE(Y2)

  980. 

  981. 	STFDU	y13, 1 * SIZE(Y2)

  982. 	STFDU	y14, 1 * SIZE(Y2)

  983. 	STFDU	y15, 1 * SIZE(Y2)

  984. 	STFDU	y16, 1 * SIZE(Y2)

  985. 	.align 4

  986. 

  987. LL(25):

  988. 	andi.	r0, M, 2

  989. 	ble	LL(27)

  990. 

  991. 	LFDU	a1,  1 * SIZE(AO1)

  992. 	LFDU	y01, 1 * SIZE(Y1)

  993. 	LFDU	a2,  1 * SIZE(AO1)

  994. 	LFDU	y02, 1 * SIZE(Y1)

  995. 	LFDU	a3,  1 * SIZE(AO1)

  996. 	LFDU	y03, 1 * SIZE(Y1)

  997. 	LFDU	a4,  1 * SIZE(AO1)

  998. 	LFDU	y04, 1 * SIZE(Y1)

  999. 

  1000. 	FMADD	y01, alpha1r, a1, y01

  1001. 	LFDU	a5,  1 * SIZE(AO2)

  1002. 	FMADD	y02, alpha1i, a1, y02

  1003. 	LFDU	a6,  1 * SIZE(AO2)

  1004. 	FMADD	y03, alpha1r, a3, y03

  1005. 	LFDU	a7,  1 * SIZE(AO2)

  1006. 	FMADD	y04, alpha1i, a3, y04

  1007. 	LFDU	a8,  1 * SIZE(AO2)

  1008. 

  1009. 	FMSUBX	y01, alpha1i, a2, y01

  1010. 	FMADDX	y02, alpha1r, a2, y02

  1011. 	FMSUBX	y03, alpha1i, a4, y03

  1012. 	FMADDX	y04, alpha1r, a4, y04

  1013. 

  1014. 	FMADD	y01, alpha2r, a5, y01

  1015. 	FMADD	y02, alpha2i, a5, y02

  1016. 	FMADD	y03, alpha2r, a7, y03

  1017. 	FMADD	y04, alpha2i, a7, y04

  1018. 

  1019. 	FMSUBX	y01, alpha2i, a6, y01

  1020. 	FMADDX	y02, alpha2r, a6, y02

  1021. 	FMSUBX	y03, alpha2i, a8, y03

  1022. 	FMADDX	y04, alpha2r, a8, y04

  1023. 

  1024. 	STFDU	y01, 1 * SIZE(Y2)

  1025. 	STFDU	y02, 1 * SIZE(Y2)

  1026. 	STFDU	y03, 1 * SIZE(Y2)

  1027. 	STFDU	y04, 1 * SIZE(Y2)

  1028. 	.align 4

  1029. 

  1030. LL(27):

  1031. 	andi.	r0, M, 1

  1032. 	ble	LL(30)

  1033. 

  1034. 	LFDU	y01, 1 * SIZE(Y1)

  1035. 	LFDU	y02, 1 * SIZE(Y1)

  1036. 

  1037. 	LFDU	a1,  1 * SIZE(AO1)

  1038. 	LFDU	a2,  1 * SIZE(AO1)

  1039. 	LFDU	a3,  1 * SIZE(AO2)

  1040. 	LFDU	a4,  1 * SIZE(AO2)

  1041. 

  1042. 	FMADD	y01, alpha1r, a1, y01

  1043. 	FMADD	y02, alpha1i, a1, y02

  1044. 	FMSUBX	y01, alpha1i, a2, y01

  1045. 	FMADDX	y02, alpha1r, a2, y02

  1046. 

  1047. 	FMADD	y01, alpha2r, a3, y01

  1048. 	FMADD	y02, alpha2i, a3, y02

  1049. 	FMSUBX	y01, alpha2i, a4, y01

  1050. 	FMADDX	y02, alpha2r, a4, y02

  1051. 

  1052. 	STFDU	y01, 1 * SIZE(Y2)

  1053. 	STFDU	y02, 1 * SIZE(Y2)

  1054. 	.align 4

  1055. 

  1056. LL(30):

  1057. 	andi.	J, N, 1

  1058. 	ble	LL(990)

  1059. 	.align 4

  1060. 

  1061. 	lfd	alpha_r, ALPHA_R

  1062. 	lfd	alpha_i, ALPHA_I

  1063. 

  1064. 	LFDUX	a1, X, INCX

  1065. 	LFDU	a2, 1 * SIZE(X)

  1066. 

  1067. 	FMUL	alpha1r, alpha_r, a1

  1068. 	mr	Y1, YY

  1069. 	mr	Y2, YY

  1070. 	FMUL	alpha1i, alpha_i, a1

  1071. 	mr	AO1, A

  1072. 	add	A,   A,   LDA

  1073. 

  1074. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  1075. 	srawi.	r0,  M, 2

  1076. 	mtspr	CTR, r0

  1077. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  1078. 	ble	LL(35)

  1079. 	.align 4

  1080. 

  1081. 	LFDU	a1,  1 * SIZE(AO1)

  1082. 	LFDU	y01, 1 * SIZE(Y1)

  1083. 	LFDU	a2,  1 * SIZE(AO1)

  1084. 	LFDU	y02, 1 * SIZE(Y1)

  1085. 	LFDU	a3,  1 * SIZE(AO1)

  1086. 	LFDU	y03, 1 * SIZE(Y1)

  1087. 	LFDU	a4,  1 * SIZE(AO1)

  1088. 	LFDU	y04, 1 * SIZE(Y1)

  1089. 

  1090. 	LFDU	a5,  1 * SIZE(AO1)

  1091. 	LFDU	y05, 1 * SIZE(Y1)

  1092. 	LFDU	a6,  1 * SIZE(AO1)

  1093. 	LFDU	y06, 1 * SIZE(Y1)

  1094. 	LFDU	a7,  1 * SIZE(AO1)

  1095. 	LFDU	y07, 1 * SIZE(Y1)

  1096. 	LFDU	a8,  1 * SIZE(AO1)

  1097. 	LFDU	y08, 1 * SIZE(Y1)

  1098. 

  1099. 	FMADD	y09, alpha1r, a1, y01

  1100. 	FMADD	y10, alpha1i, a1, y02

  1101. 	FMADD	y11, alpha1r, a3, y03

  1102. 	FMADD	y12, alpha1i, a3, y04

  1103. 

  1104. 	FMADD	y13, alpha1r, a5, y05

  1105. 	FMADD	y14, alpha1i, a5, y06

  1106. 	FMADD	y15, alpha1r, a7, y07

  1107. 	FMADD	y16, alpha1i, a7, y08

  1108. 

  1109. 	bdz	LL(33)

  1110. 	.align 4

  1111. 

  1112. LL(32):

  1113. 	FMSUBX	y09, alpha1i, a2, y09

  1114. 	LFDU	a1,  1 * SIZE(AO1)

  1115. 	FMADDX	y10, alpha1r, a2, y10

  1116. 	LFDU	a2,  1 * SIZE(AO1)

  1117. 	FMSUBX	y11, alpha1i, a4, y11

  1118. 	LFDU	a3,  1 * SIZE(AO1)

  1119. 	FMADDX	y12, alpha1r, a4, y12

  1120. 	LFDU	a4,  1 * SIZE(AO1)

  1121. 

  1122. #ifdef PPCG4

  1123. 	dcbt	AO1, PREA

  1124. #endif

  1125. 

  1126. 	LFDU	y01, 1 * SIZE(Y1)

  1127. 	LFDU	y02, 1 * SIZE(Y1)

  1128. 	LFDU	y03, 1 * SIZE(Y1)

  1129. 	LFDU	y04, 1 * SIZE(Y1)

  1130. 

  1131. #ifdef PPCG4

  1132. 	dcbtst	Y1, PREA

  1133. #endif

  1134. 

  1135. 	FMSUBX	y13, alpha1i, a6, y13

  1136. 	LFDU	a5,  1 * SIZE(AO1)

  1137. 	FMADDX	y14, alpha1r, a6, y14

  1138. 	LFDU	a6,  1 * SIZE(AO1)

  1139. 	FMSUBX	y15, alpha1i, a8, y15

  1140. 	LFDU	a7,  1 * SIZE(AO1)

  1141. 	FMADDX	y16, alpha1r, a8, y16

  1142. 	LFDU	a8,  1 * SIZE(AO1)

  1143. 

  1144. #if defined(PPCG4) && defined(DOUBLE)

  1145. 	dcbt	AO1, PREA

  1146. #endif

  1147. 

  1148. 	LFDU	y05, 1 * SIZE(Y1)

  1149. 	LFDU	y06, 1 * SIZE(Y1)

  1150. 	LFDU	y07, 1 * SIZE(Y1)

  1151. 	LFDU	y08, 1 * SIZE(Y1)

  1152. 

  1153. #if defined(PPCG4) && defined(DOUBLE)

  1154. 	dcbtst	Y1, PREA

  1155. #endif

  1156. 

  1157. 	STFDU	y09, 1 * SIZE(Y2)

  1158. 	FMADD	y09, alpha1r, a1, y01

  1159. 	STFDU	y10, 1 * SIZE(Y2)

  1160. 	FMADD	y10, alpha1i, a1, y02

  1161. 	STFDU	y11, 1 * SIZE(Y2)

  1162. 	FMADD	y11, alpha1r, a3, y03

  1163. 	STFDU	y12, 1 * SIZE(Y2)

  1164. 	FMADD	y12, alpha1i, a3, y04

  1165. 

  1166. 	STFDU	y13, 1 * SIZE(Y2)

  1167. 	FMADD	y13, alpha1r, a5, y05

  1168. 	STFDU	y14, 1 * SIZE(Y2)

  1169. 	FMADD	y14, alpha1i, a5, y06

  1170. 	STFDU	y15, 1 * SIZE(Y2)

  1171. 	FMADD	y15, alpha1r, a7, y07

  1172. 	STFDU	y16, 1 * SIZE(Y2)

  1173. 	FMADD	y16, alpha1i, a7, y08

  1174. 	bdnz	LL(32)

  1175. 	.align 4

  1176. 

  1177. LL(33):

  1178. 	FMSUBX	y09, alpha1i, a2, y09

  1179. 	FMADDX	y10, alpha1r, a2, y10

  1180. 	FMSUBX	y11, alpha1i, a4, y11

  1181. 	FMADDX	y12, alpha1r, a4, y12

  1182. 

  1183. 	FMSUBX	y13, alpha1i, a6, y13

  1184. 	STFDU	y09, 1 * SIZE(Y2)

  1185. 	FMADDX	y14, alpha1r, a6, y14

  1186. 	STFDU	y10, 1 * SIZE(Y2)

  1187. 	FMSUBX	y15, alpha1i, a8, y15

  1188. 	STFDU	y11, 1 * SIZE(Y2)

  1189. 	FMADDX	y16, alpha1r, a8, y16

  1190. 	STFDU	y12, 1 * SIZE(Y2)

  1191. 

  1192. 	STFDU	y13, 1 * SIZE(Y2)

  1193. 	STFDU	y14, 1 * SIZE(Y2)

  1194. 	STFDU	y15, 1 * SIZE(Y2)

  1195. 	STFDU	y16, 1 * SIZE(Y2)

  1196. 	.align 4

  1197. 

  1198. LL(35):

  1199. 	andi.	r0, M, 2

  1200. 	ble	LL(37)

  1201. 

  1202. 	LFDU	a1,  1 * SIZE(AO1)

  1203. 	LFDU	y01, 1 * SIZE(Y1)

  1204. 	LFDU	a2,  1 * SIZE(AO1)

  1205. 	LFDU	y02, 1 * SIZE(Y1)

  1206. 	LFDU	a3,  1 * SIZE(AO1)

  1207. 	LFDU	y03, 1 * SIZE(Y1)

  1208. 	LFDU	a4,  1 * SIZE(AO1)

  1209. 	LFDU	y04, 1 * SIZE(Y1)

  1210. 

  1211. 	FMADD	y01, alpha1r, a1, y01

  1212. 	FMADD	y02, alpha1i, a1, y02

  1213. 	FMADD	y03, alpha1r, a3, y03

  1214. 	FMADD	y04, alpha1i, a3, y04

  1215. 

  1216. 	FMSUBX	y01, alpha1i, a2, y01

  1217. 	FMADDX	y02, alpha1r, a2, y02

  1218. 	FMSUBX	y03, alpha1i, a4, y03

  1219. 	FMADDX	y04, alpha1r, a4, y04

  1220. 

  1221. 	STFDU	y01, 1 * SIZE(Y2)

  1222. 	STFDU	y02, 1 * SIZE(Y2)

  1223. 	STFDU	y03, 1 * SIZE(Y2)

  1224. 	STFDU	y04, 1 * SIZE(Y2)

  1225. 	.align 4

  1226. 

  1227. LL(37):

  1228. 	andi.	r0, M, 1

  1229. 	ble	LL(990)

  1230. 

  1231. 	LFDU	y01, 1 * SIZE(Y1)

  1232. 	LFDU	a1,  1 * SIZE(AO1)

  1233. 	LFDU	y02, 1 * SIZE(Y1)

  1234. 	LFDU	a2,  1 * SIZE(AO1)

  1235. 

  1236. 	FMADD	y01, alpha1r, a1, y01

  1237. 	FMADD	y02, alpha1i, a1, y02

  1238. 	FMSUBX	y01, alpha1i, a2, y01

  1239. 	FMADDX	y02, alpha1r, a2, y02

  1240. 

  1241. 	STFDU	y01, 1 * SIZE(Y2)

  1242. 	STFDU	y02, 1 * SIZE(Y2)

  1243. 	.align 4

  1244. 

  1245. LL(990):

  1246. 	cmpi	cr0, 0, INCY, SIZE

  1247. 	beq	LL(999)

  1248. 

  1249. 	addi	YY, BUFFER, -SIZE

  1250. 	mr	Y1, Y

  1251. 

  1252. 	srawi.	r0, M, 2

  1253. 	mtspr	CTR, r0

  1254. 	ble	LL(995)

  1255. 	.align 4

  1256. 

  1257. LL(991):

  1258. 	LFDUX	f0, Y, INCY

  1259. 	LFDU	f1,  1 * SIZE(Y)

  1260. 	LFDUX	f2, Y, INCY

  1261. 	LFDU	f3,  1 * SIZE(Y)

  1262. 	LFDUX	f4, Y, INCY

  1263. 	LFDU	f5,  1 * SIZE(Y)

  1264. 	LFDUX	f6, Y, INCY

  1265. 	LFDU	f7,  1 * SIZE(Y)

  1266. 

  1267. 	LFDU	f8,   1 * SIZE(YY)

  1268. 	LFDU	f9,   1 * SIZE(YY)

  1269. 	LFDU	f10,  1 * SIZE(YY)

  1270. 	LFDU	f11,  1 * SIZE(YY)

  1271. 	LFDU	f12,  1 * SIZE(YY)

  1272. 	LFDU	f13,  1 * SIZE(YY)

  1273. 	LFDU	f14,  1 * SIZE(YY)

  1274. 	LFDU	f15,  1 * SIZE(YY)

  1275. 

  1276. 	FADD	f8,  f8,  f0

  1277. 	FADD	f9,  f9,  f1

  1278. 	FADD	f10, f10, f2

  1279. 	FADD	f11, f11, f3

  1280. 	FADD	f12, f12, f4

  1281. 	FADD	f13, f13, f5

  1282. 	FADD	f14, f14, f6

  1283. 	FADD	f15, f15, f7

  1284. 

  1285. 	STFDUX	f8,  Y1, INCY

  1286. 	STFDU	f9,  1 * SIZE(Y1)

  1287. 	STFDUX	f10, Y1, INCY

  1288. 	STFDU	f11, 1 * SIZE(Y1)

  1289. 	STFDUX	f12, Y1, INCY

  1290. 	STFDU	f13, 1 * SIZE(Y1)

  1291. 	STFDUX	f14, Y1, INCY

  1292. 	STFDU	f15, 1 * SIZE(Y1)

  1293. 	bdnz	LL(991)

  1294. 	.align 4

  1295. 

  1296. LL(995):

  1297. 	andi.	J, M, 2

  1298. 	ble	LL(996)

  1299. 

  1300. 	LFDUX	f0, Y, INCY

  1301. 	LFDU	f1, 1 * SIZE(Y)

  1302. 	LFDUX	f2, Y, INCY

  1303. 	LFDU	f3, 1 * SIZE(Y)

  1304. 

  1305. 	LFDU	f8,   1 * SIZE(YY)

  1306. 	LFDU	f9,   1 * SIZE(YY)

  1307. 	LFDU	f10,  1 * SIZE(YY)

  1308. 	LFDU	f11,  1 * SIZE(YY)

  1309. 

  1310. 	FADD	f8,  f8,  f0

  1311. 	FADD	f9,  f9,  f1

  1312. 	FADD	f10, f10, f2

  1313. 	FADD	f11, f11, f3

  1314. 

  1315. 	STFDUX	f8,  Y1, INCY

  1316. 	STFDU	f9,  1 * SIZE(Y1)

  1317. 	STFDUX	f10, Y1, INCY

  1318. 	STFDU	f11, 1 * SIZE(Y1)

  1319. 	.align 4

  1320. 

  1321. LL(996):

  1322. 	andi.	J, M, 1

  1323. 	ble	LL(999)

  1324. 

  1325. 	LFDUX	f0, Y, INCY

  1326. 	LFDU	f1, 1 * SIZE(Y)

  1327. 

  1328. 	LFDU	f8,   1 * SIZE(YY)

  1329. 	LFDU	f9,   1 * SIZE(YY)

  1330. 

  1331. 	FADD	f8,  f8,  f0

  1332. 	FADD	f9,  f9,  f1

  1333. 

  1334. 	STFDUX	f8,  Y1, INCY

  1335. 	STFDU	f9,  1 * SIZE(Y1)

  1336. 	.align 4

  1337. 

  1338. LL(999):

  1339. 	li	r3, 0

  1340. 

  1341. 	lfd	f14,     0(SP)

  1342. 	lfd	f15,     8(SP)

  1343. 	lfd	f16,    16(SP)

  1344. 	lfd	f17,    24(SP)

  1345. 	lfd	f18,    32(SP)

  1346. 	lfd	f19,    40(SP)

  1347. 	lfd	f20,    48(SP)

  1348. 	lfd	f21,    56(SP)

  1349. 	lfd	f22,    64(SP)

  1350. 	lfd	f23,    72(SP)

  1351. 	lfd	f24,    80(SP)

  1352. 	lfd	f25,    88(SP)

  1353. 	lfd	f26,    96(SP)

  1354. 	lfd	f27,   104(SP)

  1355. 	lfd	f28,   112(SP)

  1356. 	lfd	f29,   120(SP)

  1357. 	lfd	f30,   128(SP)

  1358. 	lfd	f31,   136(SP)

  1359. 

  1360. #ifdef __64BIT__

  1361. 	ld	r14,   144(SP)

  1362. 	ld	r15,   152(SP)

  1363. 	ld	r16,   160(SP)

  1364. 	ld	r17,   168(SP)

  1365. 	ld	r18,   176(SP)

  1366. 	ld	r19,   184(SP)

  1367. 	ld	r20,   192(SP)

  1368. 	ld	r21,   200(SP)

  1369. 	ld	r22,   208(SP)

  1370. #else

  1371. 	lwz	r14,   144(SP)

  1372. 	lwz	r15,   148(SP)

  1373. 	lwz	r16,   152(SP)

  1374. 	lwz	r17,   156(SP)

  1375. 	lwz	r18,   160(SP)

  1376. 	lwz	r19,   164(SP)

  1377. 	lwz	r20,   168(SP)

  1378. 	lwz	r21,   172(SP)

  1379. 	lwz	r22,   176(SP)

  1380. #endif

  1381. 

  1382. 	addi	SP, SP, STACKSIZE

  1383. 	blr

  1384. 

  1385. 	EPILOGUE

  1386. #endif