OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_ncopy_2.S

604 lines
11 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef NEHALEM

  43. #define PREFETCHSIZE	16

  44. #define PREFETCH	prefetcht0

  45. #define PREFETCHW	prefetcht0

  46. #endif

  47. 

  48. #ifdef SANDYBRIDGE

  49. #define PREFETCHSIZE	16

  50. #define PREFETCH	prefetcht0

  51. #define PREFETCHW	prefetcht0

  52. #endif

  53. 

  54. #ifndef MOVAPS

  55. #define MOVAPS	movaps

  56. #endif

  57. 

  58. #ifndef WINDOWS_ABI

  59. 

  60. #define M	ARG1	/* rdi */

  61. #define N	ARG2	/* rsi */

  62. #define A	ARG3	/* rdx */

  63. #define LDA	ARG4	/* rcx */

  64. #define B	ARG5	/* r8  */

  65. 

  66. #define I	%r9

  67. 

  68. #else

  69. 

  70. #define STACKSIZE 256

  71. 

  72. #define M	ARG1	/* rcx */

  73. #define N	ARG2	/* rdx */

  74. #define A	ARG3	/* r8  */

  75. #define LDA	ARG4	/* r9  */

  76. #define OLD_B		40 + 32 + STACKSIZE(%rsp)

  77. 

  78. #define B	%r14

  79. #define I	%r15

  80. 

  81. #endif

  82. 

  83. #define J	%r10

  84. #define AO1	%r11

  85. #define AO2	%r12

  86. #define MM	%r13

  87. 

  88. 	PROLOGUE

  89. 	PROFCODE

  90. 

  91. #ifdef WINDOWS_ABI

  92. 	pushq	%r15

  93. 	pushq	%r14

  94. #endif

  95. 	pushq	%r13

  96. 	pushq	%r12

  97. 

  98. #ifdef WINDOWS_ABI

  99. 	subq	$STACKSIZE, %rsp

  100. 

  101. 	movups	%xmm6,    0(%rsp)

  102. 

  103. 	movq	OLD_B,     B

  104. #endif

  105. 

  106. 	leaq	(,LDA, SIZE), LDA

  107. 	subq	$-16 * SIZE, B

  108. 

  109. 	movq	M, MM

  110. 	leaq	-1(M), %rax

  111. 	testq	$SIZE, A

  112. 	cmovne	%rax, MM

  113. 

  114. 	testq	$SIZE, LDA

  115. 	jne	.L50

  116. 

  117. 	movq	N,  J

  118. 	sarq	$1, J

  119. 	jle	.L30

  120. 	ALIGN_4

  121. 

  122. .L21:

  123. 	movq	A, AO1

  124. 	leaq	(A, LDA), AO2

  125. 	leaq	(A, LDA, 2), A

  126. 

  127. 	testq	$SIZE, A

  128. 	je	.L22

  129. 

  130. 	movsd	0 * SIZE(AO1),      %xmm0

  131. 	movsd	0 * SIZE(AO2),      %xmm1

  132. 

  133. 	unpcklpd %xmm1, %xmm0

  134. 

  135. 	movaps	%xmm0,  -16 * SIZE(B)

  136. 

  137. 	addq	$1 * SIZE, AO1

  138. 	addq	$1 * SIZE, AO2

  139. 	subq	$-2 * SIZE, B

  140. 	ALIGN_3

  141. 

  142. .L22:

  143. 	movq	MM, I

  144. 	sarq	$3, I

  145. 	jle	.L24

  146. 	ALIGN_4

  147. 

  148. .L23:

  149. #ifdef PREFETCH

  150. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO1)

  151. #endif

  152. 

  153. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  154. 	MOVAPS	0 * SIZE(AO2),      %xmm1

  155. 	MOVAPS	2 * SIZE(AO1),      %xmm2

  156. 	MOVAPS	2 * SIZE(AO2),      %xmm3

  157. 

  158. 	movaps	 %xmm0, %xmm4

  159. 	unpcklpd %xmm1, %xmm0

  160. 	movaps	 %xmm2, %xmm6

  161. 	unpcklpd %xmm3, %xmm2

  162. 

  163. 	unpckhpd %xmm1, %xmm4

  164. 	unpckhpd %xmm3, %xmm6

  165. 

  166. #ifdef PREFETCHW

  167. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  168. #endif

  169. 

  170. 	movaps	%xmm0,  -16 * SIZE(B)

  171. 	movaps	%xmm4,  -14 * SIZE(B)

  172. 	movaps	%xmm2,  -12 * SIZE(B)

  173. 	movaps	%xmm6,  -10 * SIZE(B)

  174. 

  175. #ifdef PREFETCH

  176. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO2)

  177. #endif

  178. 

  179. 	MOVAPS	4 * SIZE(AO1), %xmm0

  180. 	MOVAPS	4 * SIZE(AO2), %xmm1

  181. 	MOVAPS	6 * SIZE(AO1), %xmm2

  182. 	MOVAPS	6 * SIZE(AO2), %xmm3

  183. 

  184. 	movaps	 %xmm0, %xmm4

  185. 	unpcklpd %xmm1, %xmm0

  186. 	unpckhpd %xmm1, %xmm4

  187. 

  188. 	movaps	 %xmm2, %xmm6

  189. 	unpcklpd %xmm3, %xmm2

  190. 	unpckhpd %xmm3, %xmm6

  191. 

  192. #ifdef PREFETCHW

  193. 	PREFETCHW	(PREFETCHSIZE * 4 +  8) * SIZE(B)

  194. #endif

  195. 

  196. 	movaps	%xmm0,  -8 * SIZE(B)

  197. 	movaps	%xmm4,  -6 * SIZE(B)

  198. 	movaps	%xmm2,  -4 * SIZE(B)

  199. 	movaps	%xmm6,  -2 * SIZE(B)

  200. 

  201. 	addq	$8 * SIZE, AO1

  202. 	addq	$8 * SIZE, AO2

  203. 	subq	$-16 * SIZE, B

  204. 

  205. 	decq	I

  206. 	jg	.L23

  207. 	ALIGN_4

  208. 

  209. .L24:

  210. 	testq	$4, MM

  211. 	jle	.L26

  212. 

  213. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  214. 	MOVAPS	0 * SIZE(AO2),      %xmm1

  215. 	MOVAPS	2 * SIZE(AO1),      %xmm2

  216. 	MOVAPS	2 * SIZE(AO2),      %xmm3

  217. 

  218. 	movaps	 %xmm0, %xmm4

  219. 	unpcklpd %xmm1, %xmm0

  220. 	unpckhpd %xmm1, %xmm4

  221. 

  222. 	movaps	 %xmm2, %xmm6

  223. 	unpcklpd %xmm3, %xmm2

  224. 	unpckhpd %xmm3, %xmm6

  225. 

  226. 	movaps	%xmm0,  -16 * SIZE(B)

  227. 	movaps	%xmm4,  -14 * SIZE(B)

  228. 	movaps	%xmm2,  -12 * SIZE(B)

  229. 	movaps	%xmm6,  -10 * SIZE(B)

  230. 

  231. 	addq	$4 * SIZE, AO1

  232. 	addq	$4 * SIZE, AO2

  233. 	subq	$-8 * SIZE, B

  234. 	ALIGN_4

  235. 

  236. .L26:

  237. 	testq	$2, MM

  238. 	jle	.L28

  239. 

  240. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  241. 	MOVAPS	0 * SIZE(AO2),      %xmm1

  242. 

  243. 	movaps	 %xmm0, %xmm2

  244. 	unpcklpd %xmm1, %xmm0

  245. 	unpckhpd %xmm1, %xmm2

  246. 

  247. 	movaps	%xmm0,  -16 * SIZE(B)

  248. 	movaps	%xmm2,  -14 * SIZE(B)

  249. 

  250. 	addq	$2 * SIZE, AO1

  251. 	addq	$2 * SIZE, AO2

  252. 	subq	$-4 * SIZE, B

  253. 	ALIGN_4

  254. 

  255. .L28:

  256. 	testq	$1, MM

  257. 	jle	.L29

  258. 

  259. 	movsd	0 * SIZE(AO1),      %xmm0

  260. 	movsd	0 * SIZE(AO2),      %xmm1

  261. 

  262. 	unpcklpd %xmm1, %xmm0

  263. 

  264. 	movaps	%xmm0,  -16 * SIZE(B)

  265. 	subq	$-2 * SIZE, B

  266. 	ALIGN_4

  267. 

  268. .L29:

  269. 	decq	J

  270. 	jg	.L21

  271. 	ALIGN_4

  272. 

  273. .L30:

  274. 	testq	$1, N

  275. 	jle	.L999

  276. 

  277. .L30x:

  278. 	movq	A, AO1

  279. 

  280. 	testq	$SIZE, A

  281. 	jne	.L35

  282. 

  283. 	movq	M,  I

  284. 	sarq	$3, I

  285. 	jle	.L32

  286. 	ALIGN_4

  287. 

  288. .L31:

  289. #ifdef PREFETCH

  290. 	PREFETCH	PREFETCHSIZE * 4 * SIZE(AO1)

  291. #endif

  292. 

  293. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  294. 	MOVAPS	2 * SIZE(AO1),      %xmm1

  295. 	MOVAPS	4 * SIZE(AO1),      %xmm2

  296. 	MOVAPS	6 * SIZE(AO1),      %xmm3

  297. 

  298. #ifdef PREFETCHW

  299. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  300. #endif

  301. 

  302. 	movaps	%xmm0,  -16 * SIZE(B)

  303. 	movaps	%xmm1,  -14 * SIZE(B)

  304. 	movaps	%xmm2,  -12 * SIZE(B)

  305. 	movaps	%xmm3,  -10 * SIZE(B)

  306. 

  307. 	addq	$8 * SIZE, AO1

  308. 	addq	$8 * SIZE, B

  309. 

  310. 	decq	I

  311. 	jg	.L31

  312. 	ALIGN_4

  313. 

  314. .L32:

  315. 	testq	$4, M

  316. 	jle	.L33

  317. 

  318. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  319. 	MOVAPS	2 * SIZE(AO1),      %xmm1

  320. 

  321. 	movaps	%xmm0, -16 * SIZE(B)

  322. 	movaps	%xmm1, -14 * SIZE(B)

  323. 

  324. 	addq	$4 * SIZE, AO1

  325. 	subq	$-4 * SIZE, B

  326. 	ALIGN_4

  327. 

  328. .L33:

  329. 	testq	$2, M

  330. 	jle	.L34

  331. 

  332. 	MOVAPS	0 * SIZE(AO1),      %xmm0

  333. 

  334. 	movaps	%xmm0,  -16 * SIZE(B)

  335. 

  336. 	addq	$2 * SIZE, AO1

  337. 	subq	$-2 * SIZE, B

  338. 	ALIGN_4

  339. 

  340. .L34:

  341. 	testq	$1, M

  342. 	jle	.L999

  343. 

  344. 	movsd	0 * SIZE(AO1),      %xmm0

  345. 	movlpd	%xmm0,  -16 * SIZE(B)

  346. 	jmp	.L999

  347. 	ALIGN_4

  348. 

  349. .L35:

  350. 	movaps	-1 * SIZE(AO1),      %xmm0

  351. 

  352. 	movq	M,  I

  353. 	sarq	$3, I

  354. 	jle	.L36

  355. 	ALIGN_4

  356. 

  357. .L36:

  358. #ifdef PREFETCH

  359. 	PREFETCH	PREFETCHSIZE * 4 * SIZE(AO1)

  360. #endif

  361. 

  362. 	MOVAPS	1 * SIZE(AO1),      %xmm1

  363. 	MOVAPS	3 * SIZE(AO1),      %xmm2

  364. 	MOVAPS	5 * SIZE(AO1),      %xmm3

  365. 	MOVAPS	7 * SIZE(AO1),      %xmm4

  366. 

  367. 	shufpd	$1, %xmm1, %xmm0

  368. 	shufpd	$1, %xmm2, %xmm1

  369. 	shufpd	$1, %xmm3, %xmm2

  370. 	shufpd	$1, %xmm4, %xmm3

  371. 

  372. #ifdef PREFETCHW

  373. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  374. #endif

  375. 

  376. 	movaps	%xmm0,  -16 * SIZE(B)

  377. 	movaps	%xmm1,  -14 * SIZE(B)

  378. 	movaps	%xmm2,  -12 * SIZE(B)

  379. 	movaps	%xmm3,  -10 * SIZE(B)

  380. 

  381. 	movaps	%xmm4, %xmm0

  382. 

  383. 	addq	$8 * SIZE, AO1

  384. 	subq	$-8 * SIZE, B

  385. 

  386. 	decq	I

  387. 	jg	.L36

  388. 	ALIGN_4

  389. 

  390. .L37:

  391. 	testq	$4, M

  392. 	jle	.L38

  393. 

  394. 	MOVAPS	1 * SIZE(AO1),      %xmm1

  395. 	MOVAPS	3 * SIZE(AO1),      %xmm2

  396. 

  397. 	shufpd	$1, %xmm1, %xmm0

  398. 	shufpd	$1, %xmm2, %xmm1

  399. 

  400. 	movaps	%xmm0,  -16 * SIZE(B)

  401. 	movaps	%xmm1,  -14 * SIZE(B)

  402. 

  403. 	movaps	%xmm2, %xmm0

  404. 

  405. 	addq	$4 * SIZE, AO1

  406. 	addq	$4 * SIZE, B

  407. 	ALIGN_4

  408. 

  409. .L38:

  410. 	testq	$2, M

  411. 	jle	.L39

  412. 

  413. 	MOVAPS	1 * SIZE(AO1),      %xmm1

  414. 

  415. 	shufpd	$1, %xmm1, %xmm0

  416. 

  417. 	movaps	%xmm0,  -16 * SIZE(B)

  418. 

  419. 	movaps	%xmm1, %xmm0

  420. 

  421. 	addq	$2 * SIZE, AO1

  422. 	subq	$-2 * SIZE, B

  423. 	ALIGN_4

  424. 

  425. .L39:

  426. 	testq	$1, M

  427. 	jle	.L999

  428. 

  429. 	movhpd	%xmm0,  -16 * SIZE(B)

  430. 	jmp	.L999

  431. 	ALIGN_4

  432. 

  433. .L50:

  434. 	movq	N,  J

  435. 	sarq	$1, J

  436. 	jle	.L30

  437. 	ALIGN_4

  438. 

  439. .L61:

  440. 	movq	A, AO1

  441. 	leaq	(A, LDA), AO2

  442. 	leaq	(A, LDA, 2), A

  443. 

  444. 	testq	$SIZE, A

  445. 	je	.L62

  446. 

  447. 	movsd	0 * SIZE(AO1), %xmm0

  448. 	movsd	0 * SIZE(AO2), %xmm1

  449. 

  450. 	unpcklpd %xmm1, %xmm0

  451. 

  452. 	movaps	%xmm0,  -16 * SIZE(B)

  453. 

  454. 	addq	$1 * SIZE, AO1

  455. 	addq	$1 * SIZE, AO2

  456. 	subq	$-2 * SIZE, B

  457. 	ALIGN_3

  458. 

  459. .L62:

  460. 	MOVAPS	-1 * SIZE(AO2), %xmm5

  461. 

  462. 	movq	MM, I

  463. 	sarq	$3, I

  464. 	jle	.L64

  465. 	ALIGN_4

  466. 

  467. .L63:

  468. #ifdef PREFETCH

  469. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO1)

  470. #endif

  471. 

  472. 	MOVAPS	0 * SIZE(AO1), %xmm0

  473. 	MOVAPS	1 * SIZE(AO2), %xmm1

  474. 	MOVAPS	2 * SIZE(AO1), %xmm2

  475. 	MOVAPS	3 * SIZE(AO2), %xmm3

  476. 

  477. 	movsd	 %xmm0, %xmm5

  478. 	shufpd	 $1, %xmm1, %xmm0

  479. 	movsd	 %xmm2, %xmm1

  480. 	shufpd	 $1, %xmm3, %xmm2

  481. 

  482. #ifdef PREFETCHW

  483. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  484. #endif

  485. 

  486. 	movaps	%xmm5,  -16 * SIZE(B)

  487. 	movaps	%xmm0,  -14 * SIZE(B)

  488. 	movaps	%xmm1,  -12 * SIZE(B)

  489. 	movaps	%xmm2,  -10 * SIZE(B)

  490. 

  491. #ifdef PREFETCH

  492. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO2)

  493. #endif

  494. 

  495. 	MOVAPS	4 * SIZE(AO1), %xmm0

  496. 	MOVAPS	5 * SIZE(AO2), %xmm1

  497. 	MOVAPS	6 * SIZE(AO1), %xmm2

  498. 	MOVAPS	7 * SIZE(AO2), %xmm5

  499. 

  500. 	movsd	 %xmm0, %xmm3

  501. 	shufpd	 $1, %xmm1, %xmm0

  502. 	movsd	 %xmm2, %xmm1

  503. 	shufpd	 $1, %xmm5, %xmm2

  504. 

  505. #ifdef PREFETCHW

  506. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  507. #endif

  508. 

  509. 	movaps	%xmm3,   -8 * SIZE(B)

  510. 	movaps	%xmm0,   -6 * SIZE(B)

  511. 	movaps	%xmm1,   -4 * SIZE(B)

  512. 	movaps	%xmm2,   -2 * SIZE(B)

  513. 

  514. 	addq	$8 * SIZE, AO1

  515. 	addq	$8 * SIZE, AO2

  516. 	subq	$-16 * SIZE, B

  517. 

  518. 	decq	I

  519. 	jg	.L63

  520. 	ALIGN_4

  521. 

  522. .L64:

  523. 	testq	$4, MM

  524. 	jle	.L66

  525. 

  526. 	MOVAPS	0 * SIZE(AO1), %xmm0

  527. 	MOVAPS	1 * SIZE(AO2), %xmm1

  528. 	MOVAPS	2 * SIZE(AO1), %xmm2

  529. 	MOVAPS	3 * SIZE(AO2), %xmm3

  530. 

  531. 	movsd	 %xmm0, %xmm5

  532. 	shufpd	 $1, %xmm1, %xmm0

  533. 	movsd	 %xmm2, %xmm1

  534. 	shufpd	 $1, %xmm3, %xmm2

  535. 

  536. 	movaps	%xmm5,  -16 * SIZE(B)

  537. 	movaps	%xmm0,  -14 * SIZE(B)

  538. 	movaps	%xmm1,  -12 * SIZE(B)

  539. 	movaps	%xmm2,  -10 * SIZE(B)

  540. 

  541. 	movaps	%xmm3, %xmm5

  542. 

  543. 	addq	$4 * SIZE, AO1

  544. 	addq	$4 * SIZE, AO2

  545. 	subq	$-8 * SIZE, B

  546. 	ALIGN_4

  547. 

  548. .L66:

  549. 	testq	$2, MM

  550. 	jle	.L68

  551. 

  552. 	MOVAPS	0 * SIZE(AO1), %xmm0

  553. 	MOVAPS	1 * SIZE(AO2), %xmm1

  554. 

  555. 	movsd	 %xmm0, %xmm5

  556. 	shufpd	 $1, %xmm1, %xmm0

  557. 

  558. 	movaps	%xmm5,  -16 * SIZE(B)

  559. 	movaps	%xmm0,  -14 * SIZE(B)

  560. 

  561. 	addq	$2 * SIZE, AO1

  562. 	addq	$2 * SIZE, AO2

  563. 	subq	$-4 * SIZE, B

  564. 	ALIGN_4

  565. 

  566. .L68:

  567. 	testq	$1, MM

  568. 	jle	.L69

  569. 

  570. 	movsd	0 * SIZE(AO1),      %xmm0

  571. 	movsd	0 * SIZE(AO2),      %xmm1

  572. 

  573. 	unpcklpd %xmm1, %xmm0

  574. 

  575. 	movaps	%xmm0,  -16 * SIZE(B)

  576. 	subq	$-2 * SIZE, B

  577. 	ALIGN_4

  578. 

  579. .L69:

  580. 	decq	J

  581. 	jg	.L61

  582. 

  583. 	testq	$1, N

  584. 	jne	.L30

  585. 	ALIGN_4

  586. 

  587. .L999:

  588. #ifdef WINDOWS_ABI

  589. 	movups	  0(%rsp), %xmm6

  590. 

  591. 	addq	$STACKSIZE, %rsp

  592. #endif

  593. 

  594. 	popq	%r12

  595. 	popq	%r13

  596. 

  597. #ifdef WINDOWS_ABI

  598. 	popq	%r14

  599. 	popq	%r15

  600. #endif

  601. 	ret

  602. 

  603. 	EPILOGUE