OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: b329e45288
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'b329e45288'
${ noResults }
OpenBLAS/kernel/x86_64/swap.S

440 lines
8.7 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define N	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #define Y	ARG6

  47. #define INCY	ARG2

  48. #else

  49. #define N	ARG1

  50. #define X	ARG2

  51. #define INCX	ARG3

  52. #define Y	ARG4

  53. #define INCY	%rbx

  54. #endif

  55. 

  56. #define XX	%r10

  57. #define YY	%r11

  58. 

  59. #include "l1param.h"

  60. 

  61. 	PROLOGUE

  62. 	PROFCODE

  63. 

  64. #ifndef WINDOWS_ABI

  65. #ifndef XDOUBLE

  66. 	movq	 8(%rsp), INCY

  67. #else

  68. 	movq	24(%rsp), INCY

  69. #endif

  70. #else

  71. 	pushq	%rbx

  72. 

  73. 	movq	48(%rsp), X

  74. 	movq	56(%rsp), INCX

  75. 	movq	64(%rsp), Y

  76. 	movq	72(%rsp), INCY

  77. #endif

  78. 

  79. 	EMMS

  80. 

  81. 	salq	$BASE_SHIFT, INCX

  82. 	salq	$BASE_SHIFT, INCY

  83. 

  84. 	cmpq	$SIZE, INCX

  85. 	jne	.L14

  86. 	cmpq	$SIZE, INCY

  87. 	jne	.L14

  88. 

  89. 	movq	N,  %rax

  90. 	sarq	$3, %rax

  91. 	jle	.L15

  92. 	ALIGN_3

  93. 

  94. .L16:

  95. #ifdef XDOUBLE

  96. 	movq	  0(X), %mm0

  97. 	movq	  8(X), %mm1

  98. 	movq	 16(X), %mm2

  99. 	movq	 24(X), %mm3

  100. 	movq	  0(Y), %mm4

  101. 	movq	  8(Y), %mm5

  102. 	movq	 16(Y), %mm6

  103. 	movq	 24(Y), %mm7

  104. 

  105. 	movq	%mm4,   0(X)

  106. 	movq	%mm5,   8(X)

  107. 	movq	%mm6,  16(X)

  108. 	movq	%mm7,  24(X)

  109. 	movq	%mm0,   0(Y)

  110. 	movq	%mm1,   8(Y)

  111. 	movq	%mm2,  16(Y)

  112. 	movq	%mm3,  24(Y)

  113. 

  114. 	movq	 32(X), %mm0

  115. 	movq	 40(X), %mm1

  116. 	movq	 48(X), %mm2

  117. 	movq	 56(X), %mm3

  118. 	movq	 32(Y), %mm4

  119. 	movq	 40(Y), %mm5

  120. 	movq	 48(Y), %mm6

  121. 	movq	 56(Y), %mm7

  122. 

  123. 	movq	%mm4,  32(X)

  124. 	movq	%mm5,  40(X)

  125. 	movq	%mm6,  48(X)

  126. 	movq	%mm7,  56(X)

  127. 	movq	%mm0,  32(Y)

  128. 	movq	%mm1,  40(Y)

  129. 	movq	%mm2,  48(Y)

  130. 	movq	%mm3,  56(Y)

  131. 

  132. 	movq	 64(X), %mm0

  133. 	movq	 72(X), %mm1

  134. 	movq	 80(X), %mm2

  135. 	movq	 88(X), %mm3

  136. 	movq	 64(Y), %mm4

  137. 	movq	 72(Y), %mm5

  138. 	movq	 80(Y), %mm6

  139. 	movq	 88(Y), %mm7

  140. 

  141. 	movq	%mm4,  64(X)

  142. 	movq	%mm5,  72(X)

  143. 	movq	%mm6,  80(X)

  144. 	movq	%mm7,  88(X)

  145. 	movq	%mm0,  64(Y)

  146. 	movq	%mm1,  72(Y)

  147. 	movq	%mm2,  80(Y)

  148. 	movq	%mm3,  88(Y)

  149. 

  150. 	movq	 96(X), %mm0

  151. 	movq	104(X), %mm1

  152. 	movq	112(X), %mm2

  153. 	movq	120(X), %mm3

  154. 	movq	 96(Y), %mm4

  155. 	movq	104(Y), %mm5

  156. 	movq	112(Y), %mm6

  157. 	movq	120(Y), %mm7

  158. 

  159. 	movq	%mm4,  96(X)

  160. 	movq	%mm5, 104(X)

  161. 	movq	%mm6, 112(X)

  162. 	movq	%mm7, 120(X)

  163. 	movq	%mm0,  96(Y)

  164. 	movq	%mm1, 104(Y)

  165. 	movq	%mm2, 112(Y)

  166. 	movq	%mm3, 120(Y)

  167. 

  168. #elif defined(DOUBLE)

  169. 

  170. #ifdef PREFETCHW

  171. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  172. #endif

  173. 

  174. 	movq	0 * SIZE(X), %mm0

  175. 	movq	1 * SIZE(X), %mm1

  176. 	movq	2 * SIZE(X), %mm2

  177. 	movq	3 * SIZE(X), %mm3

  178. 	movq	0 * SIZE(Y), %mm4

  179. 	movq	1 * SIZE(Y), %mm5

  180. 	movq	2 * SIZE(Y), %mm6

  181. 	movq	3 * SIZE(Y), %mm7

  182. 

  183. #ifdef PREFETCHW

  184. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  185. #endif

  186. 

  187. 	movq	%mm4, 0 * SIZE(X)

  188. 	movq	%mm5, 1 * SIZE(X)

  189. 	movq	%mm6, 2 * SIZE(X)

  190. 	movq	%mm7, 3 * SIZE(X)

  191. 	movq	%mm0, 0 * SIZE(Y)

  192. 	movq	%mm1, 1 * SIZE(Y)

  193. 	movq	%mm2, 2 * SIZE(Y)

  194. 	movq	%mm3, 3 * SIZE(Y)

  195. 

  196. #ifdef PREFETCHW

  197. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  198. #endif

  199. 

  200. 	movq	4 * SIZE(X), %mm0

  201. 	movq	5 * SIZE(X), %mm1

  202. 	movq	6 * SIZE(X), %mm2

  203. 	movq	7 * SIZE(X), %mm3

  204. 	movq	4 * SIZE(Y), %mm4

  205. 	movq	5 * SIZE(Y), %mm5

  206. 	movq	6 * SIZE(Y), %mm6

  207. 	movq	7 * SIZE(Y), %mm7

  208. 

  209. #ifdef PREFETCHW

  210. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  211. #endif

  212. 

  213. 	movq	%mm4, 4 * SIZE(X)

  214. 	movq	%mm5, 5 * SIZE(X)

  215. 	movq	%mm6, 6 * SIZE(X)

  216. 	movq	%mm7, 7 * SIZE(X)

  217. 	movq	%mm0, 4 * SIZE(Y)

  218. 	movq	%mm1, 5 * SIZE(Y)

  219. 	movq	%mm2, 6 * SIZE(Y)

  220. 	movq	%mm3, 7 * SIZE(Y)

  221. 

  222. #else

  223. #ifdef PREFETCHW

  224. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  225. #endif

  226. 	movq	0 * SIZE(X), %mm0

  227. 	movq	2 * SIZE(X), %mm1

  228. 	movq	4 * SIZE(X), %mm2

  229. 	movq	6 * SIZE(X), %mm3

  230. 	movq	0 * SIZE(Y), %mm4

  231. 	movq	2 * SIZE(Y), %mm5

  232. 	movq	4 * SIZE(Y), %mm6

  233. 	movq	6 * SIZE(Y), %mm7

  234. 

  235. #ifdef PREFETCHW

  236. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  237. #endif

  238. 

  239. 	movq	%mm4, 0 * SIZE(X)

  240. 	movq	%mm5, 2 * SIZE(X)

  241. 	movq	%mm6, 4 * SIZE(X)

  242. 	movq	%mm7, 6 * SIZE(X)

  243. 

  244. 	movq	%mm0, 0 * SIZE(Y)

  245. 	movq	%mm1, 2 * SIZE(Y)

  246. 	movq	%mm2, 4 * SIZE(Y)

  247. 	movq	%mm3, 6 * SIZE(Y)

  248. #endif

  249. 

  250. 	addq	$8 * SIZE, X

  251. 	addq	$8 * SIZE, Y

  252. 	decq	%rax

  253. 	jg	.L16

  254. 	ALIGN_3

  255. 

  256. .L15:

  257. 	movq	N, %rax

  258. 	andq	$7,   %rax

  259. 	jle	.L27

  260. 	ALIGN_3

  261. 

  262. .L22:

  263. 

  264. #ifdef XDOUBLE

  265. 	movq	  0(X), %mm0

  266. 	movq	  8(X), %mm1

  267. 	movq	  0(Y), %mm4

  268. 	movq	  8(Y), %mm5

  269. 

  270. 	movq	%mm4,   0(X)

  271. 	movq	%mm5,   8(X)

  272. 	movq	%mm0,   0(Y)

  273. 	movq	%mm1,   8(Y)

  274. #else

  275. 	MOVQ	0 * SIZE(X), %mm0

  276. 	MOVQ	0 * SIZE(Y), %mm4

  277. 	MOVQ	%mm4, 0 * SIZE(X)

  278. 	MOVQ	%mm0, 0 * SIZE(Y)

  279. #endif

  280. 

  281. 	addq	$SIZE, X

  282. 	addq	$SIZE, Y

  283. 	decq	%rax

  284. 	jg	.L22

  285. 	jmp	.L27

  286. 	ALIGN_3

  287. 

  288. /* INCX != 1 or INCY != 1 */

  289. 

  290. .L14:

  291. 	movq	N, %rax

  292. 	movq	X, XX

  293. 	movq	Y, YY

  294. 	sarq	$2,   %rax

  295. 	jle	.L28

  296. 	ALIGN_2

  297. 

  298. .L29:

  299. #ifdef XDOUBLE

  300. 	movq	  0(X), %mm0

  301. 	movq	  8(X), %mm1

  302. 	addq	INCX, X

  303. 	movq	  0(Y), %mm4

  304. 	movq	  8(Y), %mm5

  305. 	addq	INCY, Y

  306. 

  307. 	movq	%mm4,   0(XX)

  308. 	movq	%mm5,   8(XX)

  309. 	addq	INCX, XX

  310. 	movq	%mm0,   0(YY)

  311. 	movq	%mm1,   8(YY)

  312. 	addq	INCY, YY

  313. 

  314. 	movq	  0(X), %mm0

  315. 	movq	  8(X), %mm1

  316. 	addq	INCX, X

  317. 	movq	  0(Y), %mm4

  318. 	movq	  8(Y), %mm5

  319. 	addq	INCY, Y

  320. 

  321. 	movq	%mm4,   0(XX)

  322. 	movq	%mm5,   8(XX)

  323. 	addq	INCX, XX

  324. 	movq	%mm0,   0(YY)

  325. 	movq	%mm1,   8(YY)

  326. 	addq	INCY, YY

  327. 

  328. 	movq	  0(X), %mm0

  329. 	movq	  8(X), %mm1

  330. 	addq	INCX, X

  331. 	movq	  0(Y), %mm4

  332. 	movq	  8(Y), %mm5

  333. 	addq	INCY, Y

  334. 

  335. 	movq	%mm4,   0(XX)

  336. 	movq	%mm5,   8(XX)

  337. 	addq	INCX, XX

  338. 	movq	%mm0,   0(YY)

  339. 	movq	%mm1,   8(YY)

  340. 	addq	INCY, YY

  341. 

  342. 	movq	  0(X), %mm0

  343. 	movq	  8(X), %mm1

  344. 	addq	INCX, X

  345. 	movq	  0(Y), %mm4

  346. 	movq	  8(Y), %mm5

  347. 	addq	INCY, Y

  348. 

  349. 	movq	%mm4,   0(XX)

  350. 	movq	%mm5,   8(XX)

  351. 	addq	INCX, XX

  352. 	movq	%mm0,   0(YY)

  353. 	movq	%mm1,   8(YY)

  354. 	addq	INCY, YY

  355. #else

  356. 	MOVQ	(X), %mm0

  357. 	addq	INCX, X

  358. 	MOVQ	(X), %mm1

  359. 	addq	INCX, X

  360. 	MOVQ	(X), %mm2

  361. 	addq	INCX, X

  362. 	MOVQ	(X), %mm3

  363. 	addq	INCX, X

  364. 

  365. 	MOVQ	(Y), %mm4

  366. 	addq	INCY, Y

  367. 	MOVQ	(Y), %mm5

  368. 	addq	INCY, Y

  369. 	MOVQ	(Y), %mm6

  370. 	addq	INCY, Y

  371. 	MOVQ	(Y), %mm7

  372. 	addq	INCY, Y

  373. 

  374. 	MOVQ	%mm4, (XX)

  375. 	addq	INCX, XX

  376. 	MOVQ	%mm5, (XX)

  377. 	addq	INCX, XX

  378. 	MOVQ	%mm6, (XX)

  379. 	addq	INCX, XX

  380. 	MOVQ	%mm7, (XX)

  381. 	addq	INCX, XX

  382. 

  383. 	MOVQ	%mm0, (YY)

  384. 	addq	INCY, YY

  385. 	MOVQ	%mm1, (YY)

  386. 	addq	INCY, YY

  387. 	MOVQ	%mm2, (YY)

  388. 	addq	INCY, YY

  389. 	MOVQ	%mm3, (YY)

  390. 	addq	INCY, YY

  391. #endif

  392. 

  393. 	decq	%rax

  394. 	jg	.L29

  395. 	ALIGN_3

  396. 

  397. .L28:

  398. 	movq	N, %rax

  399. 	andq	$3,   %rax

  400. 	jle	.L27

  401. 	ALIGN_3

  402. 

  403. .L35:

  404. #ifdef XDOUBLE

  405. 	movq	  0(X), %mm0

  406. 	movq	  8(X), %mm1

  407. 	movq	  0(Y), %mm4

  408. 	movq	  8(Y), %mm5

  409. 

  410. 	movq	%mm4,   0(X)

  411. 	movq	%mm5,   8(X)

  412. 	movq	%mm0,   0(Y)

  413. 	movq	%mm1,   8(Y)

  414. #else

  415. 	MOVQ	(X), %mm0

  416. 	MOVQ	(Y), %mm4

  417. 

  418. 	MOVQ	%mm4, (X)

  419. 	MOVQ	%mm0, (Y)

  420. #endif

  421. 	addq	INCX, X

  422. 	addq	INCY, Y

  423. 

  424. 	decq	%rax

  425. 	jg	.L35

  426. 	ALIGN_3

  427. 

  428. .L27:

  429. 	EMMS

  430. 	xorq	%rax,%rax

  431. 

  432. #ifdef WINDOWS_ABI

  433. 	popq	%rbx

  434. #endif

  435. 

  436. 	ret

  437. 

  438. 	EPILOGUE