OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_kernel_4x4_haswell.S

3495 lines
63 kB
Raw Permalink Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2013, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. 

  29. /*********************************************************************

  30. * 2013/10/28 Saar

  31. *        BLASTEST               : OK

  32. *        CTEST                  : OK

  33. *        TEST                   : OK

  34. 

  35. *

  36. *

  37. * 2013/10/27 Saar

  38. * Parameter:

  39. *       DGEMM_DEFAULT_UNROLL_N  4

  40. *       DGEMM_DEFAULT_UNROLL_M  4

  41. *       DGEMM_DEFAULT_P         512

  42. *       DGEMM_DEFAULT_Q         256

  43. *	A_PR1			512

  44. *	B_PR1			512

  45. *

  46. *

  47. * Performance at 9216x9216x9216:

  48. *       1 thread:       53.3 GFLOPS (MKL:  54)

  49. *       2 threads:     100.0 GFLOPS (MKL:  97)

  50. *       3 threads:     147.0 GFLOPS (MKL: 133)

  51. *       4 threads:     184.0 GFLOPS (MKL: 170)

  52. *********************************************************************/

  53. 

  54. 

  55. #define ASSEMBLER

  56. #include "common.h"

  57.  

  58. #define OLD_M	%rdi

  59. #define OLD_N	%rsi

  60. #define M	%r13

  61. #define J	%r14

  62. #define OLD_K	%rdx

  63. 

  64. #define A	%rcx

  65. #define B	%r8

  66. #define C	%r9

  67. #define LDC	%r10

  68. 	

  69. #define I	%r11

  70. #define AO	%rdi

  71. #define BO	%rsi

  72. #define	CO1	%r15

  73. #define K	%r12

  74. #define	SP	%rbx

  75. 

  76. #define BO1	%rdi

  77. #define BO2	%r15

  78. #define BO3	%rbp

  79. 

  80. #ifndef WINDOWS_ABI

  81. 

  82. #define STACKSIZE 96

  83. #define L_BUFFER_SIZE 256*8*12+4096

  84. 

  85. #else

  86. 

  87. #define STACKSIZE 256

  88. #define L_BUFFER_SIZE 128*8*12+512

  89. 

  90. #define OLD_A		40 + STACKSIZE(%rsp)

  91. #define OLD_B		48 + STACKSIZE(%rsp)

  92. #define OLD_C		56 + STACKSIZE(%rsp)

  93. #define OLD_LDC		64 + STACKSIZE(%rsp)

  94. #define OLD_OFFSET	72 + STACKSIZE(%rsp)

  95. 

  96. #endif

  97. 

  98. 

  99. #define Ndiv12	 24(%rsp)

  100. #define Nmod12	 32(%rsp)

  101. #define N	 40(%rsp)

  102. #define ALPHA	 48(%rsp)

  103. #define OFFSET	 56(%rsp)

  104. #define KK	 64(%rsp)

  105. #define KKK	 72(%rsp)

  106. #define BUFFER1	           128(%rsp)

  107. 

  108. #if defined(OS_WINDOWS)

  109. #if   L_BUFFER_SIZE > 16384

  110. #define STACK_TOUCH \

  111.         movl    $ 0,  4096 * 4(%rsp);\

  112.         movl    $ 0,  4096 * 3(%rsp);\

  113.         movl    $ 0,  4096 * 2(%rsp);\

  114.         movl    $ 0,  4096 * 1(%rsp);

  115. #elif L_BUFFER_SIZE > 12288

  116. #define STACK_TOUCH \

  117.         movl    $ 0,  4096 * 3(%rsp);\

  118.         movl    $ 0,  4096 * 2(%rsp);\

  119.         movl    $ 0,  4096 * 1(%rsp);

  120. #elif L_BUFFER_SIZE > 8192

  121. #define STACK_TOUCH \

  122.         movl    $ 0,  4096 * 2(%rsp);\

  123.         movl    $ 0,  4096 * 1(%rsp);

  124. #elif L_BUFFER_SIZE > 4096

  125. #define STACK_TOUCH \

  126.         movl    $ 0,  4096 * 1(%rsp);

  127. #else

  128. #define STACK_TOUCH

  129. #endif

  130. #else

  131. #define STACK_TOUCH

  132. #endif

  133. 

  134. #define	A_PR1	512

  135. #define	B_PR1	512

  136. 

  137. /*******************************************************************************************

  138. * Macro definitions

  139. *******************************************************************************************/

  140. 

  141. .macro INIT4x12

  142. 

  143. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  144. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  145. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  146. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  147. 	vxorpd		%ymm8 , %ymm8 , %ymm8

  148. 	vxorpd		%ymm9 , %ymm9 , %ymm9

  149. 	vxorpd		%ymm10, %ymm10, %ymm10

  150. 	vxorpd		%ymm11, %ymm11, %ymm11

  151. 	vxorpd		%ymm12, %ymm12, %ymm12

  152. 	vxorpd		%ymm13, %ymm13, %ymm13

  153. 	vxorpd		%ymm14, %ymm14, %ymm14

  154. 	vxorpd		%ymm15, %ymm15, %ymm15

  155. 

  156. .endm

  157. 

  158. .macro KERNEL4x12_I

  159. 	prefetcht0	A_PR1(AO)

  160. 	vmovups		-12 * SIZE(BO), %ymm1

  161. 	prefetcht0	B_PR1(BO)

  162. 	vmovups 	-16 * SIZE(AO), %ymm0

  163. 	prefetcht0	B_PR1+64(BO)

  164. 	vmovups		 -8 * SIZE(BO), %ymm2

  165. 	prefetcht0	B_PR1+128(BO)

  166. 	vmovups		 -4 * SIZE(BO), %ymm3

  167. 	vmulpd  	%ymm0 ,%ymm1  , %ymm4

  168. 	prefetcht0	B_PR1+192(BO)

  169. 	vmulpd  	%ymm0 ,%ymm2  , %ymm8

  170. 	vmulpd  	%ymm0 ,%ymm3  , %ymm12

  171. 	prefetcht0	B_PR1+256(BO)

  172. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  173. 	vmulpd  	%ymm0 ,%ymm1  , %ymm5

  174. 	vmulpd  	%ymm0 ,%ymm2  , %ymm9

  175. 	vmulpd  	%ymm0 ,%ymm3  , %ymm13

  176. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  177. 	vmulpd  	%ymm0 ,%ymm1  , %ymm6

  178. 	vmulpd  	%ymm0 ,%ymm2  , %ymm10

  179. 

  180. 	addq		$ 12*SIZE, BO

  181. 	vmulpd  	%ymm0 ,%ymm3  , %ymm14

  182. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  183. 	vmulpd  	%ymm0 ,%ymm1  , %ymm7

  184. 	vmovups		-12 * SIZE(BO), %ymm1

  185. 	vmulpd  	%ymm0 ,%ymm2  , %ymm11

  186. 	vmovups		 -8 * SIZE(BO), %ymm2

  187. 	vmulpd  	%ymm0 ,%ymm3  , %ymm15

  188. 	vmovups		 -4 * SIZE(BO), %ymm3

  189. 

  190. .endm

  191. 

  192. .macro KERNEL4x12_M1

  193. 	prefetcht0	A_PR1(AO)

  194. 	vmovups 	-16 * SIZE(AO), %ymm0

  195. 	prefetcht0	B_PR1(BO)

  196. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  197. 	prefetcht0	B_PR1+64(BO)

  198. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  199. 	prefetcht0	B_PR1+128(BO)

  200. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  201. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  202. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  203. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  204. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  205. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  206. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  207. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  208. 

  209. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  210. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  211. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  212. 	vmovups		-12 * SIZE(BO), %ymm1

  213. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  214. 	vmovups		 -8 * SIZE(BO), %ymm2

  215. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  216. 	vmovups		 -4 * SIZE(BO), %ymm3

  217. 

  218. .endm

  219. 

  220. .macro KERNEL4x12_M2

  221. 	vmovups 	-12 * SIZE(AO), %ymm0

  222. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  223. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  224. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  225. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  226. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  227. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  228. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  229. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  230. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  231. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  232. 

  233. 	addq		$ 8*SIZE, AO

  234. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  235. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  236. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  237. 	vmovups		  0 * SIZE(BO), %ymm1

  238. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  239. 	vmovups		  4 * SIZE(BO), %ymm2

  240. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  241. 	vmovups		  8 * SIZE(BO), %ymm3

  242. 	addq		$ 24*SIZE, BO

  243. .endm

  244. 

  245. 

  246. .macro KERNEL4x12_E

  247. 	vmovups 	-12 * SIZE(AO), %ymm0

  248. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  249. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  250. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  251. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  252. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  253. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  254. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  255. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  256. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  257. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  258. 

  259. 	addq		$ 8*SIZE, AO

  260. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  261. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  262. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  263. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  264. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  265. 	addq		$ 12*SIZE, BO

  266. .endm

  267. 

  268. .macro KERNEL4x12_SUB

  269. 	vmovups		-12 * SIZE(BO), %ymm1

  270. 	vmovups 	-16 * SIZE(AO), %ymm0

  271. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  272. 	vmovups		 -8 * SIZE(BO), %ymm2

  273. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  274. 	vmovups		 -4 * SIZE(BO), %ymm3

  275. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  276. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  277. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  278. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  279. 	addq		$ 12*SIZE, BO

  280. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  281. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  282. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  283. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  284. 	addq		$ 4*SIZE, AO

  285. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  286. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  287. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  288. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  289. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  290. 

  291. .endm

  292. 

  293. 

  294. .macro SAVE4x12

  295. 

  296. 	vbroadcastsd	ALPHA, %ymm0

  297. 

  298. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  299. 	vmulpd	%ymm0 , %ymm5 , %ymm5

  300. 	vmulpd	%ymm0 , %ymm6 , %ymm6

  301. 	vmulpd	%ymm0 , %ymm7 , %ymm7

  302. 

  303. 	vmulpd	%ymm0 , %ymm8 , %ymm8

  304. 	vmulpd	%ymm0 , %ymm9 , %ymm9

  305. 	vmulpd	%ymm0 , %ymm10, %ymm10

  306. 	vmulpd	%ymm0 , %ymm11, %ymm11

  307. 

  308. 	vmulpd	%ymm0 , %ymm12, %ymm12

  309. 	vmulpd	%ymm0 , %ymm13, %ymm13

  310. 	vmulpd	%ymm0 , %ymm14, %ymm14

  311. 	vmulpd	%ymm0 , %ymm15, %ymm15

  312. 

  313. 	vpermpd $ 0xb1 , %ymm5, %ymm5

  314. 	vpermpd $ 0xb1 , %ymm7, %ymm7

  315. 

  316. 	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0

  317. 	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1

  318. 	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2

  319. 	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3

  320. 

  321. 	vpermpd $ 0x1b , %ymm2, %ymm2

  322. 	vpermpd $ 0x1b , %ymm3, %ymm3

  323. 	vpermpd $ 0xb1 , %ymm2, %ymm2

  324. 	vpermpd $ 0xb1 , %ymm3, %ymm3

  325. 

  326. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  327. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  328. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  329. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  330. 

  331.         leaq    (CO1, LDC, 2), %rax     

  332. 	

  333. 

  334. #if !defined(TRMMKERNEL)

  335. 

  336. 	vaddpd 	                (CO1), %ymm4, %ymm4

  337. 	vaddpd 	           (CO1, LDC), %ymm5, %ymm5

  338. 	vaddpd 	               (%rax), %ymm6, %ymm6

  339. 	vaddpd 	          (%rax, LDC), %ymm7, %ymm7

  340. 

  341. #endif

  342. 

  343. 	vmovups	%ymm4 ,  	(CO1)

  344. 	vmovups	%ymm5 ,  	(CO1, LDC)

  345. 	vmovups	%ymm6 ,  	(%rax)

  346. 	vmovups	%ymm7 ,  	(%rax, LDC)

  347. 

  348. 	prefetcht0	32(CO1)

  349. 	prefetcht0	32(CO1,LDC)

  350. 	prefetcht0	32(%rax)

  351. 	prefetcht0	32(%rax,LDC)

  352. 

  353. 	vpermpd $ 0xb1 , %ymm9 , %ymm9

  354. 	vpermpd $ 0xb1 , %ymm11, %ymm11

  355. 

  356. 	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0

  357. 	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1

  358. 	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2

  359. 	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3

  360. 

  361. 	vpermpd $ 0x1b , %ymm2, %ymm2

  362. 	vpermpd $ 0x1b , %ymm3, %ymm3

  363. 	vpermpd $ 0xb1 , %ymm2, %ymm2

  364. 	vpermpd $ 0xb1 , %ymm3, %ymm3

  365. 

  366. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  367. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  368. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  369. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  370. 

  371. 

  372. 	leaq	(%rax, LDC, 2), %rax

  373. 	leaq	(%rax, LDC, 2), %rbp

  374. 

  375. #if !defined(TRMMKERNEL)

  376. 

  377. 	vaddpd 	                (%rax), %ymm4, %ymm4

  378. 	vaddpd 	           (%rax, LDC), %ymm5, %ymm5

  379. 	vaddpd 	                (%rbp), %ymm6, %ymm6

  380. 	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7

  381. 

  382. #endif

  383. 

  384. 	vmovups	%ymm4 ,  	(%rax)

  385. 	vmovups	%ymm5 ,  	(%rax, LDC)

  386. 	vmovups	%ymm6 ,  	(%rbp)

  387. 	vmovups	%ymm7 ,  	(%rbp, LDC)

  388. 

  389. 	prefetcht0	32(%rax)

  390. 	prefetcht0	32(%rax,LDC)

  391. 	prefetcht0	32(%rbp)

  392. 	prefetcht0	32(%rbp,LDC)

  393. 

  394. 	vpermpd $ 0xb1 , %ymm13, %ymm13

  395. 	vpermpd $ 0xb1 , %ymm15, %ymm15

  396. 

  397. 	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0

  398. 	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1

  399. 	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2

  400. 	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3

  401. 

  402. 	vpermpd $ 0x1b , %ymm2, %ymm2

  403. 	vpermpd $ 0x1b , %ymm3, %ymm3

  404. 	vpermpd $ 0xb1 , %ymm2, %ymm2

  405. 	vpermpd $ 0xb1 , %ymm3, %ymm3

  406. 

  407. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  408. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  409. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  410. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  411. 

  412. 

  413. 	leaq	(%rax, LDC, 4), %rax

  414. 	leaq	(%rbp, LDC, 4), %rbp

  415. 

  416. #if !defined(TRMMKERNEL)

  417. 

  418. 	vaddpd 	                (%rax), %ymm4, %ymm4

  419. 	vaddpd 	           (%rax, LDC), %ymm5, %ymm5

  420. 	vaddpd 	                (%rbp), %ymm6, %ymm6

  421. 	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7

  422. 

  423. #endif

  424. 

  425. 	vmovups	%ymm4 ,  	(%rax)

  426. 	vmovups	%ymm5 ,  	(%rax, LDC)

  427. 	vmovups	%ymm6 ,  	(%rbp)

  428. 	vmovups	%ymm7 ,  	(%rbp, LDC)

  429. 

  430. 	prefetcht0	32(%rax)

  431. 	prefetcht0	32(%rax,LDC)

  432. 	prefetcht0	32(%rbp)

  433. 	prefetcht0	32(%rbp,LDC)

  434. 

  435. 	addq	$ 4*SIZE, CO1

  436. .endm

  437. 

  438. /******************************************************************************************/

  439. 

  440. .macro INIT2x12

  441. 

  442. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  443. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  444. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  445. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  446. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  447. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  448. 	vxorpd		%xmm10, %xmm10, %xmm10

  449. 	vxorpd		%xmm11, %xmm11, %xmm11

  450. 	vxorpd		%xmm12, %xmm12, %xmm12

  451. 	vxorpd		%xmm13, %xmm13, %xmm13

  452. 	vxorpd		%xmm14, %xmm14, %xmm14

  453. 	vxorpd		%xmm15, %xmm15, %xmm15

  454. 

  455. .endm

  456. 

  457. .macro KERNEL2x12_SUB

  458. 	vmovups 	-16 * SIZE(AO), %xmm0

  459. 	vmovddup	-12 * SIZE(BO), %xmm1

  460. 	vmovddup	-11 * SIZE(BO), %xmm2

  461. 	vmovddup	-10 * SIZE(BO), %xmm3

  462. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4

  463. 	vmovddup	 -9 * SIZE(BO), %xmm1

  464. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5

  465. 	vmovddup	 -8 * SIZE(BO), %xmm2

  466. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  467. 	vmovddup	 -7 * SIZE(BO), %xmm3

  468. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7

  469. 	vmovddup	 -6 * SIZE(BO), %xmm1

  470. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8

  471. 	vmovddup	 -5 * SIZE(BO), %xmm2

  472. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9

  473. 	vmovddup	 -4 * SIZE(BO), %xmm3

  474. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10

  475. 	vmovddup	 -3 * SIZE(BO), %xmm1

  476. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11

  477. 	vmovddup	 -2 * SIZE(BO), %xmm2

  478. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12

  479. 	vmovddup	 -1 * SIZE(BO), %xmm3

  480. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13

  481. 	addq		$ 12*SIZE, BO

  482. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14

  483. 	addq		$ 2*SIZE, AO

  484. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15

  485. 

  486. .endm

  487. 

  488. .macro SAVE2x12

  489. 

  490. 	vmovddup	ALPHA, %xmm0

  491. 

  492. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  493. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  494. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  495. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  496. 

  497. 	vmulpd	%xmm0 , %xmm8 , %xmm8

  498. 	vmulpd	%xmm0 , %xmm9 , %xmm9

  499. 	vmulpd	%xmm0 , %xmm10, %xmm10

  500. 	vmulpd	%xmm0 , %xmm11, %xmm11

  501. 

  502. 	vmulpd	%xmm0 , %xmm12, %xmm12

  503. 	vmulpd	%xmm0 , %xmm13, %xmm13

  504. 	vmulpd	%xmm0 , %xmm14, %xmm14

  505. 	vmulpd	%xmm0 , %xmm15, %xmm15

  506. 

  507. 

  508.         leaq    (CO1, LDC, 2), %rax     

  509. 	

  510. 

  511. #if !defined(TRMMKERNEL)

  512. 

  513. 	vaddpd 	                (CO1), %xmm4, %xmm4

  514. 	vaddpd 	           (CO1, LDC), %xmm5, %xmm5

  515. 	vaddpd 	               (%rax), %xmm6, %xmm6

  516. 	vaddpd 	          (%rax, LDC), %xmm7, %xmm7

  517. 

  518. #endif

  519. 

  520. 	vmovups	%xmm4 ,  	(CO1)

  521. 	vmovups	%xmm5 ,  	(CO1, LDC)

  522. 	vmovups	%xmm6 ,  	(%rax)

  523. 	vmovups	%xmm7 ,  	(%rax, LDC)

  524. 

  525. 

  526. 	leaq	(%rax, LDC, 2), %rax

  527. 	leaq	(%rax, LDC, 2), %rbp

  528. 

  529. #if !defined(TRMMKERNEL)

  530. 

  531. 	vaddpd 	                (%rax), %xmm8 , %xmm4

  532. 	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5

  533. 	vaddpd 	                (%rbp), %xmm10, %xmm6

  534. 	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7

  535. 

  536. #endif

  537. 

  538. 	vmovups	%xmm4 ,  	(%rax)

  539. 	vmovups	%xmm5 ,  	(%rax, LDC)

  540. 	vmovups	%xmm6 ,  	(%rbp)

  541. 	vmovups	%xmm7 ,  	(%rbp, LDC)

  542. 

  543. 

  544. 	leaq	(%rax, LDC, 4), %rax

  545. 	leaq	(%rbp, LDC, 4), %rbp

  546. 

  547. #if !defined(TRMMKERNEL)

  548. 

  549. 	vaddpd 	                (%rax), %xmm12, %xmm4

  550. 	vaddpd 	           (%rax, LDC), %xmm13, %xmm5

  551. 	vaddpd 	                (%rbp), %xmm14, %xmm6

  552. 	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7

  553. 

  554. #endif

  555. 

  556. 	vmovups	%xmm4 ,  	(%rax)

  557. 	vmovups	%xmm5 ,  	(%rax, LDC)

  558. 	vmovups	%xmm6 ,  	(%rbp)

  559. 	vmovups	%xmm7 ,  	(%rbp, LDC)

  560. 

  561. 	addq	$ 2*SIZE, CO1

  562. .endm

  563. 

  564. 

  565. /******************************************************************************************/

  566. 

  567. .macro INIT1x12

  568. 

  569. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  570. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  571. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  572. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  573. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  574. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  575. 	vxorpd		%xmm10, %xmm10, %xmm10

  576. 	vxorpd		%xmm11, %xmm11, %xmm11

  577. 	vxorpd		%xmm12, %xmm12, %xmm12

  578. 	vxorpd		%xmm13, %xmm13, %xmm13

  579. 	vxorpd		%xmm14, %xmm14, %xmm14

  580. 	vxorpd		%xmm15, %xmm15, %xmm15

  581. 

  582. .endm

  583. 

  584. .macro KERNEL1x12_SUB

  585. 	vmovsd 	-16 * SIZE(AO), %xmm0

  586. 	vmovsd	-12 * SIZE(BO), %xmm1

  587. 	vmovsd	-11 * SIZE(BO), %xmm2

  588. 	vmovsd	-10 * SIZE(BO), %xmm3

  589. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  590. 	vmovsd	 -9 * SIZE(BO), %xmm1

  591. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  592. 	vmovsd	 -8 * SIZE(BO), %xmm2

  593. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6

  594. 	vmovsd	 -7 * SIZE(BO), %xmm3

  595. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7

  596. 	vmovsd	 -6 * SIZE(BO), %xmm1

  597. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8

  598. 	vmovsd	 -5 * SIZE(BO), %xmm2

  599. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9

  600. 	vmovsd	 -4 * SIZE(BO), %xmm3

  601. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10

  602. 	vmovsd	 -3 * SIZE(BO), %xmm1

  603. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11

  604. 	vmovsd	 -2 * SIZE(BO), %xmm2

  605. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12

  606. 	vmovsd	 -1 * SIZE(BO), %xmm3

  607. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13

  608. 	addq		$ 12*SIZE, BO

  609. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14

  610. 	addq		$ 1*SIZE, AO

  611. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15

  612. 

  613. .endm

  614. 

  615. .macro SAVE1x12

  616. 

  617. 	vmovsd	ALPHA, %xmm0

  618. 

  619. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  620. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  621. 	vmulsd	%xmm0 , %xmm6 , %xmm6

  622. 	vmulsd	%xmm0 , %xmm7 , %xmm7

  623. 

  624. 	vmulsd	%xmm0 , %xmm8 , %xmm8

  625. 	vmulsd	%xmm0 , %xmm9 , %xmm9

  626. 	vmulsd	%xmm0 , %xmm10, %xmm10

  627. 	vmulsd	%xmm0 , %xmm11, %xmm11

  628. 

  629. 	vmulsd	%xmm0 , %xmm12, %xmm12

  630. 	vmulsd	%xmm0 , %xmm13, %xmm13

  631. 	vmulsd	%xmm0 , %xmm14, %xmm14

  632. 	vmulsd	%xmm0 , %xmm15, %xmm15

  633. 

  634. 

  635.         leaq    (CO1, LDC, 2), %rax     

  636. 	

  637. 

  638. #if !defined(TRMMKERNEL)

  639. 

  640. 	vaddsd 	                (CO1), %xmm4, %xmm4

  641. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  642. 	vaddsd 	               (%rax), %xmm6, %xmm6

  643. 	vaddsd 	          (%rax, LDC), %xmm7, %xmm7

  644. 

  645. #endif

  646. 

  647. 	vmovsd	%xmm4 ,  	(CO1)

  648. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  649. 	vmovsd	%xmm6 ,  	(%rax)

  650. 	vmovsd	%xmm7 ,  	(%rax, LDC)

  651. 

  652. 

  653. 	leaq	(%rax, LDC, 2), %rax

  654. 	leaq	(%rax, LDC, 2), %rbp

  655. 

  656. #if !defined(TRMMKERNEL)

  657. 

  658. 	vaddsd 	                (%rax), %xmm8 , %xmm4

  659. 	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5

  660. 	vaddsd 	                (%rbp), %xmm10, %xmm6

  661. 	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7

  662. 

  663. #endif

  664. 

  665. 	vmovsd	%xmm4 ,  	(%rax)

  666. 	vmovsd	%xmm5 ,  	(%rax, LDC)

  667. 	vmovsd	%xmm6 ,  	(%rbp)

  668. 	vmovsd	%xmm7 ,  	(%rbp, LDC)

  669. 

  670. 

  671. 	leaq	(%rax, LDC, 4), %rax

  672. 	leaq	(%rbp, LDC, 4), %rbp

  673. 

  674. #if !defined(TRMMKERNEL)

  675. 

  676. 	vaddsd 	                (%rax), %xmm12, %xmm4

  677. 	vaddsd 	           (%rax, LDC), %xmm13, %xmm5

  678. 	vaddsd 	                (%rbp), %xmm14, %xmm6

  679. 	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7

  680. 

  681. #endif

  682. 

  683. 	vmovsd	%xmm4 ,  	(%rax)

  684. 	vmovsd	%xmm5 ,  	(%rax, LDC)

  685. 	vmovsd	%xmm6 ,  	(%rbp)

  686. 	vmovsd	%xmm7 ,  	(%rbp, LDC)

  687. 

  688. 	addq	$ 1*SIZE, CO1

  689. .endm

  690. 

  691. 

  692. 

  693. 

  694. /******************************************************************************************/

  695. /******************************************************************************************/

  696. 

  697. .macro INIT4x4

  698. 

  699. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  700. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  701. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  702. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  703. 

  704. .endm

  705. 

  706. .macro KERNEL4x4_I

  707. 	prefetcht0	A_PR1(AO)

  708. 	vmovups		-12 * SIZE(BO), %ymm1

  709. 	vmovups 	-16 * SIZE(AO), %ymm0

  710. 	vmulpd  	%ymm0 ,%ymm1  , %ymm4

  711. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  712. 	vmulpd  	%ymm0 ,%ymm1  , %ymm5

  713. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  714. 	vmulpd  	%ymm0 ,%ymm1  , %ymm6

  715. 

  716. 	addq		$ 4*SIZE, BO

  717. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  718. 	vmulpd  	%ymm0 ,%ymm1  , %ymm7

  719. 	vmovups		-12 * SIZE(BO), %ymm1

  720. 

  721. .endm

  722. 

  723. .macro KERNEL4x4_M1

  724. 	prefetcht0	A_PR1(AO)

  725. 	vmovups 	-16 * SIZE(AO), %ymm0

  726. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  727. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  728. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  729. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  730. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  731. 

  732. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  733. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  734. 	vmovups		-12 * SIZE(BO), %ymm1

  735. 

  736. .endm

  737. 

  738. .macro KERNEL4x4_M2

  739. 	vmovups 	-12 * SIZE(AO), %ymm0

  740. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  741. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  742. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  743. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  744. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  745. 

  746. 	addq		$ 8*SIZE, AO

  747. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  748. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  749. 	vmovups		 -8 * SIZE(BO), %ymm1

  750. 	addq		$ 8*SIZE, BO

  751. .endm

  752. 

  753. 

  754. .macro KERNEL4x4_E

  755. 	vmovups 	-12 * SIZE(AO), %ymm0

  756. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  757. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  758. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  759. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  760. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  761. 

  762. 	addq		$ 8*SIZE, AO

  763. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  764. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  765. 	addq		$ 4*SIZE, BO

  766. .endm

  767. 

  768. .macro KERNEL4x4_SUB

  769. 	vmovups		-12 * SIZE(BO), %ymm1

  770. 	vmovups 	-16 * SIZE(AO), %ymm0

  771. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  772. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  773. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  774. 	addq		$ 4*SIZE, BO

  775. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  776. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  777. 	addq		$ 4*SIZE, AO

  778. 	vpermpd		$ 0xb1, %ymm0  , %ymm0

  779. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  780. 

  781. .endm

  782. 

  783. .macro SAVE4x4

  784. 

  785. 	vbroadcastsd	ALPHA, %ymm0

  786. 

  787. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  788. 	vmulpd	%ymm0 , %ymm7 , %ymm7

  789. 	vmulpd	%ymm0 , %ymm5 , %ymm5

  790. 	vmulpd	%ymm0 , %ymm6 , %ymm6

  791. 

  792. 	vpermpd $ 0xb1 , %ymm5, %ymm5

  793. 	vpermpd $ 0xb1 , %ymm7, %ymm7

  794. 

  795. 	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0

  796. 	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1

  797. 	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2

  798. 	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3

  799. 

  800. 	vpermpd $ 0x1b , %ymm2, %ymm2

  801. 	vpermpd $ 0x1b , %ymm3, %ymm3

  802. 	vpermpd $ 0xb1 , %ymm2, %ymm2

  803. 	vpermpd $ 0xb1 , %ymm3, %ymm3

  804. 

  805. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  806. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  807. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  808. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  809. 

  810.         leaq    (CO1, LDC, 2), %rax     

  811. 	

  812. 

  813. #if !defined(TRMMKERNEL)

  814. 

  815. 	vaddpd 	                (CO1), %ymm4, %ymm4

  816. 	vaddpd 	           (CO1, LDC), %ymm5, %ymm5

  817. 	vaddpd 	               (%rax), %ymm6, %ymm6

  818. 	vaddpd 	          (%rax, LDC), %ymm7, %ymm7

  819. 

  820. #endif

  821. 

  822. 	vmovups	%ymm4 ,  	(CO1)

  823. 	vmovups	%ymm5 ,  	(CO1, LDC)

  824. 	vmovups	%ymm6 ,  	(%rax)

  825. 	vmovups	%ymm7 ,  	(%rax, LDC)

  826. 

  827. 	addq	$ 4*SIZE, CO1

  828. .endm

  829. 

  830. /******************************************************************************************/

  831. /******************************************************************************************/

  832. 

  833. .macro INIT2x4

  834. 

  835. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  836. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  837. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  838. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  839. 

  840. .endm

  841. 

  842. 

  843. .macro KERNEL2x4_SUB

  844. 	vmovddup	-12 * SIZE(BO), %xmm1

  845. 	vmovups 	-16 * SIZE(AO), %xmm0

  846. 	vmovddup	-11 * SIZE(BO), %xmm2

  847. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4

  848. 	vmovddup	-10 * SIZE(BO), %xmm3

  849. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5

  850. 	vmovddup	 -9 * SIZE(BO), %xmm8

  851. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  852. 	addq		$ 4*SIZE, BO

  853. 	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7

  854. 	addq		$ 2*SIZE, AO

  855. 

  856. .endm

  857. 

  858. 

  859. .macro SAVE2x4

  860. 

  861. 	vmovddup	ALPHA, %xmm0

  862. 

  863. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  864. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  865. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  866. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  867. 

  868.         leaq    (CO1, LDC, 2), %rax     

  869. 

  870. #if !defined(TRMMKERNEL)

  871. 

  872. 	vaddpd 	                (CO1), %xmm4, %xmm4

  873. 	vaddpd 	           (CO1, LDC), %xmm5, %xmm5

  874. 	vaddpd 	               (%rax), %xmm6, %xmm6

  875. 	vaddpd 	          (%rax, LDC), %xmm7, %xmm7

  876. 

  877. #endif

  878. 

  879. 	vmovups	%xmm4 ,  	(CO1)

  880. 	vmovups	%xmm5 ,  	(CO1, LDC)

  881. 	vmovups	%xmm6 ,  	(%rax)

  882. 	vmovups	%xmm7 ,  	(%rax, LDC)

  883. 

  884. 	addq	$ 2*SIZE, CO1

  885. .endm

  886. 

  887. /******************************************************************************************/

  888. /******************************************************************************************/

  889. 

  890. .macro INIT1x4

  891. 

  892. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  893. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  894. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  895. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  896. 

  897. .endm

  898. 

  899. 

  900. .macro KERNEL1x4_SUB

  901. 	vmovsd	-12 * SIZE(BO), %xmm1

  902. 	vmovsd 	-16 * SIZE(AO), %xmm0

  903. 	vmovsd	-11 * SIZE(BO), %xmm2

  904. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  905. 	vmovsd	-10 * SIZE(BO), %xmm3

  906. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  907. 	vmovsd	 -9 * SIZE(BO), %xmm8

  908. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6

  909. 	addq		$ 4*SIZE, BO

  910. 	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7

  911. 	addq		$ 1*SIZE, AO

  912. 

  913. .endm

  914. 

  915. 

  916. .macro SAVE1x4

  917. 

  918. 	vmovsd	ALPHA, %xmm0

  919. 

  920. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  921. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  922. 	vmulsd	%xmm0 , %xmm6 , %xmm6

  923. 	vmulsd	%xmm0 , %xmm7 , %xmm7

  924. 

  925.         leaq    (CO1, LDC, 2), %rax     

  926. 

  927. #if !defined(TRMMKERNEL)

  928. 

  929. 	vaddsd 	                (CO1), %xmm4, %xmm4

  930. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  931. 	vaddsd 	               (%rax), %xmm6, %xmm6

  932. 	vaddsd 	          (%rax, LDC), %xmm7, %xmm7

  933. 

  934. #endif

  935. 

  936. 	vmovsd	%xmm4 ,  	(CO1)

  937. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  938. 	vmovsd	%xmm6 ,  	(%rax)

  939. 	vmovsd	%xmm7 ,  	(%rax, LDC)

  940. 

  941. 	addq	$ 1*SIZE, CO1

  942. .endm

  943. 

  944. 

  945. /******************************************************************************************/

  946. /******************************************************************************************/

  947. 

  948. .macro INIT4x2

  949. 

  950. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  951. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  952. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  953. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  954. 

  955. .endm

  956. 

  957. 

  958. .macro KERNEL4x2_SUB

  959. 	vmovddup	-12 * SIZE(BO), %xmm2

  960. 	vmovups 	-16 * SIZE(AO), %xmm0

  961. 	vmovups 	-14 * SIZE(AO), %xmm1

  962. 	vmovddup	-11 * SIZE(BO), %xmm3

  963. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  964. 	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5

  965. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  966. 	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7

  967. 	addq		$ 2*SIZE, BO

  968. 	addq		$ 4*SIZE, AO

  969. 

  970. .endm

  971. 

  972. 

  973. .macro SAVE4x2

  974. 

  975. 	vmovddup	ALPHA, %xmm0

  976. 

  977. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  978. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  979. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  980. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  981. 

  982. 

  983. #if !defined(TRMMKERNEL)

  984. 

  985. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  986. 	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5

  987. 	vaddpd 	                (CO1, LDC), %xmm6, %xmm6

  988. 	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7

  989. 

  990. #endif

  991. 

  992. 	vmovups	%xmm4 ,  	(CO1)

  993. 	vmovups	%xmm5 , 2 * SIZE(CO1)

  994. 	vmovups	%xmm6 ,  	(CO1, LDC)

  995. 	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)

  996. 

  997. 	addq	$ 4*SIZE, CO1

  998. .endm

  999. 

  1000. 

  1001. /******************************************************************************************/

  1002. /******************************************************************************************/

  1003. 

  1004. .macro INIT2x2

  1005. 

  1006. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1007. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1008. 

  1009. .endm

  1010. 

  1011. 

  1012. .macro KERNEL2x2_SUB

  1013. 	vmovddup	-12 * SIZE(BO), %xmm2

  1014. 	vmovups 	-16 * SIZE(AO), %xmm0

  1015. 	vmovddup	-11 * SIZE(BO), %xmm3

  1016. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  1017. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  1018. 	addq		$ 2*SIZE, BO

  1019. 	addq		$ 2*SIZE, AO

  1020. 

  1021. .endm

  1022. 

  1023. 

  1024. .macro SAVE2x2

  1025. 

  1026. 	vmovddup	ALPHA, %xmm0

  1027. 

  1028. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1029. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  1030. 

  1031. 

  1032. #if !defined(TRMMKERNEL)

  1033. 

  1034. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  1035. 	vaddpd 	                (CO1, LDC), %xmm6, %xmm6

  1036. 

  1037. #endif

  1038. 

  1039. 	vmovups	%xmm4 ,  	(CO1)

  1040. 	vmovups	%xmm6 ,  	(CO1, LDC)

  1041. 

  1042. 	addq	$ 2*SIZE, CO1

  1043. .endm

  1044. 

  1045. /******************************************************************************************/

  1046. /******************************************************************************************/

  1047. 

  1048. .macro INIT1x2

  1049. 

  1050. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1051. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1052. 

  1053. .endm

  1054. 

  1055. 

  1056. .macro KERNEL1x2_SUB

  1057. 	vmovsd	-12 * SIZE(BO), %xmm1

  1058. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1059. 	vmovsd	-11 * SIZE(BO), %xmm2

  1060. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1061. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  1062. 	addq		$ 2*SIZE, BO

  1063. 	addq		$ 1*SIZE, AO

  1064. 

  1065. .endm

  1066. 

  1067. 

  1068. .macro SAVE1x2

  1069. 

  1070. 	vmovsd	ALPHA, %xmm0

  1071. 

  1072. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1073. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  1074. 

  1075. 

  1076. #if !defined(TRMMKERNEL)

  1077. 

  1078. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1079. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  1080. 

  1081. #endif

  1082. 

  1083. 	vmovsd	%xmm4 ,  	(CO1)

  1084. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  1085. 

  1086. 	addq	$ 1*SIZE, CO1

  1087. .endm

  1088. 

  1089. 

  1090. /******************************************************************************************/

  1091. /******************************************************************************************/

  1092. 

  1093. .macro INIT4x1

  1094. 

  1095. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  1096. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  1097. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  1098. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  1099. 

  1100. .endm

  1101. 

  1102. 

  1103. .macro KERNEL4x1

  1104. 

  1105. 	vbroadcastsd	-12 * SIZE(BO), %ymm0

  1106. 	vbroadcastsd	-11 * SIZE(BO), %ymm1

  1107. 	vbroadcastsd	-10 * SIZE(BO), %ymm2

  1108. 	vbroadcastsd	-9  * SIZE(BO), %ymm3

  1109. 

  1110. 	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4

  1111. 	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5

  1112. 

  1113. 	vbroadcastsd	-8  * SIZE(BO), %ymm0

  1114. 	vbroadcastsd	-7  * SIZE(BO), %ymm1

  1115. 

  1116. 	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6

  1117. 	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7

  1118. 

  1119. 	vbroadcastsd	-6  * SIZE(BO), %ymm2

  1120. 	vbroadcastsd	-5  * SIZE(BO), %ymm3

  1121. 

  1122. 	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4

  1123. 	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5

  1124. 	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6

  1125. 	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7

  1126. 

  1127. 	addq		$ 8 *SIZE, BO

  1128. 	addq		$ 32*SIZE, AO

  1129. 

  1130. .endm

  1131. 

  1132. 

  1133. .macro KERNEL4x1_SUB

  1134. 	vbroadcastsd	-12 * SIZE(BO), %ymm2

  1135. 	vmovups 	-16 * SIZE(AO), %ymm0

  1136. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4

  1137. 	addq		$ 1*SIZE, BO

  1138. 	addq		$ 4*SIZE, AO

  1139. 

  1140. .endm

  1141. 

  1142. 

  1143. .macro SAVE4x1

  1144. 

  1145. 	vbroadcastsd	ALPHA, %ymm0

  1146. 

  1147. 	vaddpd	%ymm4,%ymm5, %ymm4 

  1148. 	vaddpd	%ymm6,%ymm7, %ymm6 

  1149. 	vaddpd	%ymm4,%ymm6, %ymm4 

  1150. 

  1151. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  1152. 

  1153. 

  1154. #if !defined(TRMMKERNEL)

  1155. 

  1156. 	vaddpd 	                (CO1)     , %ymm4, %ymm4

  1157. 

  1158. #endif

  1159. 

  1160. 	vmovups	%ymm4 ,  	(CO1)

  1161. 

  1162. 	addq	$ 4*SIZE, CO1

  1163. .endm

  1164. 

  1165. 

  1166. /******************************************************************************************/

  1167. /******************************************************************************************/

  1168. 

  1169. .macro INIT2x1

  1170. 

  1171. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1172. 

  1173. .endm

  1174. 

  1175. 

  1176. .macro KERNEL2x1_SUB

  1177. 	vmovddup	-12 * SIZE(BO), %xmm2

  1178. 	vmovups 	-16 * SIZE(AO), %xmm0

  1179. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  1180. 	addq		$ 1*SIZE, BO

  1181. 	addq		$ 2*SIZE, AO

  1182. 

  1183. .endm

  1184. 

  1185. 

  1186. .macro SAVE2x1

  1187. 

  1188. 	vmovddup	ALPHA, %xmm0

  1189. 

  1190. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1191. 

  1192. 

  1193. #if !defined(TRMMKERNEL)

  1194. 

  1195. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  1196. 

  1197. #endif

  1198. 

  1199. 	vmovups	%xmm4 ,  	(CO1)

  1200. 

  1201. 	addq	$ 2*SIZE, CO1

  1202. .endm

  1203. 

  1204. 

  1205. /******************************************************************************************/

  1206. /******************************************************************************************/

  1207. 

  1208. .macro INIT1x1

  1209. 

  1210. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1211. 

  1212. .endm

  1213. 

  1214. 

  1215. .macro KERNEL1x1_SUB

  1216. 	vmovsd	-12 * SIZE(BO), %xmm1

  1217. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1218. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1219. 	addq		$ 1*SIZE, BO

  1220. 	addq		$ 1*SIZE, AO

  1221. 

  1222. .endm

  1223. 

  1224. 

  1225. .macro SAVE1x1

  1226. 

  1227. 	vmovsd	ALPHA, %xmm0

  1228. 

  1229. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1230. 

  1231. 

  1232. #if !defined(TRMMKERNEL)

  1233. 

  1234. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1235. 

  1236. #endif

  1237. 

  1238. 	vmovsd	%xmm4 ,  	(CO1)

  1239. 

  1240. 	addq	$ 1*SIZE, CO1

  1241. .endm

  1242. 

  1243. 

  1244. /*******************************************************************************************/

  1245. 

  1246. #if !defined(TRMMKERNEL)

  1247. 

  1248. 

  1249. 	PROLOGUE

  1250. 	PROFCODE

  1251. 	

  1252. 	subq	$STACKSIZE, %rsp

  1253. 	movq	%rbx,   (%rsp)

  1254. 	movq	%rbp,  8(%rsp)

  1255. 	movq	%r12, 16(%rsp)

  1256. 	movq	%r13, 24(%rsp)

  1257. 	movq	%r14, 32(%rsp)

  1258. 	movq	%r15, 40(%rsp)

  1259. 

  1260. 	vzeroupper

  1261. 

  1262. #ifdef WINDOWS_ABI

  1263. 	movq	%rdi,    48(%rsp)

  1264. 	movq	%rsi,    56(%rsp)

  1265. 	vmovups	%xmm6,   64(%rsp)

  1266. 	vmovups	%xmm7,   80(%rsp)

  1267. 	vmovups	%xmm8,   96(%rsp)

  1268. 	vmovups	%xmm9,  112(%rsp)

  1269. 	vmovups	%xmm10, 128(%rsp)

  1270. 	vmovups	%xmm11, 144(%rsp)

  1271. 	vmovups	%xmm12, 160(%rsp)

  1272. 	vmovups	%xmm13, 176(%rsp)

  1273. 	vmovups	%xmm14, 192(%rsp)

  1274. 	vmovups	%xmm15, 208(%rsp)

  1275. 

  1276. 	movq	ARG1,      OLD_M

  1277. 	movq	ARG2,      OLD_N

  1278. 	movq	ARG3,      OLD_K

  1279. 	movq	OLD_A,     A

  1280. 	movq	OLD_B,     B

  1281. 	movq	OLD_C,     C

  1282. 	movq	OLD_LDC,   LDC

  1283. 

  1284. 	vmovups	%xmm3, %xmm0

  1285. 

  1286. #else

  1287. 	movq	STACKSIZE +  8(%rsp), LDC

  1288. 

  1289. #endif

  1290. 

  1291. 	movq    %rsp, SP      # save old stack

  1292.         subq    $128 + L_BUFFER_SIZE, %rsp

  1293.         andq    $-4096, %rsp    # align stack

  1294. 

  1295.         STACK_TOUCH

  1296. 

  1297. 	cmpq	$ 0, OLD_M

  1298. 	je	.L999

  1299. 

  1300. 	cmpq	$ 0, OLD_N

  1301. 	je	.L999

  1302. 

  1303. 	cmpq	$ 0, OLD_K

  1304. 	je	.L999

  1305. 

  1306. 	movq	OLD_M, M

  1307. 	movq	OLD_N, N

  1308. 	movq	OLD_K, K

  1309. 

  1310. 	vmovsd	 %xmm0, ALPHA

  1311. 

  1312. 	salq	$BASE_SHIFT, LDC

  1313. 

  1314. 	movq    N, %rax

  1315.         xorq    %rdx, %rdx

  1316.         movq    $12,  %rdi

  1317.         divq    %rdi                    //    N / 12

  1318.         movq    %rax, Ndiv12             //    N / 12

  1319.         movq    %rdx, Nmod12             //    N % 12

  1320. 

  1321. 

  1322. 	movq	Ndiv12,  J

  1323. 	cmpq	$ 0, J

  1324. 	je	.L4_0

  1325. 	ALIGN_4

  1326. 

  1327. .L12_01:

  1328.         // copy to sub buffer

  1329.         movq    K, %rax

  1330.         salq    $2,%rax                 // K * 4 ; read 2 values

  1331.         movq    B, BO1

  1332.         leaq    (B,%rax, SIZE), BO2     // next offset to BO2

  1333.         leaq    (BO2,%rax, SIZE), BO3     // next offset to BO2

  1334. 

  1335. 

  1336.         leaq    BUFFER1, BO             // first buffer to BO

  1337.         movq    K, %rax

  1338. 	sarq	$1 , %rax		// K / 2

  1339. 	jz	.L12_01a_2

  1340.         ALIGN_4

  1341. 

  1342. .L12_01a_1:

  1343. 

  1344.         prefetcht0 512(BO1)

  1345.         prefetcht0 512(BO2)

  1346.         prefetcht0 512(BO3)

  1347.         prefetchw  512(BO)

  1348. 

  1349. 

  1350. 	vmovups	0 * SIZE(BO1), %ymm1

  1351. 	vmovups	4 * SIZE(BO1), %ymm5

  1352. 	vmovups	0 * SIZE(BO2), %ymm2

  1353. 	vmovups	4 * SIZE(BO2), %ymm6

  1354. 	vmovups	0 * SIZE(BO3), %ymm3

  1355. 	vmovups	4 * SIZE(BO3), %ymm7

  1356. 

  1357. 	vmovups	%ymm1,  0 * SIZE(BO)

  1358. 	vmovups	%ymm2,  4 * SIZE(BO)

  1359. 	vmovups	%ymm3,  8 * SIZE(BO)

  1360. 

  1361. 	vmovups	%ymm5, 12 * SIZE(BO)

  1362. 	vmovups	%ymm6, 16 * SIZE(BO)

  1363. 	vmovups	%ymm7, 20 * SIZE(BO)

  1364. 

  1365. 	addq	$ 8 * SIZE ,BO1

  1366. 	addq	$ 8 * SIZE ,BO2

  1367. 	addq	$ 8 * SIZE ,BO3

  1368. 	addq    $ 24 *SIZE ,BO

  1369. 

  1370. 	decq	%rax

  1371. 	jnz	.L12_01a_1

  1372. 

  1373. 

  1374. 

  1375. .L12_01a_2:

  1376. 

  1377. 	movq    K, %rax

  1378.         andq    $1, %rax                // K % 2

  1379.         jz      .L12_03c

  1380.         ALIGN_4

  1381. 

  1382. 

  1383. .L12_02b:

  1384. 

  1385. 	vmovups	0 * SIZE(BO1), %ymm1

  1386. 	vmovups	0 * SIZE(BO2), %ymm2

  1387. 	vmovups	0 * SIZE(BO3), %ymm3

  1388. 	vmovups	%ymm1, 0 * SIZE(BO)

  1389. 	vmovups	%ymm2, 4 * SIZE(BO)

  1390. 	vmovups	%ymm3, 8 * SIZE(BO)

  1391. 	addq	$ 4*SIZE,BO1

  1392. 	addq	$ 4*SIZE,BO2

  1393. 	addq	$ 4*SIZE,BO3

  1394. 	addq	$ 12*SIZE,BO

  1395. 	decq	%rax

  1396. 	jnz	.L12_02b

  1397. 

  1398. .L12_03c:

  1399. 

  1400. 	movq	BO3, B			// next offset of B

  1401. 

  1402. .L12_10:

  1403. 	movq	C, CO1

  1404. 	leaq	(C, LDC, 8), C		 

  1405. 	leaq	(C, LDC, 4), C		// c += 12 * ldc

  1406. 

  1407. 	

  1408. 	movq	A, AO		 	// aoffset = a

  1409. 	addq	$16 * SIZE, AO

  1410. 

  1411. 	movq	M,  I

  1412. 	sarq	$2, I			// i = m / 4

  1413. 	je	.L12_20

  1414. 

  1415. 	ALIGN_4

  1416. 

  1417. .L12_11:

  1418.         leaq    BUFFER1, BO             // first buffer to BO

  1419.         addq    $12 * SIZE, BO

  1420. 

  1421.         movq    K, %rax

  1422. 

  1423. 	sarq $3, %rax			//  K / 8

  1424. 	cmpq $2, %rax

  1425. 

  1426. 	jl	.L12_13

  1427. 

  1428. 

  1429. 	KERNEL4x12_I

  1430. 	KERNEL4x12_M2

  1431. 	KERNEL4x12_M1

  1432. 	KERNEL4x12_M2

  1433. 

  1434. 	KERNEL4x12_M1

  1435. 	KERNEL4x12_M2

  1436. 	KERNEL4x12_M1

  1437. 	KERNEL4x12_M2

  1438. 

  1439. 	subq $2, %rax

  1440. 	je	.L12_12a

  1441. 

  1442. 	ALIGN_5

  1443. .L12_12:

  1444. 

  1445. 	KERNEL4x12_M1

  1446. 	KERNEL4x12_M2

  1447. 	KERNEL4x12_M1

  1448. 	KERNEL4x12_M2

  1449. 

  1450. 	KERNEL4x12_M1

  1451. 	KERNEL4x12_M2

  1452. 	KERNEL4x12_M1

  1453. 	KERNEL4x12_M2

  1454. 

  1455. 	dec	%rax

  1456. 	jne	.L12_12

  1457. 

  1458. .L12_12a:

  1459. 

  1460. 	KERNEL4x12_M1

  1461. 	KERNEL4x12_M2

  1462. 	KERNEL4x12_M1

  1463. 	KERNEL4x12_M2

  1464. 

  1465. 	KERNEL4x12_M1

  1466. 	KERNEL4x12_M2

  1467. 	KERNEL4x12_M1

  1468. 	KERNEL4x12_E

  1469. 

  1470. 	jmp .L12_16

  1471. 

  1472. 

  1473. .L12_13:

  1474. 

  1475. 	test $1, %rax

  1476. 	jz .L12_14

  1477. 

  1478. 	KERNEL4x12_I

  1479. 	KERNEL4x12_M2

  1480. 	KERNEL4x12_M1

  1481. 	KERNEL4x12_M2

  1482. 

  1483. 	KERNEL4x12_M1

  1484. 	KERNEL4x12_M2

  1485. 	KERNEL4x12_M1

  1486. 	KERNEL4x12_E

  1487. 

  1488. 	jmp .L12_16

  1489. 

  1490. 

  1491. .L12_14:

  1492. 

  1493. 	INIT4x12

  1494. 

  1495. 

  1496. .L12_16:

  1497.         movq    K, %rax

  1498. 

  1499. 	andq	$7, %rax		# if (k & 1)

  1500. 	je .L12_19

  1501. 

  1502. 	ALIGN_4

  1503. 

  1504. .L12_17:

  1505. 

  1506. 	KERNEL4x12_SUB

  1507. 

  1508. 	dec	%rax

  1509. 	jne	.L12_17

  1510. 	ALIGN_4

  1511. 

  1512. 

  1513. .L12_19:

  1514. 

  1515. 	SAVE4x12

  1516. 

  1517. 	decq	I			# i --

  1518. 	jne	.L12_11

  1519. 	ALIGN_4	

  1520. 

  1521. /**************************************************************************

  1522. * Rest of M 

  1523. ***************************************************************************/

  1524. .L12_20:

  1525. 	// Test rest of M

  1526. 

  1527. 	testq	$3, M

  1528. 	jz	.L12_100			// to next 16 lines of N

  1529. 

  1530. 

  1531. .L12_30:

  1532. 	testq	$2, M		

  1533. 	jz	.L12_40

  1534. 

  1535. 	ALIGN_4

  1536. 

  1537. .L12_31:

  1538.         leaq    BUFFER1, BO             // first buffer to BO

  1539.         addq    $12 * SIZE, BO

  1540. 

  1541. 	INIT2x12

  1542. 

  1543.         movq    K, %rax

  1544. 

  1545. 	sarq	$3, %rax

  1546. 	je	.L12_36

  1547. 	ALIGN_4

  1548. 

  1549. .L12_32:

  1550. 

  1551. 	KERNEL2x12_SUB

  1552. 	KERNEL2x12_SUB

  1553. 	KERNEL2x12_SUB

  1554. 	KERNEL2x12_SUB

  1555. 

  1556. 	KERNEL2x12_SUB

  1557. 	KERNEL2x12_SUB

  1558. 	KERNEL2x12_SUB

  1559. 	KERNEL2x12_SUB

  1560. 

  1561. 	dec %rax

  1562. 	jne	.L12_32

  1563. 	ALIGN_4

  1564. 

  1565. .L12_36:

  1566.         movq    K, %rax

  1567. 

  1568. 	andq	$7, %rax		# if (k & 1)

  1569. 	je .L12_39

  1570. 

  1571. 	ALIGN_4

  1572. 

  1573. .L12_37:

  1574. 

  1575. 	KERNEL2x12_SUB

  1576. 

  1577. 	dec %rax

  1578. 	jne	.L12_37

  1579. 	ALIGN_4

  1580. 

  1581. 

  1582. .L12_39:

  1583. 

  1584. 	SAVE2x12

  1585. 

  1586. 	ALIGN_4

  1587. 

  1588. .L12_40:

  1589. 	testq	$1, M		

  1590. 	jz	.L12_100		// to next 3 lines of N

  1591. 

  1592. 	ALIGN_4

  1593. 

  1594. .L12_41:

  1595.         leaq    BUFFER1, BO             // first buffer to BO

  1596.         addq    $12 * SIZE, BO

  1597. 

  1598. 	INIT1x12

  1599. 

  1600.         movq    K, %rax

  1601. 

  1602. 	sarq	$3,%rax

  1603. 	je	.L12_46

  1604. 

  1605. 	ALIGN_4

  1606. 

  1607. .L12_42:

  1608. 

  1609. 	KERNEL1x12_SUB

  1610. 	KERNEL1x12_SUB

  1611. 	KERNEL1x12_SUB

  1612. 	KERNEL1x12_SUB

  1613. 

  1614. 	KERNEL1x12_SUB

  1615. 	KERNEL1x12_SUB

  1616. 	KERNEL1x12_SUB

  1617. 	KERNEL1x12_SUB

  1618. 

  1619. 

  1620. 	dec %rax

  1621. 	jne	.L12_42

  1622. 	ALIGN_4

  1623. 

  1624. .L12_46:

  1625.         movq    K, %rax

  1626. 

  1627. 	andq	$7, %rax		# if (k & 1)

  1628. 	je .L12_49

  1629. 

  1630. 	ALIGN_4

  1631. 

  1632. .L12_47:

  1633. 

  1634. 	KERNEL1x12_SUB

  1635. 

  1636. 	dec	%rax

  1637. 	jne	.L12_47

  1638. 	ALIGN_4

  1639. 

  1640. 

  1641. .L12_49:

  1642. 

  1643. 	SAVE1x12

  1644. 

  1645. 	ALIGN_4

  1646. 	

  1647. .L12_100:

  1648. 

  1649. 	decq	J			// j --

  1650. 	jg	.L12_01

  1651. 

  1652. 

  1653. .L4_0:

  1654. 

  1655. 	cmpq	$ 0, Nmod12		// N % 12 == 0

  1656. 	je	.L999

  1657. 

  1658. 	movq	Nmod12, J		

  1659. 	sarq	$2, J			// j = j / 4

  1660. 	je	.L2_0

  1661. 

  1662. .L4_10:

  1663. 	movq	C, CO1

  1664. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  1665. 

  1666. 	

  1667. 	movq	A, AO		 	// aoffset = a

  1668. 	addq	$16 * SIZE, AO

  1669. 

  1670. 	movq	M,  I

  1671. 	sarq	$2, I			// i = m / 4

  1672. 	je	.L4_20

  1673. 

  1674. 	ALIGN_4

  1675. 

  1676. .L4_11:

  1677.         movq    B, BO        

  1678.         addq    $12 * SIZE, BO

  1679. 

  1680.         movq    K, %rax

  1681. 

  1682. 	sarq	$3, %rax			//  K / 8

  1683. 	cmpq    $2, %rax

  1684. 	jl	.L4_13

  1685. 

  1686. 

  1687. 	KERNEL4x4_I

  1688. 	KERNEL4x4_M2

  1689. 	KERNEL4x4_M1

  1690. 	KERNEL4x4_M2

  1691. 

  1692. 	KERNEL4x4_M1

  1693. 	KERNEL4x4_M2

  1694. 	KERNEL4x4_M1

  1695. 	KERNEL4x4_M2

  1696. 

  1697. 	subq $2, %rax

  1698. 	je	.L4_12a

  1699. 

  1700. 	ALIGN_5

  1701. 

  1702. .L4_12:

  1703. 

  1704. 	KERNEL4x4_M1

  1705. 	KERNEL4x4_M2

  1706. 	KERNEL4x4_M1

  1707. 	KERNEL4x4_M2

  1708. 

  1709. 	KERNEL4x4_M1

  1710. 	KERNEL4x4_M2

  1711. 	KERNEL4x4_M1

  1712. 	KERNEL4x4_M2

  1713. 

  1714. 	dec	%rax

  1715. 	jne	.L4_12

  1716. 

  1717. .L4_12a:

  1718. 

  1719. 	KERNEL4x4_M1

  1720. 	KERNEL4x4_M2

  1721. 	KERNEL4x4_M1

  1722. 	KERNEL4x4_M2

  1723. 

  1724. 	KERNEL4x4_M1

  1725. 	KERNEL4x4_M2

  1726. 	KERNEL4x4_M1

  1727. 	KERNEL4x4_E

  1728. 

  1729. 	jmp .L4_16

  1730. 

  1731. 

  1732. .L4_13:

  1733. 

  1734. 	test $1, %rax

  1735. 	jz .L4_14

  1736. 

  1737. 	KERNEL4x4_I

  1738. 	KERNEL4x4_M2

  1739. 	KERNEL4x4_M1

  1740. 	KERNEL4x4_M2

  1741. 

  1742. 	KERNEL4x4_M1

  1743. 	KERNEL4x4_M2

  1744. 	KERNEL4x4_M1

  1745. 	KERNEL4x4_E

  1746. 

  1747. 	jmp .L4_16

  1748. 

  1749. 

  1750. .L4_14:

  1751. 

  1752. 	INIT4x4

  1753. 

  1754. 

  1755. .L4_16:

  1756.         movq    K, %rax

  1757. 

  1758. 	andq	$7, %rax		# if (k & 1)

  1759. 	je .L4_19

  1760. 

  1761. 	ALIGN_4

  1762. 

  1763. .L4_17:

  1764. 

  1765. 	KERNEL4x4_SUB

  1766. 

  1767. 	dec	%rax

  1768. 	jne	.L4_17

  1769. 	ALIGN_4

  1770. 

  1771. 

  1772. .L4_19:

  1773. 

  1774. 	SAVE4x4

  1775. 

  1776. 	decq	I			# i --

  1777. 	jg	.L4_11

  1778. 	ALIGN_4	

  1779. 

  1780. /**************************************************************************

  1781. * Rest of M 

  1782. ***************************************************************************/

  1783. .L4_20:

  1784. 	// Test rest of M

  1785. 

  1786. 	testq	$3, M

  1787. 	jz	.L4_100			// to next 16 lines of N

  1788. 

  1789. 

  1790. .L4_30:

  1791. 	testq	$2, M		

  1792. 	jz	.L4_40

  1793. 

  1794. 	ALIGN_4

  1795. 

  1796. .L4_31:

  1797.         movq    B, BO             // first buffer to BO

  1798.         addq    $12 * SIZE, BO

  1799. 

  1800. 	INIT2x4

  1801. 

  1802.         movq    K, %rax

  1803. 

  1804. 	sarq	$3, %rax

  1805. 	je	.L4_36

  1806. 	ALIGN_4

  1807. 

  1808. .L4_32:

  1809. 

  1810. 	KERNEL2x4_SUB

  1811. 	KERNEL2x4_SUB

  1812. 	KERNEL2x4_SUB

  1813. 	KERNEL2x4_SUB

  1814. 

  1815. 	KERNEL2x4_SUB

  1816. 	KERNEL2x4_SUB

  1817. 	KERNEL2x4_SUB

  1818. 	KERNEL2x4_SUB

  1819. 

  1820. 	dec %rax

  1821. 	jne	.L4_32

  1822. 	ALIGN_4

  1823. 

  1824. .L4_36:

  1825.         movq    K, %rax

  1826. 

  1827. 	andq	$7, %rax		# if (k & 1)

  1828. 	je .L4_39

  1829. 

  1830. 	ALIGN_4

  1831. 

  1832. .L4_37:

  1833. 

  1834. 	KERNEL2x4_SUB

  1835. 

  1836. 	dec %rax

  1837. 	jne	.L4_37

  1838. 

  1839. 

  1840. .L4_39:

  1841. 

  1842. 	SAVE2x4

  1843. 

  1844. .L4_40:

  1845. 	testq	$1, M		

  1846. 	jz	.L4_100		// to next 3 lines of N

  1847. 

  1848. 	ALIGN_4

  1849. 

  1850. .L4_41:

  1851.         movq    B, BO             // first buffer to BO

  1852.         addq    $12 * SIZE, BO

  1853. 

  1854. 	INIT1x4

  1855. 

  1856.         movq    K, %rax

  1857. 

  1858. 	sarq	$3,%rax

  1859. 	je	.L4_46

  1860. 

  1861. 	ALIGN_4

  1862. 

  1863. .L4_42:

  1864. 

  1865. 	KERNEL1x4_SUB

  1866. 	KERNEL1x4_SUB

  1867. 	KERNEL1x4_SUB

  1868. 	KERNEL1x4_SUB

  1869. 

  1870. 	KERNEL1x4_SUB

  1871. 	KERNEL1x4_SUB

  1872. 	KERNEL1x4_SUB

  1873. 	KERNEL1x4_SUB

  1874. 

  1875. 	dec %rax

  1876. 	jne	.L4_42

  1877. 	ALIGN_4

  1878. 

  1879. .L4_46:

  1880.         movq    K, %rax

  1881. 

  1882. 	andq	$7, %rax		# if (k & 1)

  1883. 	je .L4_49

  1884. 

  1885. 	ALIGN_4

  1886. 

  1887. .L4_47:

  1888. 

  1889. 	KERNEL1x4_SUB

  1890. 

  1891. 	dec	%rax

  1892. 	jne	.L4_47

  1893. 	ALIGN_4

  1894. 

  1895. 

  1896. .L4_49:

  1897. 

  1898. 	SAVE1x4

  1899. 

  1900. 	ALIGN_4

  1901. 	

  1902. .L4_100:

  1903. 

  1904. 	movq	K, %rax

  1905. 	salq	$2, %rax		// * 4

  1906. 	leaq	(B , %rax, SIZE), B

  1907. 	decq	J			// j --

  1908. 	jg	.L4_10

  1909. 

  1910. 

  1911. 

  1912. 

  1913. /***************************************************************************************************************/

  1914. 

  1915. .L2_0:

  1916. 

  1917. 	movq	Nmod12, J		

  1918. 	testq	$2, J

  1919. 	je	.L1_0

  1920. 

  1921. .L2_10:

  1922. 	movq	C, CO1

  1923. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  1924. 

  1925. 	

  1926. 	movq	A, AO		 	// aoffset = a

  1927. 	addq	$16 * SIZE, AO

  1928. 

  1929. 	movq	M,  I

  1930. 	sarq	$2, I			// i = m / 4

  1931. 	je	.L2_20

  1932. 

  1933. 	ALIGN_4

  1934. 

  1935. .L2_11:

  1936.         movq    B, BO        

  1937.         addq    $12 * SIZE, BO

  1938. 

  1939. 	INIT4x2

  1940. 

  1941.         movq    K, %rax

  1942. 	sarq $3, %rax			//  K / 8

  1943. 

  1944. 	je	.L2_16

  1945. 

  1946. 	ALIGN_5

  1947. 

  1948. .L2_12:

  1949. 

  1950. 	KERNEL4x2_SUB

  1951. 	KERNEL4x2_SUB

  1952. 	KERNEL4x2_SUB

  1953. 	KERNEL4x2_SUB

  1954. 

  1955. 	KERNEL4x2_SUB

  1956. 	KERNEL4x2_SUB

  1957. 	KERNEL4x2_SUB

  1958. 	KERNEL4x2_SUB

  1959. 

  1960. 	dec	%rax

  1961. 	jne	.L2_12

  1962. 

  1963. 

  1964. .L2_16:

  1965.         movq    K, %rax

  1966. 

  1967. 	andq	$7, %rax		# if (k & 1)

  1968. 	je .L2_19

  1969. 

  1970. 	ALIGN_4

  1971. 

  1972. .L2_17:

  1973. 

  1974. 	KERNEL4x2_SUB

  1975. 

  1976. 	dec	%rax

  1977. 	jne	.L2_17

  1978. 	ALIGN_4

  1979. 

  1980. 

  1981. .L2_19:

  1982. 

  1983. 	SAVE4x2

  1984. 

  1985. 	decq	I			# i --

  1986. 	jg	.L2_11

  1987. 	ALIGN_4	

  1988. 

  1989. /**************************************************************************

  1990. * Rest of M 

  1991. ***************************************************************************/

  1992. .L2_20:

  1993. 	// Test rest of M

  1994. 

  1995. 	testq	$3, M

  1996. 	jz	.L2_100			// to next 16 lines of N

  1997. 

  1998. 

  1999. .L2_30:

  2000. 	testq	$2, M		

  2001. 	jz	.L2_40

  2002. 

  2003. 	ALIGN_4

  2004. 

  2005. .L2_31:

  2006.         movq    B, BO             // first buffer to BO

  2007.         addq    $12 * SIZE, BO

  2008. 

  2009. 	INIT2x2

  2010. 

  2011.         movq    K, %rax

  2012. 

  2013. 	sarq	$3, %rax

  2014. 	je	.L2_36

  2015. 	ALIGN_4

  2016. 

  2017. .L2_32:

  2018. 

  2019. 	KERNEL2x2_SUB

  2020. 	KERNEL2x2_SUB

  2021. 	KERNEL2x2_SUB

  2022. 	KERNEL2x2_SUB

  2023. 

  2024. 	KERNEL2x2_SUB

  2025. 	KERNEL2x2_SUB

  2026. 	KERNEL2x2_SUB

  2027. 	KERNEL2x2_SUB

  2028. 

  2029. 	dec %rax

  2030. 	jne	.L2_32

  2031. 

  2032. .L2_36:

  2033.         movq    K, %rax

  2034. 

  2035. 	andq	$7, %rax		# if (k & 1)

  2036. 	je .L2_39

  2037. 

  2038. 	ALIGN_4

  2039. 

  2040. .L2_37:

  2041. 

  2042. 	KERNEL2x2_SUB

  2043. 

  2044. 	dec %rax

  2045. 	jne	.L2_37

  2046. 

  2047. 

  2048. .L2_39:

  2049. 

  2050. 	SAVE2x2

  2051. 

  2052. .L2_40:

  2053. 	testq	$1, M		

  2054. 	jz	.L2_100		// to next 3 lines of N

  2055. 

  2056. .L2_41:

  2057.         movq    B, BO             // first buffer to BO

  2058.         addq    $12 * SIZE, BO

  2059. 

  2060. 	INIT1x2

  2061. 

  2062.         movq    K, %rax

  2063. 

  2064. 	sarq	$3,%rax

  2065. 	je	.L2_46

  2066. 

  2067. 	ALIGN_4

  2068. 

  2069. .L2_42:

  2070. 

  2071. 	KERNEL1x2_SUB

  2072. 	KERNEL1x2_SUB

  2073. 	KERNEL1x2_SUB

  2074. 	KERNEL1x2_SUB

  2075. 

  2076. 	KERNEL1x2_SUB

  2077. 	KERNEL1x2_SUB

  2078. 	KERNEL1x2_SUB

  2079. 	KERNEL1x2_SUB

  2080. 

  2081. 	dec %rax

  2082. 	jne	.L2_42

  2083. 

  2084. .L2_46:

  2085.         movq    K, %rax

  2086. 

  2087. 	andq	$7, %rax		# if (k & 1)

  2088. 	je .L2_49

  2089. 

  2090. 	ALIGN_4

  2091. 

  2092. .L2_47:

  2093. 

  2094. 	KERNEL1x2_SUB

  2095. 

  2096. 	dec	%rax

  2097. 	jne	.L2_47

  2098. 

  2099. .L2_49:

  2100. 

  2101. 	SAVE1x2

  2102. 

  2103. .L2_100:

  2104. 

  2105. 	movq	K, %rax

  2106. 	salq	$1, %rax		// * 2

  2107. 	leaq	(B , %rax, SIZE), B

  2108. 

  2109. /***************************************************************************************************************/

  2110. 

  2111. .L1_0:

  2112. 

  2113. 	movq	Nmod12, J		

  2114. 	testq	$1, J

  2115. 	je	.L999

  2116. 

  2117. .L1_10:

  2118. 	movq	C, CO1

  2119. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  2120. 

  2121. 	

  2122. 	movq	A, AO		 	// aoffset = a

  2123. 	addq	$16 * SIZE, AO

  2124. 

  2125. 	movq	M,  I

  2126. 	sarq	$2, I			// i = m / 4

  2127. 	je	.L1_20

  2128. 

  2129. 	ALIGN_4

  2130. 

  2131. .L1_11:

  2132.         movq    B, BO        

  2133.         addq    $12 * SIZE, BO

  2134. 

  2135. 	INIT4x1

  2136. 

  2137.         movq    K, %rax

  2138. 

  2139. 	sarq	$3, %rax			//  K / 8

  2140. 	je	.L1_16

  2141. 

  2142. 	ALIGN_5

  2143. 

  2144. .L1_12:

  2145. 

  2146. 	KERNEL4x1

  2147. 

  2148. 	dec	%rax

  2149. 	jne	.L1_12

  2150. 

  2151. 

  2152. .L1_16:

  2153.         movq    K, %rax

  2154. 

  2155. 	andq	$7, %rax		# if (k & 1)

  2156. 	je .L1_19

  2157. 

  2158. 	ALIGN_4

  2159. 

  2160. .L1_17:

  2161. 

  2162. 	KERNEL4x1_SUB

  2163. 

  2164. 	dec	%rax

  2165. 	jne	.L1_17

  2166. 	ALIGN_4

  2167. 

  2168. 

  2169. .L1_19:

  2170. 

  2171. 	SAVE4x1

  2172. 

  2173. 	decq	I			# i --

  2174. 	jg	.L1_11

  2175. 

  2176. /**************************************************************************

  2177. * Rest of M 

  2178. ***************************************************************************/

  2179. .L1_20:

  2180. 	// Test rest of M

  2181. 

  2182. 	testq	$3, M

  2183. 	jz	.L1_100	

  2184. 

  2185. 

  2186. .L1_30:

  2187. 	testq	$2, M		

  2188. 	jz	.L1_40

  2189. 

  2190. 	ALIGN_4

  2191. 

  2192. .L1_31:

  2193.         movq    B, BO             // first buffer to BO

  2194.         addq    $12 * SIZE, BO

  2195. 

  2196. 	INIT2x1

  2197. 

  2198.         movq    K, %rax

  2199. 

  2200. 	sarq	$3, %rax

  2201. 	je	.L1_36

  2202. 	ALIGN_4

  2203. 

  2204. .L1_32:

  2205. 

  2206. 	KERNEL2x1_SUB

  2207. 	KERNEL2x1_SUB

  2208. 	KERNEL2x1_SUB

  2209. 	KERNEL2x1_SUB

  2210. 

  2211. 	KERNEL2x1_SUB

  2212. 	KERNEL2x1_SUB

  2213. 	KERNEL2x1_SUB

  2214. 	KERNEL2x1_SUB

  2215. 

  2216. 

  2217. 	dec %rax

  2218. 	jne	.L1_32

  2219. 

  2220. .L1_36:

  2221.         movq    K, %rax

  2222. 

  2223. 	andq	$7, %rax		# if (k & 1)

  2224. 	je .L1_39

  2225. 

  2226. 	ALIGN_4

  2227. 

  2228. .L1_37:

  2229. 

  2230. 	KERNEL2x1_SUB

  2231. 

  2232. 	dec %rax

  2233. 	jne	.L1_37

  2234. 

  2235. .L1_39:

  2236. 

  2237. 	SAVE2x1

  2238. 

  2239. .L1_40:

  2240. 	testq	$1, M		

  2241. 	jz	.L1_100		// to next 3 lines of N

  2242. 

  2243. 

  2244. .L1_41:

  2245.         movq    B, BO             // first buffer to BO

  2246.         addq    $12 * SIZE, BO

  2247. 

  2248. 	INIT1x1

  2249. 

  2250.         movq    K, %rax

  2251. 

  2252. 	sarq	$3,%rax

  2253. 	je	.L1_46

  2254. 

  2255. 	ALIGN_4

  2256. 

  2257. .L1_42:

  2258. 

  2259. 	KERNEL1x1_SUB

  2260. 	KERNEL1x1_SUB

  2261. 	KERNEL1x1_SUB

  2262. 	KERNEL1x1_SUB

  2263. 

  2264. 	KERNEL1x1_SUB

  2265. 	KERNEL1x1_SUB

  2266. 	KERNEL1x1_SUB

  2267. 	KERNEL1x1_SUB

  2268. 

  2269. 	dec %rax

  2270. 	jne	.L1_42

  2271. 

  2272. .L1_46:

  2273.         movq    K, %rax

  2274. 

  2275. 	andq	$7, %rax		# if (k & 1)

  2276. 	je .L1_49

  2277. 

  2278. 	ALIGN_4

  2279. 

  2280. .L1_47:

  2281. 

  2282. 	KERNEL1x1_SUB

  2283. 

  2284. 	dec	%rax

  2285. 	jne	.L1_47

  2286. 

  2287. 

  2288. .L1_49:

  2289. 

  2290. 	SAVE1x1

  2291. 

  2292. .L1_100:

  2293. 

  2294. 

  2295. 

  2296. 

  2297. .L999:

  2298. 	vzeroupper

  2299. 

  2300. 	movq   		SP, %rsp

  2301. 	movq	   (%rsp), %rbx

  2302. 	movq	  8(%rsp), %rbp

  2303. 	movq	 16(%rsp), %r12

  2304. 	movq	 24(%rsp), %r13

  2305. 	movq	 32(%rsp), %r14

  2306. 	movq	 40(%rsp), %r15

  2307. 

  2308. #ifdef WINDOWS_ABI

  2309. 	movq	 48(%rsp), %rdi

  2310. 	movq	 56(%rsp), %rsi

  2311. 	vmovups	 64(%rsp), %xmm6

  2312. 	vmovups	 80(%rsp), %xmm7

  2313. 	vmovups	 96(%rsp), %xmm8

  2314. 	vmovups	112(%rsp), %xmm9

  2315. 	vmovups	128(%rsp), %xmm10

  2316. 	vmovups	144(%rsp), %xmm11

  2317. 	vmovups	160(%rsp), %xmm12

  2318. 	vmovups	176(%rsp), %xmm13

  2319. 	vmovups	192(%rsp), %xmm14

  2320. 	vmovups	208(%rsp), %xmm15

  2321. #endif

  2322. 

  2323. 	addq	$STACKSIZE, %rsp

  2324. 	ret

  2325. 

  2326. 	EPILOGUE

  2327. 

  2328. 

  2329. #else

  2330. /*************************************************************************************

  2331. * TRMM Kernel

  2332. *************************************************************************************/

  2333. 

  2334. 

  2335. 	PROLOGUE

  2336. 	PROFCODE

  2337. 	

  2338. 	subq	$STACKSIZE, %rsp

  2339. 	movq	%rbx,   (%rsp)

  2340. 	movq	%rbp,  8(%rsp)

  2341. 	movq	%r12, 16(%rsp)

  2342. 	movq	%r13, 24(%rsp)

  2343. 	movq	%r14, 32(%rsp)

  2344. 	movq	%r15, 40(%rsp)

  2345. 

  2346. 	vzeroupper

  2347. 

  2348. #ifdef WINDOWS_ABI

  2349. 	movq	%rdi,    48(%rsp)

  2350. 	movq	%rsi,    56(%rsp)

  2351. 	vmovups	%xmm6,   64(%rsp)

  2352. 	vmovups	%xmm7,   80(%rsp)

  2353. 	vmovups	%xmm8,   96(%rsp)

  2354. 	vmovups	%xmm9,  112(%rsp)

  2355. 	vmovups	%xmm10, 128(%rsp)

  2356. 	vmovups	%xmm11, 144(%rsp)

  2357. 	vmovups	%xmm12, 160(%rsp)

  2358. 	vmovups	%xmm13, 176(%rsp)

  2359. 	vmovups	%xmm14, 192(%rsp)

  2360. 	vmovups	%xmm15, 208(%rsp)

  2361. 

  2362. 	movq	ARG1,      OLD_M

  2363. 	movq	ARG2,      OLD_N

  2364. 	movq	ARG3,      OLD_K

  2365. 	movq	OLD_A,     A

  2366. 	movq	OLD_B,     B

  2367. 	movq	OLD_C,     C

  2368. 	movq	OLD_LDC,   LDC

  2369. #ifdef TRMMKERNEL

  2370. 	vmovsd	OLD_OFFSET, %xmm12

  2371. #endif

  2372. 	vmovups	%xmm3, %xmm0

  2373. 

  2374. #else

  2375. 	movq	STACKSIZE +  8(%rsp), LDC

  2376. #ifdef TRMMKERNEL

  2377. 	vmovsd	STACKSIZE + 16(%rsp), %xmm12

  2378. #endif

  2379. 

  2380. #endif

  2381. 

  2382. 	movq    %rsp, SP      # save old stack

  2383.         subq    $128 + L_BUFFER_SIZE, %rsp

  2384.         andq    $-4096, %rsp    # align stack

  2385. 

  2386.         STACK_TOUCH

  2387. 

  2388. 	cmpq	$ 0, OLD_M

  2389. 	je	.L999

  2390. 

  2391. 	cmpq	$ 0, OLD_N

  2392. 	je	.L999

  2393. 

  2394. 	cmpq	$ 0, OLD_K

  2395. 	je	.L999

  2396. 

  2397. 	movq	OLD_M, M

  2398. 	movq	OLD_N, N

  2399. 	movq	OLD_K, K

  2400. 

  2401. 	vmovsd	 %xmm0, ALPHA

  2402. 

  2403. 	salq	$BASE_SHIFT, LDC

  2404. 

  2405. 	movq    N, %rax

  2406.         xorq    %rdx, %rdx

  2407.         movq    $4,  %rdi

  2408.         divq    %rdi                     //    N / 4

  2409.         movq    %rax, Ndiv12             //    N / 4

  2410.         movq    %rdx, Nmod12             //    N % 4

  2411. 

  2412. #ifdef TRMMKERNEL

  2413.         vmovsd  %xmm12, OFFSET

  2414.         vmovsd  %xmm12, KK

  2415. #ifndef LEFT

  2416.         negq    KK

  2417. #endif  

  2418. #endif

  2419. 

  2420. 

  2421. 

  2422. 	movq	Ndiv12,  J

  2423. 	cmpq	$ 0, J

  2424. 	je	.L2_0

  2425. 	ALIGN_4

  2426. 

  2427. .L4_10:

  2428. 	movq	C, CO1

  2429. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  2430. 

  2431. #if defined(TRMMKERNEL) && defined(LEFT)

  2432.         movq    OFFSET, %rax

  2433.         movq    %rax, KK

  2434. #endif

  2435. 

  2436. 	

  2437. 	movq	A, AO		 	// aoffset = a

  2438. 	addq	$16 * SIZE, AO

  2439. 

  2440. 	movq	M,  I

  2441. 	sarq	$2, I			// i = m / 4

  2442. 	je	.L4_20

  2443. 

  2444. 	ALIGN_4

  2445. 

  2446. .L4_11:

  2447. 

  2448. #if !defined(TRMMKERNEL) || \

  2449.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2450.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2451.         movq    B, BO        

  2452.         addq    $12 * SIZE, BO

  2453. #else

  2454.         movq    B, BO        

  2455.         addq    $12 * SIZE, BO

  2456.         movq    KK, %rax

  2457. 	salq	$3, %rax		// rax * SIZE

  2458. 	leaq	(BO,%rax,4), BO		// add number of values in B

  2459. 	leaq	(AO,%rax,4), AO		// add number of values in A

  2460. #endif

  2461. 

  2462. 

  2463. #ifndef TRMMKERNEL

  2464.         movq    K, %rax

  2465. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2466.         movq    K, %rax

  2467.         subq    KK, %rax

  2468.         movq    %rax, KKK

  2469. #else

  2470.         movq    KK, %rax

  2471. #ifdef LEFT

  2472.         addq    $4, %rax        // number of values in AO

  2473. #else

  2474.         addq    $4, %rax        // number of values in BO

  2475. #endif

  2476.         movq    %rax, KKK

  2477. #endif

  2478. 

  2479. 	sarq	$3, %rax			//  K / 8

  2480. 	cmpq    $2, %rax

  2481. 	jl	.L4_13

  2482. 

  2483. 

  2484. 	KERNEL4x4_I

  2485. 	KERNEL4x4_M2

  2486. 	KERNEL4x4_M1

  2487. 	KERNEL4x4_M2

  2488. 

  2489. 	KERNEL4x4_M1

  2490. 	KERNEL4x4_M2

  2491. 	KERNEL4x4_M1

  2492. 	KERNEL4x4_M2

  2493. 

  2494. 	subq $2, %rax

  2495. 	je	.L4_12a

  2496. 

  2497. 	ALIGN_5

  2498. 

  2499. .L4_12:

  2500. 

  2501. 	KERNEL4x4_M1

  2502. 	KERNEL4x4_M2

  2503. 	KERNEL4x4_M1

  2504. 	KERNEL4x4_M2

  2505. 

  2506. 	KERNEL4x4_M1

  2507. 	KERNEL4x4_M2

  2508. 	KERNEL4x4_M1

  2509. 	KERNEL4x4_M2

  2510. 

  2511. 	dec	%rax

  2512. 	jne	.L4_12

  2513. 

  2514. .L4_12a:

  2515. 

  2516. 	KERNEL4x4_M1

  2517. 	KERNEL4x4_M2

  2518. 	KERNEL4x4_M1

  2519. 	KERNEL4x4_M2

  2520. 

  2521. 	KERNEL4x4_M1

  2522. 	KERNEL4x4_M2

  2523. 	KERNEL4x4_M1

  2524. 	KERNEL4x4_E

  2525. 

  2526. 	jmp .L4_16

  2527. 

  2528. 

  2529. .L4_13:

  2530. 

  2531. 	test $1, %rax

  2532. 	jz .L4_14

  2533. 

  2534. 	KERNEL4x4_I

  2535. 	KERNEL4x4_M2

  2536. 	KERNEL4x4_M1

  2537. 	KERNEL4x4_M2

  2538. 

  2539. 	KERNEL4x4_M1

  2540. 	KERNEL4x4_M2

  2541. 	KERNEL4x4_M1

  2542. 	KERNEL4x4_E

  2543. 

  2544. 	jmp .L4_16

  2545. 

  2546. 

  2547. .L4_14:

  2548. 

  2549. 	INIT4x4

  2550. 

  2551. 

  2552. .L4_16:

  2553.         movq    KKK, %rax

  2554. 

  2555. 	andq	$7, %rax		# if (k & 1)

  2556. 	je .L4_19

  2557. 

  2558. 	ALIGN_4

  2559. 

  2560. .L4_17:

  2561. 

  2562. 	KERNEL4x4_SUB

  2563. 

  2564. 	dec	%rax

  2565. 	jne	.L4_17

  2566. 	ALIGN_4

  2567. 

  2568. 

  2569. .L4_19:

  2570. 

  2571. 	SAVE4x4

  2572. 

  2573. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2574.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2575.         movq    K, %rax

  2576.         subq    KKK, %rax

  2577. 	salq	$3, %rax			// rax + SIZE

  2578.         leaq    (BO, %rax, 4), BO		// number of values in B

  2579.         leaq    (AO, %rax, 4), AO		// number of values in A

  2580. #endif

  2581. 

  2582. #if defined(TRMMKERNEL) && defined(LEFT)

  2583.         addq    $4, KK				// number of values in A

  2584. #endif

  2585. 

  2586. 	decq	I			# i --

  2587. 	jg	.L4_11

  2588. 	ALIGN_4	

  2589. 

  2590. /**************************************************************************

  2591. * Rest of M 

  2592. ***************************************************************************/

  2593. .L4_20:

  2594. 	// Test rest of M

  2595. 

  2596. 	testq	$3, M

  2597. 	jz	.L4_100			// to next 16 lines of N

  2598. 

  2599. 

  2600. .L4_30:

  2601. 	testq	$2, M		

  2602. 	jz	.L4_40

  2603. 

  2604. 	ALIGN_4

  2605. 

  2606. .L4_31:

  2607. 

  2608. #if !defined(TRMMKERNEL) || \

  2609.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2610.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2611.         movq    B, BO        

  2612.         addq    $12 * SIZE, BO

  2613. #else

  2614.         movq    B, BO        

  2615.         addq    $12 * SIZE, BO

  2616.         movq    KK, %rax

  2617. 	salq	$3, %rax		// rax * SIZE

  2618. 	leaq	(BO,%rax,4), BO		// add number of values in B

  2619. 	leaq	(AO,%rax,2), AO		// add number of values in A

  2620. #endif

  2621. 

  2622. 

  2623. #ifndef TRMMKERNEL

  2624.         movq    K, %rax

  2625. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2626.         movq    K, %rax

  2627.         subq    KK, %rax

  2628.         movq    %rax, KKK

  2629. #else

  2630.         movq    KK, %rax

  2631. #ifdef LEFT

  2632.         addq    $2, %rax        // number of values in AO

  2633. #else

  2634.         addq    $4, %rax        // number of values in BO

  2635. #endif

  2636.         movq    %rax, KKK

  2637. #endif

  2638. 

  2639. 	INIT2x4

  2640. 

  2641. 	sarq	$3, %rax

  2642. 	je	.L4_36

  2643. 	ALIGN_4

  2644. 

  2645. .L4_32:

  2646. 

  2647. 	KERNEL2x4_SUB

  2648. 	KERNEL2x4_SUB

  2649. 	KERNEL2x4_SUB

  2650. 	KERNEL2x4_SUB

  2651. 

  2652. 	KERNEL2x4_SUB

  2653. 	KERNEL2x4_SUB

  2654. 	KERNEL2x4_SUB

  2655. 	KERNEL2x4_SUB

  2656. 

  2657. 	dec %rax

  2658. 	jne	.L4_32

  2659. 	ALIGN_4

  2660. 

  2661. .L4_36:

  2662.         movq    KKK, %rax

  2663. 

  2664. 	andq	$7, %rax		# if (k & 1)

  2665. 	je .L4_39

  2666. 

  2667. 	ALIGN_4

  2668. 

  2669. .L4_37:

  2670. 

  2671. 	KERNEL2x4_SUB

  2672. 

  2673. 	dec %rax

  2674. 	jne	.L4_37

  2675. 

  2676. 

  2677. .L4_39:

  2678. 

  2679. 	SAVE2x4

  2680. 

  2681. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2682.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2683.         movq    K, %rax

  2684.         subq    KKK, %rax

  2685. 	salq	$3, %rax			// rax + SIZE

  2686.         leaq    (BO, %rax, 4), BO		// number of values in B

  2687.         leaq    (AO, %rax, 2), AO		// number of values in A

  2688. #endif

  2689. 

  2690. #if defined(TRMMKERNEL) && defined(LEFT)

  2691.         addq    $2, KK				// number of values in A

  2692. #endif

  2693. 

  2694. 

  2695. .L4_40:

  2696. 	testq	$1, M		

  2697. 	jz	.L4_100		// to next 3 lines of N

  2698. 

  2699. 	ALIGN_4

  2700. 

  2701. .L4_41:

  2702. 

  2703. #if !defined(TRMMKERNEL) || \

  2704.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2705.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2706.         movq    B, BO        

  2707.         addq    $12 * SIZE, BO

  2708. #else

  2709.         movq    B, BO        

  2710.         addq    $12 * SIZE, BO

  2711.         movq    KK, %rax

  2712. 	salq	$3, %rax		// rax * SIZE

  2713. 	leaq	(BO,%rax,4), BO		// add number of values in B

  2714. 	leaq	(AO,%rax,1), AO		// add number of values in A

  2715. #endif

  2716. 

  2717. 

  2718. #ifndef TRMMKERNEL

  2719.         movq    K, %rax

  2720. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2721.         movq    K, %rax

  2722.         subq    KK, %rax

  2723.         movq    %rax, KKK

  2724. #else

  2725.         movq    KK, %rax

  2726. #ifdef LEFT

  2727.         addq    $1, %rax        // number of values in AO

  2728. #else

  2729.         addq    $4, %rax        // number of values in BO

  2730. #endif

  2731.         movq    %rax, KKK

  2732. #endif

  2733. 

  2734. 	INIT1x4

  2735. 

  2736. 	sarq	$3,%rax

  2737. 	je	.L4_46

  2738. 

  2739. 	ALIGN_4

  2740. 

  2741. .L4_42:

  2742. 

  2743. 	KERNEL1x4_SUB

  2744. 	KERNEL1x4_SUB

  2745. 	KERNEL1x4_SUB

  2746. 	KERNEL1x4_SUB

  2747. 

  2748. 	KERNEL1x4_SUB

  2749. 	KERNEL1x4_SUB

  2750. 	KERNEL1x4_SUB

  2751. 	KERNEL1x4_SUB

  2752. 

  2753. 	dec %rax

  2754. 	jne	.L4_42

  2755. 	ALIGN_4

  2756. 

  2757. .L4_46:

  2758.         movq    KKK, %rax

  2759. 

  2760. 	andq	$7, %rax		# if (k & 1)

  2761. 	je .L4_49

  2762. 

  2763. 	ALIGN_4

  2764. 

  2765. .L4_47:

  2766. 

  2767. 	KERNEL1x4_SUB

  2768. 

  2769. 	dec	%rax

  2770. 	jne	.L4_47

  2771. 	ALIGN_4

  2772. 

  2773. 

  2774. .L4_49:

  2775. 

  2776. 	SAVE1x4

  2777. 

  2778. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2779.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2780.         movq    K, %rax

  2781.         subq    KKK, %rax

  2782. 	salq	$3, %rax			// rax + SIZE

  2783.         leaq    (BO, %rax, 4), BO		// number of values in B

  2784.         leaq    (AO, %rax, 1), AO		// number of values in A

  2785. #endif

  2786. 

  2787. #if defined(TRMMKERNEL) && defined(LEFT)

  2788.         addq    $1, KK				// number of values in A

  2789. #endif

  2790. 

  2791. .L4_100:

  2792. 

  2793. #if defined(TRMMKERNEL) && !defined(LEFT)

  2794.         addq    $4, KK				// number of values in B

  2795. #endif

  2796. 

  2797. 

  2798. 	movq	K, %rax

  2799. 	salq	$2, %rax		// * 4

  2800. 	leaq	(B , %rax, SIZE), B

  2801. 	decq	J			// j --

  2802. 	jg	.L4_10

  2803. 

  2804. 

  2805. 

  2806. 

  2807. /***************************************************************************************************************/

  2808. 

  2809. .L2_0:

  2810. 

  2811. 	movq	Nmod12, J		

  2812. 	testq	$2, J

  2813. 	je	.L1_0

  2814. 

  2815. .L2_10:

  2816. 	movq	C, CO1

  2817. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  2818. 

  2819. #if defined(TRMMKERNEL) && defined(LEFT)

  2820.         movq    OFFSET, %rax

  2821.         movq    %rax, KK

  2822. #endif

  2823. 

  2824. 

  2825. 	

  2826. 	movq	A, AO		 	// aoffset = a

  2827. 	addq	$16 * SIZE, AO

  2828. 

  2829. 	movq	M,  I

  2830. 	sarq	$2, I			// i = m / 4

  2831. 	je	.L2_20

  2832. 

  2833. 	ALIGN_4

  2834. 

  2835. .L2_11:

  2836. 

  2837. #if !defined(TRMMKERNEL) || \

  2838.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2839.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2840.         movq    B, BO        

  2841.         addq    $12 * SIZE, BO

  2842. #else

  2843.         movq    B, BO        

  2844.         addq    $12 * SIZE, BO

  2845.         movq    KK, %rax

  2846. 	salq	$3, %rax		// rax * SIZE

  2847. 	leaq	(BO,%rax,2), BO		// add number of values in B

  2848. 	leaq	(AO,%rax,4), AO		// add number of values in A

  2849. #endif

  2850. 

  2851. 

  2852. #ifndef TRMMKERNEL

  2853.         movq    K, %rax

  2854. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2855.         movq    K, %rax

  2856.         subq    KK, %rax

  2857.         movq    %rax, KKK

  2858. #else

  2859.         movq    KK, %rax

  2860. #ifdef LEFT

  2861.         addq    $4, %rax        // number of values in AO

  2862. #else

  2863.         addq    $2, %rax        // number of values in BO

  2864. #endif

  2865.         movq    %rax, KKK

  2866. #endif

  2867. 

  2868. 	INIT4x2

  2869. 

  2870. 	sarq $3, %rax			//  K / 8

  2871. 

  2872. 	je	.L2_16

  2873. 

  2874. 	ALIGN_5

  2875. 

  2876. .L2_12:

  2877. 

  2878. 	KERNEL4x2_SUB

  2879. 	KERNEL4x2_SUB

  2880. 	KERNEL4x2_SUB

  2881. 	KERNEL4x2_SUB

  2882. 

  2883. 	KERNEL4x2_SUB

  2884. 	KERNEL4x2_SUB

  2885. 	KERNEL4x2_SUB

  2886. 	KERNEL4x2_SUB

  2887. 

  2888. 	dec	%rax

  2889. 	jne	.L2_12

  2890. 

  2891. 

  2892. .L2_16:

  2893.         movq    KKK, %rax

  2894. 

  2895. 	andq	$7, %rax		# if (k & 1)

  2896. 	je .L2_19

  2897. 

  2898. 	ALIGN_4

  2899. 

  2900. .L2_17:

  2901. 

  2902. 	KERNEL4x2_SUB

  2903. 

  2904. 	dec	%rax

  2905. 	jne	.L2_17

  2906. 	ALIGN_4

  2907. 

  2908. 

  2909. .L2_19:

  2910. 

  2911. 	SAVE4x2

  2912. 

  2913. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2914.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2915.         movq    K, %rax

  2916.         subq    KKK, %rax

  2917. 	salq	$3, %rax			// rax + SIZE

  2918.         leaq    (BO, %rax, 2), BO		// number of values in B

  2919.         leaq    (AO, %rax, 4), AO		// number of values in A

  2920. #endif

  2921. 

  2922. #if defined(TRMMKERNEL) && defined(LEFT)

  2923.         addq    $4, KK				// number of values in A

  2924. #endif

  2925. 

  2926. 

  2927. 	decq	I			# i --

  2928. 	jg	.L2_11

  2929. 	ALIGN_4	

  2930. 

  2931. /**************************************************************************

  2932. * Rest of M 

  2933. ***************************************************************************/

  2934. .L2_20:

  2935. 	// Test rest of M

  2936. 

  2937. 	testq	$3, M

  2938. 	jz	.L2_100			// to next 16 lines of N

  2939. 

  2940. 

  2941. .L2_30:

  2942. 	testq	$2, M		

  2943. 	jz	.L2_40

  2944. 

  2945. 	ALIGN_4

  2946. 

  2947. .L2_31:

  2948. 

  2949. #if !defined(TRMMKERNEL) || \

  2950.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2951.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2952.         movq    B, BO        

  2953.         addq    $12 * SIZE, BO

  2954. #else

  2955.         movq    B, BO        

  2956.         addq    $12 * SIZE, BO

  2957.         movq    KK, %rax

  2958. 	salq	$3, %rax		// rax * SIZE

  2959. 	leaq	(BO,%rax,2), BO		// add number of values in B

  2960. 	leaq	(AO,%rax,2), AO		// add number of values in A

  2961. #endif

  2962. 

  2963. 

  2964. #ifndef TRMMKERNEL

  2965.         movq    K, %rax

  2966. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2967.         movq    K, %rax

  2968.         subq    KK, %rax

  2969.         movq    %rax, KKK

  2970. #else

  2971.         movq    KK, %rax

  2972. #ifdef LEFT

  2973.         addq    $2, %rax        // number of values in AO

  2974. #else

  2975.         addq    $2, %rax        // number of values in BO

  2976. #endif

  2977.         movq    %rax, KKK

  2978. #endif

  2979. 

  2980. 	INIT2x2

  2981. 

  2982. 	sarq	$3, %rax

  2983. 	je	.L2_36

  2984. 	ALIGN_4

  2985. 

  2986. .L2_32:

  2987. 

  2988. 	KERNEL2x2_SUB

  2989. 	KERNEL2x2_SUB

  2990. 	KERNEL2x2_SUB

  2991. 	KERNEL2x2_SUB

  2992. 

  2993. 	KERNEL2x2_SUB

  2994. 	KERNEL2x2_SUB

  2995. 	KERNEL2x2_SUB

  2996. 	KERNEL2x2_SUB

  2997. 

  2998. 	dec %rax

  2999. 	jne	.L2_32

  3000. 

  3001. .L2_36:

  3002.         movq    KKK, %rax

  3003. 

  3004. 	andq	$7, %rax		# if (k & 1)

  3005. 	je .L2_39

  3006. 

  3007. 	ALIGN_4

  3008. 

  3009. .L2_37:

  3010. 

  3011. 	KERNEL2x2_SUB

  3012. 

  3013. 	dec %rax

  3014. 	jne	.L2_37

  3015. 

  3016. 

  3017. .L2_39:

  3018. 

  3019. 	SAVE2x2

  3020. 

  3021. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3022.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3023.         movq    K, %rax

  3024.         subq    KKK, %rax

  3025. 	salq	$3, %rax			// rax + SIZE

  3026.         leaq    (BO, %rax, 2), BO		// number of values in B

  3027.         leaq    (AO, %rax, 2), AO		// number of values in A

  3028. #endif

  3029. 

  3030. #if defined(TRMMKERNEL) && defined(LEFT)

  3031.         addq    $2, KK				// number of values in A

  3032. #endif

  3033. 

  3034. 

  3035. .L2_40:

  3036. 	testq	$1, M		

  3037. 	jz	.L2_100		// to next 3 lines of N

  3038. 

  3039. .L2_41:

  3040. 

  3041. #if !defined(TRMMKERNEL) || \

  3042.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3043.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3044.         movq    B, BO        

  3045.         addq    $12 * SIZE, BO

  3046. #else

  3047.         movq    B, BO        

  3048.         addq    $12 * SIZE, BO

  3049.         movq    KK, %rax

  3050. 	salq	$3, %rax		// rax * SIZE

  3051. 	leaq	(BO,%rax,2), BO		// add number of values in B

  3052. 	leaq	(AO,%rax,1), AO		// add number of values in A

  3053. #endif

  3054. 

  3055. 

  3056. #ifndef TRMMKERNEL

  3057.         movq    K, %rax

  3058. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3059.         movq    K, %rax

  3060.         subq    KK, %rax

  3061.         movq    %rax, KKK

  3062. #else

  3063.         movq    KK, %rax

  3064. #ifdef LEFT

  3065.         addq    $1, %rax        // number of values in AO

  3066. #else

  3067.         addq    $2, %rax        // number of values in BO

  3068. #endif

  3069.         movq    %rax, KKK

  3070. #endif

  3071. 

  3072. 	INIT1x2

  3073. 

  3074. 	sarq	$3,%rax

  3075. 	je	.L2_46

  3076. 

  3077. 	ALIGN_4

  3078. 

  3079. .L2_42:

  3080. 

  3081. 	KERNEL1x2_SUB

  3082. 	KERNEL1x2_SUB

  3083. 	KERNEL1x2_SUB

  3084. 	KERNEL1x2_SUB

  3085. 

  3086. 	KERNEL1x2_SUB

  3087. 	KERNEL1x2_SUB

  3088. 	KERNEL1x2_SUB

  3089. 	KERNEL1x2_SUB

  3090. 

  3091. 	dec %rax

  3092. 	jne	.L2_42

  3093. 

  3094. .L2_46:

  3095.         movq    KKK, %rax

  3096. 

  3097. 	andq	$7, %rax		# if (k & 1)

  3098. 	je .L2_49

  3099. 

  3100. 	ALIGN_4

  3101. 

  3102. .L2_47:

  3103. 

  3104. 	KERNEL1x2_SUB

  3105. 

  3106. 	dec	%rax

  3107. 	jne	.L2_47

  3108. 

  3109. .L2_49:

  3110. 

  3111. 	SAVE1x2

  3112. 

  3113. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3114.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3115.         movq    K, %rax

  3116.         subq    KKK, %rax

  3117. 	salq	$3, %rax			// rax * SIZE

  3118.         leaq    (BO, %rax, 2), BO		// number of values in B

  3119.         leaq    (AO, %rax, 1), AO		// number of values in A

  3120. #endif

  3121. 

  3122. #if defined(TRMMKERNEL) && defined(LEFT)

  3123.         addq    $1, KK				// number of values in A

  3124. #endif

  3125. 

  3126. 

  3127. .L2_100:

  3128. 

  3129. 

  3130. #if defined(TRMMKERNEL) && !defined(LEFT)

  3131.         addq    $2, KK				// number of values in B

  3132. #endif

  3133. 

  3134. 	movq	K, %rax

  3135. 	salq	$1, %rax		// * 2

  3136. 	leaq	(B , %rax, SIZE), B

  3137. 

  3138. /***************************************************************************************************************/

  3139. 

  3140. .L1_0:

  3141. 

  3142. 	movq	Nmod12, J		

  3143. 	testq	$1, J

  3144. 	je	.L999

  3145. 

  3146. .L1_10:

  3147. 	movq	C, CO1

  3148. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  3149. 

  3150. #if defined(TRMMKERNEL) && defined(LEFT)

  3151.         movq    OFFSET, %rax

  3152.         movq    %rax, KK

  3153. #endif

  3154. 	

  3155. 	movq	A, AO		 	// aoffset = a

  3156. 	addq	$16 * SIZE, AO

  3157. 

  3158. 	movq	M,  I

  3159. 	sarq	$2, I			// i = m / 4

  3160. 	je	.L1_20

  3161. 

  3162. 	ALIGN_4

  3163. 

  3164. .L1_11:

  3165. 

  3166. #if !defined(TRMMKERNEL) || \

  3167.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3168.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3169.         movq    B, BO        

  3170.         addq    $12 * SIZE, BO

  3171. #else

  3172.         movq    B, BO        

  3173.         addq    $12 * SIZE, BO

  3174.         movq    KK, %rax

  3175. 	salq	$3, %rax		// rax * SIZE

  3176. 	leaq	(BO,%rax,1), BO		// add number of values in B

  3177. 	leaq	(AO,%rax,4), AO		// add number of values in A

  3178. #endif

  3179. 

  3180. 

  3181. #ifndef TRMMKERNEL

  3182.         movq    K, %rax

  3183. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3184.         movq    K, %rax

  3185.         subq    KK, %rax

  3186.         movq    %rax, KKK

  3187. #else

  3188.         movq    KK, %rax

  3189. #ifdef LEFT

  3190.         addq    $4, %rax        // number of values in AO

  3191. #else

  3192.         addq    $1, %rax        // number of values in BO

  3193. #endif

  3194.         movq    %rax, KKK

  3195. #endif

  3196. 

  3197. 	INIT4x1

  3198. 

  3199. 	sarq	$3, %rax			//  K / 8

  3200. 	je	.L1_16

  3201. 

  3202. 	ALIGN_5

  3203. 

  3204. .L1_12:

  3205. 

  3206. 	KERNEL4x1

  3207. 

  3208. 	dec	%rax

  3209. 	jne	.L1_12

  3210. 

  3211. 

  3212. .L1_16:

  3213.         movq    KKK, %rax

  3214. 

  3215. 	andq	$7, %rax		# if (k & 1)

  3216. 	je .L1_19

  3217. 

  3218. 	ALIGN_4

  3219. 

  3220. .L1_17:

  3221. 

  3222. 	KERNEL4x1_SUB

  3223. 

  3224. 	dec	%rax

  3225. 	jne	.L1_17

  3226. 	ALIGN_4

  3227. 

  3228. 

  3229. .L1_19:

  3230. 

  3231. 	SAVE4x1

  3232. 

  3233. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3234.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3235.         movq    K, %rax

  3236.         subq    KKK, %rax

  3237. 	salq	$3, %rax			// rax * SIZE

  3238.         leaq    (BO, %rax, 1), BO		// number of values in B

  3239.         leaq    (AO, %rax, 4), AO		// number of values in A

  3240. #endif

  3241. 

  3242. #if defined(TRMMKERNEL) && defined(LEFT)

  3243.         addq    $4, KK				// number of values in A

  3244. #endif

  3245. 

  3246. 

  3247. 	decq	I			# i --

  3248. 	jg	.L1_11

  3249. 

  3250. /**************************************************************************

  3251. * Rest of M 

  3252. ***************************************************************************/

  3253. .L1_20:

  3254. 	// Test rest of M

  3255. 

  3256. 	testq	$3, M

  3257. 	jz	.L1_100	

  3258. 

  3259. 

  3260. .L1_30:

  3261. 	testq	$2, M		

  3262. 	jz	.L1_40

  3263. 

  3264. 	ALIGN_4

  3265. 

  3266. .L1_31:

  3267. 

  3268. #if !defined(TRMMKERNEL) || \

  3269.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3270.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3271.         movq    B, BO        

  3272.         addq    $12 * SIZE, BO

  3273. #else

  3274.         movq    B, BO        

  3275.         addq    $12 * SIZE, BO

  3276.         movq    KK, %rax

  3277. 	salq	$3, %rax		// rax * SIZE

  3278. 	leaq	(BO,%rax,1), BO		// add number of values in B

  3279. 	leaq	(AO,%rax,2), AO		// add number of values in A

  3280. #endif

  3281. 

  3282. 

  3283. #ifndef TRMMKERNEL

  3284.         movq    K, %rax

  3285. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3286.         movq    K, %rax

  3287.         subq    KK, %rax

  3288.         movq    %rax, KKK

  3289. #else

  3290.         movq    KK, %rax

  3291. #ifdef LEFT

  3292.         addq    $2, %rax        // number of values in AO

  3293. #else

  3294.         addq    $1, %rax        // number of values in BO

  3295. #endif

  3296.         movq    %rax, KKK

  3297. #endif

  3298. 

  3299. 	INIT2x1

  3300. 

  3301. 	sarq	$3, %rax

  3302. 	je	.L1_36

  3303. 	ALIGN_4

  3304. 

  3305. .L1_32:

  3306. 

  3307. 	KERNEL2x1_SUB

  3308. 	KERNEL2x1_SUB

  3309. 	KERNEL2x1_SUB

  3310. 	KERNEL2x1_SUB

  3311. 

  3312. 	KERNEL2x1_SUB

  3313. 	KERNEL2x1_SUB

  3314. 	KERNEL2x1_SUB

  3315. 	KERNEL2x1_SUB

  3316. 

  3317. 

  3318. 	dec %rax

  3319. 	jne	.L1_32

  3320. 

  3321. .L1_36:

  3322.         movq    KKK, %rax

  3323. 

  3324. 	andq	$7, %rax		# if (k & 1)

  3325. 	je .L1_39

  3326. 

  3327. 	ALIGN_4

  3328. 

  3329. .L1_37:

  3330. 

  3331. 	KERNEL2x1_SUB

  3332. 

  3333. 	dec %rax

  3334. 	jne	.L1_37

  3335. 

  3336. .L1_39:

  3337. 

  3338. 	SAVE2x1

  3339. 

  3340. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3341.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3342.         movq    K, %rax

  3343.         subq    KKK, %rax

  3344. 	salq	$3, %rax			// rax * SIZE

  3345.         leaq    (BO, %rax, 1), BO		// number of values in B

  3346.         leaq    (AO, %rax, 2), AO		// number of values in A

  3347. #endif

  3348. 

  3349. #if defined(TRMMKERNEL) && defined(LEFT)

  3350.         addq    $2, KK				// number of values in A

  3351. #endif

  3352. 

  3353. 

  3354. .L1_40:

  3355. 	testq	$1, M		

  3356. 	jz	.L1_100		// to next 3 lines of N

  3357. 

  3358. 

  3359. .L1_41:

  3360. 

  3361. #if !defined(TRMMKERNEL) || \

  3362.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3363.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3364.         movq    B, BO        

  3365.         addq    $12 * SIZE, BO

  3366. #else

  3367.         movq    B, BO        

  3368.         addq    $12 * SIZE, BO

  3369.         movq    KK, %rax

  3370. 	salq	$3, %rax		// rax * SIZE

  3371. 	leaq	(BO,%rax,1), BO		// add number of values in B

  3372. 	leaq	(AO,%rax,1), AO		// add number of values in A

  3373. #endif

  3374. 

  3375. 

  3376. #ifndef TRMMKERNEL

  3377.         movq    K, %rax

  3378. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3379.         movq    K, %rax

  3380.         subq    KK, %rax

  3381.         movq    %rax, KKK

  3382. #else

  3383.         movq    KK, %rax

  3384. #ifdef LEFT

  3385.         addq    $1, %rax        // number of values in AO

  3386. #else

  3387.         addq    $1, %rax        // number of values in BO

  3388. #endif

  3389.         movq    %rax, KKK

  3390. #endif

  3391. 

  3392. 	INIT1x1

  3393. 

  3394. 	sarq	$3,%rax

  3395. 	je	.L1_46

  3396. 

  3397. 	ALIGN_4

  3398. 

  3399. .L1_42:

  3400. 

  3401. 	KERNEL1x1_SUB

  3402. 	KERNEL1x1_SUB

  3403. 	KERNEL1x1_SUB

  3404. 	KERNEL1x1_SUB

  3405. 

  3406. 	KERNEL1x1_SUB

  3407. 	KERNEL1x1_SUB

  3408. 	KERNEL1x1_SUB

  3409. 	KERNEL1x1_SUB

  3410. 

  3411. 	dec %rax

  3412. 	jne	.L1_42

  3413. 

  3414. .L1_46:

  3415.         movq    KKK, %rax

  3416. 

  3417. 	andq	$7, %rax		# if (k & 1)

  3418. 	je .L1_49

  3419. 

  3420. 	ALIGN_4

  3421. 

  3422. .L1_47:

  3423. 

  3424. 	KERNEL1x1_SUB

  3425. 

  3426. 	dec	%rax

  3427. 	jne	.L1_47

  3428. 

  3429. 

  3430. .L1_49:

  3431. 

  3432. 	SAVE1x1

  3433. 

  3434. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3435.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3436.         movq    K, %rax

  3437.         subq    KKK, %rax

  3438. 	salq	$3, %rax			// rax * SIZE

  3439.         leaq    (BO, %rax, 1), BO		// number of values in B

  3440.         leaq    (AO, %rax, 1), AO		// number of values in A

  3441. #endif

  3442. 

  3443. #if defined(TRMMKERNEL) && defined(LEFT)

  3444.         addq    $1, KK				// number of values in A

  3445. #endif

  3446. 

  3447. 

  3448. 

  3449. .L1_100:

  3450. 

  3451. 

  3452. #if defined(TRMMKERNEL) && !defined(LEFT)

  3453.         addq    $1, KK				// number of values in B

  3454. #endif

  3455. 

  3456. 

  3457. 

  3458. .L999:

  3459. 

  3460. 	vzeroupper

  3461. 

  3462. 	movq   		SP, %rsp

  3463. 	movq	   (%rsp), %rbx

  3464. 	movq	  8(%rsp), %rbp

  3465. 	movq	 16(%rsp), %r12

  3466. 	movq	 24(%rsp), %r13

  3467. 	movq	 32(%rsp), %r14

  3468. 	movq	 40(%rsp), %r15

  3469. 

  3470. #ifdef WINDOWS_ABI

  3471. 	movq	 48(%rsp), %rdi

  3472. 	movq	 56(%rsp), %rsi

  3473. 	vmovups	 64(%rsp), %xmm6

  3474. 	vmovups	 80(%rsp), %xmm7

  3475. 	vmovups	 96(%rsp), %xmm8

  3476. 	vmovups	112(%rsp), %xmm9

  3477. 	vmovups	128(%rsp), %xmm10

  3478. 	vmovups	144(%rsp), %xmm11

  3479. 	vmovups	160(%rsp), %xmm12

  3480. 	vmovups	176(%rsp), %xmm13

  3481. 	vmovups	192(%rsp), %xmm14

  3482. 	vmovups	208(%rsp), %xmm15

  3483. #endif

  3484. 

  3485. 	addq	$STACKSIZE, %rsp

  3486. 	ret

  3487. 

  3488. 	EPILOGUE

  3489. 

  3490. 

  3491. 

  3492. 

  3493. 

  3494. #endif