OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c1f7a81663
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c1f7a81663'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_kernel_4x8_haswell.S

5154 lines
94 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2015, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. 

  29. #define ASSEMBLER

  30. #include "common.h"

  31.  

  32. #define OLD_M	%rdi

  33. #define OLD_N	%rsi

  34. #define M	%r13

  35. #define J	%r14

  36. #define OLD_K	%rdx

  37. 

  38. #define A	%rcx

  39. #define B	%r8

  40. #define C	%r9

  41. #define LDC	%r10

  42. 	

  43. #define I	%r11

  44. #define AO	%rdi

  45. #define BO	%rsi

  46. #define	CO1	%r15

  47. #define K	%r12

  48. #define	SP	%rbx

  49. 

  50. #define BO1	%rdi

  51. #define BO2	%r15

  52. #define BO3	%rbp

  53. 

  54. #ifndef WINDOWS_ABI

  55. 

  56. #define STACKSIZE 96

  57. #define L_BUFFER_SIZE 256*8*12+4096

  58. 

  59. #else

  60. 

  61. #define STACKSIZE 256

  62. #define L_BUFFER_SIZE 128*8*12+512

  63. 

  64. #define OLD_A		40 + STACKSIZE(%rsp)

  65. #define OLD_B		48 + STACKSIZE(%rsp)

  66. #define OLD_C		56 + STACKSIZE(%rsp)

  67. #define OLD_LDC		64 + STACKSIZE(%rsp)

  68. #define OLD_OFFSET	72 + STACKSIZE(%rsp)

  69. 

  70. #endif

  71. 

  72. 

  73. #define Ndiv12	 24(%rsp)

  74. #define Nmod12	 32(%rsp)

  75. #define N	 40(%rsp)

  76. #define ALPHA	 48(%rsp)

  77. #define OFFSET	 56(%rsp)

  78. #define KK	 64(%rsp)

  79. #define KKK	 72(%rsp)

  80. #define BUFFER1	           128(%rsp)

  81. 

  82. #if defined(OS_WINDOWS)

  83. #if   L_BUFFER_SIZE > 16384

  84. #define STACK_TOUCH \

  85.         movl    $ 0,  4096 * 4(%rsp);\

  86.         movl    $ 0,  4096 * 3(%rsp);\

  87.         movl    $ 0,  4096 * 2(%rsp);\

  88.         movl    $ 0,  4096 * 1(%rsp);

  89. #elif L_BUFFER_SIZE > 12288

  90. #define STACK_TOUCH \

  91.         movl    $ 0,  4096 * 3(%rsp);\

  92.         movl    $ 0,  4096 * 2(%rsp);\

  93.         movl    $ 0,  4096 * 1(%rsp);

  94. #elif L_BUFFER_SIZE > 8192

  95. #define STACK_TOUCH \

  96.         movl    $ 0,  4096 * 2(%rsp);\

  97.         movl    $ 0,  4096 * 1(%rsp);

  98. #elif L_BUFFER_SIZE > 4096

  99. #define STACK_TOUCH \

  100.         movl    $ 0,  4096 * 1(%rsp);

  101. #else

  102. #define STACK_TOUCH

  103. #endif

  104. #else

  105. #define STACK_TOUCH

  106. #endif

  107. 

  108. #define	A_PR1	512

  109. #define	B_PR1	160

  110. #define BROADCASTKERNEL

  111. 

  112. /*******************************************************************************************

  113. * Macro definitions

  114. *******************************************************************************************/

  115. 

  116. .macro INIT4x12

  117. 

  118. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  119. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  120. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  121. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  122. 	vxorpd		%ymm8 , %ymm8 , %ymm8

  123. 	vxorpd		%ymm9 , %ymm9 , %ymm9

  124. 	vxorpd		%ymm10, %ymm10, %ymm10

  125. 	vxorpd		%ymm11, %ymm11, %ymm11

  126. 	vxorpd		%ymm12, %ymm12, %ymm12

  127. 	vxorpd		%ymm13, %ymm13, %ymm13

  128. 	vxorpd		%ymm14, %ymm14, %ymm14

  129. 	vxorpd		%ymm15, %ymm15, %ymm15

  130. 

  131. .endm

  132. 

  133. .macro KERNEL4x12_I

  134. 	prefetcht0	A_PR1(AO)

  135. 	vmovups		-12 * SIZE(BO), %ymm1

  136. 	prefetcht0	B_PR1(BO)

  137. # if defined BROADCASTKERNEL

  138.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  139. # else

  140. 	vmovups 	-16 * SIZE(AO), %ymm0

  141. # endif

  142. 	prefetcht0	B_PR1+64(BO)

  143. 	vmovups		 -8 * SIZE(BO), %ymm2

  144. 	prefetcht0	B_PR1+128(BO)

  145. 	vmovups		 -4 * SIZE(BO), %ymm3

  146. 	vmulpd  	%ymm0 ,%ymm1  , %ymm4

  147. 	prefetcht0	B_PR1+192(BO)

  148. 	vmulpd  	%ymm0 ,%ymm2  , %ymm8

  149. 	vmulpd  	%ymm0 ,%ymm3  , %ymm12

  150. 	prefetcht0	B_PR1+256(BO)

  151. # if defined BROADCASTKERNEL

  152.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  153. # else

  154. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  155. # endif

  156. 	vmulpd  	%ymm0 ,%ymm1  , %ymm5

  157. 	vmulpd  	%ymm0 ,%ymm2  , %ymm9

  158. 	vmulpd  	%ymm0 ,%ymm3  , %ymm13

  159. # if defined BROADCASTKERNEL

  160.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  161. # else

  162. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  163. # endif

  164. 	vmulpd  	%ymm0 ,%ymm1  , %ymm6

  165. 	vmulpd  	%ymm0 ,%ymm2  , %ymm10

  166. 

  167. 	addq		$ 12*SIZE, BO

  168. 	vmulpd  	%ymm0 ,%ymm3  , %ymm14

  169. # if defined BROADCASTKERNEL

  170.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  171. # else

  172. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  173. # endif

  174. 	vmulpd  	%ymm0 ,%ymm1  , %ymm7

  175. 	vmovups		-12 * SIZE(BO), %ymm1

  176. 	vmulpd  	%ymm0 ,%ymm2  , %ymm11

  177. 	vmovups		 -8 * SIZE(BO), %ymm2

  178. 	vmulpd  	%ymm0 ,%ymm3  , %ymm15

  179. 	vmovups		 -4 * SIZE(BO), %ymm3

  180. 

  181. .endm

  182. 

  183. .macro KERNEL4x12_M1

  184. 	prefetcht0	A_PR1(AO)

  185. # if defined BROADCASTKERNEL

  186.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  187. # else

  188. 	vmovups 	-16 * SIZE(AO), %ymm0

  189. # endif

  190. 	prefetcht0	B_PR1(BO)

  191. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  192. 	prefetcht0	B_PR1+64(BO)

  193. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  194. 	prefetcht0	B_PR1+128(BO)

  195. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  196. # if defined BROADCASTKERNEL

  197.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  198. # else

  199. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  200. # endif

  201. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  202. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  203. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  204. # if defined BROADCASTKERNEL

  205.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  206. # else

  207. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  208. # endif

  209. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  210. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  211. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  212. # if defined BROADCASTKERNEL

  213.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  214. # else

  215. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  216. # endif

  217. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  218. 	vmovups		-12 * SIZE(BO), %ymm1

  219. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  220. 	vmovups		 -8 * SIZE(BO), %ymm2

  221. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  222. 	vmovups		 -4 * SIZE(BO), %ymm3

  223. 

  224. .endm

  225. 

  226. .macro KERNEL4x12_M2

  227. # if defined BROADCASTKERNEL

  228.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  229. # else

  230. 	vmovups 	-12 * SIZE(AO), %ymm0

  231. # endif

  232. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  233. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  234. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  235. # if defined BROADCASTKERNEL

  236.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  237. # else

  238. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  239. # endif

  240. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  241. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  242. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  243. # if defined BROADCASTKERNEL

  244.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  245. # else

  246. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  247. # endif

  248. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  249. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  250. 

  251. 	addq		$ 8*SIZE, AO

  252. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  253. # if defined BROADCASTKERNEL

  254.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  255. # else

  256. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  257. # endif

  258. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  259. 	vmovups		  0 * SIZE(BO), %ymm1

  260. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  261. 	vmovups		  4 * SIZE(BO), %ymm2

  262. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  263. 	vmovups		  8 * SIZE(BO), %ymm3

  264. 	addq		$ 24*SIZE, BO

  265. .endm

  266. 

  267. 

  268. .macro KERNEL4x12_E

  269. # if defined BROADCASTKERNEL

  270.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  271. # else

  272. 	vmovups 	-12 * SIZE(AO), %ymm0

  273. # endif

  274. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  275. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  276. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  277. # if defined BROADCASTKERNEL

  278.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  279. # else

  280. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  281. # endif

  282. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  283. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  284. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  285. # if defined BROADCASTKERNEL

  286.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  287. # else

  288. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  289. # endif

  290. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  291. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  292. 

  293. 	addq		$ 8*SIZE, AO

  294. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  295. # if defined BROADCASTKERNEL

  296.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  297. # else

  298. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  299. # endif

  300. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  301. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  302. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  303. 	addq		$ 12*SIZE, BO

  304. .endm

  305. 

  306. .macro KERNEL4x12_SUB

  307. 	vmovups		-12 * SIZE(BO), %ymm1

  308. # if defined BROADCASTKERNEL

  309.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  310. # else

  311. 	vmovups 	-16 * SIZE(AO), %ymm0

  312. # endif

  313. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  314. 	vmovups		 -8 * SIZE(BO), %ymm2

  315. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  316. 	vmovups		 -4 * SIZE(BO), %ymm3

  317. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12

  318. # if defined BROADCASTKERNEL

  319.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  320. # else

  321. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  322. # endif

  323. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  324. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  325. 	addq		$ 12*SIZE, BO

  326. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13

  327. # if defined BROADCASTKERNEL

  328.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  329. # else

  330. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  331. # endif

  332. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  333. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  334. 	addq		$ 4*SIZE, AO

  335. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14

  336. # if defined BROADCASTKERNEL

  337.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  338. # else

  339. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  340. # endif

  341. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  342. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  343. 	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15

  344. 

  345. .endm

  346. 

  347. 

  348. .macro SAVE4x12

  349. 

  350.         prefetcht0      BUFFER1

  351. 	vbroadcastsd	ALPHA, %ymm0

  352. 

  353. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  354. 	vmulpd	%ymm0 , %ymm5 , %ymm5

  355. 	vmulpd	%ymm0 , %ymm6 , %ymm6

  356. 	vmulpd	%ymm0 , %ymm7 , %ymm7

  357.         prefetcht0      64 + BUFFER1

  358. 	vmulpd	%ymm0 , %ymm8 , %ymm8

  359. 	vmulpd	%ymm0 , %ymm9 , %ymm9

  360. 	vmulpd	%ymm0 , %ymm10, %ymm10

  361. 	vmulpd	%ymm0 , %ymm11, %ymm11

  362. #if B_PR1 > 32

  363.         prefetcht0      128 + BUFFER1

  364. #endif

  365. 	vmulpd	%ymm0 , %ymm12, %ymm12

  366. 	vmulpd	%ymm0 , %ymm13, %ymm13

  367. 	vmulpd	%ymm0 , %ymm14, %ymm14

  368. 	vmulpd	%ymm0 , %ymm15, %ymm15

  369. #if B_PR1 > 96

  370.         prefetcht0      192 + BUFFER1

  371. #endif

  372. 

  373. #if defined BROADCASTKERNEL

  374.         vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0

  375.         vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1

  376.         vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2

  377.         vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3

  378. #else

  379. 	vpermilpd $ 0x05 , %ymm5, %ymm5

  380. 	vpermilpd $ 0x05 , %ymm7, %ymm7

  381. #endif

  382. 

  383. #if B_PR1 > 160

  384.         prefetcht0      256 + BUFFER1

  385. #endif

  386. 

  387. #if defined BROADCASTKERNEL

  388.         vunpcklpd %ymm1, %ymm0, %ymm4

  389.         vunpckhpd %ymm1, %ymm0, %ymm5

  390.         vunpcklpd %ymm3, %ymm2, %ymm6

  391.         vunpckhpd %ymm3, %ymm2, %ymm7

  392. #else

  393. 	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0

  394. 	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1

  395. 	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2

  396. 	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3

  397. #endif

  398. 

  399. #if B_PR1 > 224

  400.         prefetcht0      320 + BUFFER1

  401. #endif

  402. 

  403. #ifndef BROADCASTKERNEL

  404. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  405. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  406. #endif

  407. 

  408. #if B_PR1 > 288

  409.         prefetcht0      384 + BUFFER1

  410. #endif

  411. 

  412. #ifndef BROADCASTKERNEL

  413. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  414. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  415. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  416. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  417. #endif

  418. 

  419. #if B_PR1 > 352

  420.         prefetcht0      448 + BUFFER1

  421. #endif

  422.         leaq    (CO1, LDC, 2), %rax     

  423. 	

  424. #if B_PR1 > 416

  425.         prefetcht0      512 + BUFFER1

  426. #endif

  427. 

  428. #if !defined(TRMMKERNEL)

  429. 

  430. 	vaddpd 	                (CO1), %ymm4, %ymm4

  431. 	vaddpd 	           (CO1, LDC), %ymm5, %ymm5

  432. 	vaddpd 	               (%rax), %ymm6, %ymm6

  433. 	vaddpd 	          (%rax, LDC), %ymm7, %ymm7

  434. 

  435. #endif

  436. 

  437. 	vmovups	%ymm4 ,  	(CO1)

  438. 	vmovups	%ymm5 ,  	(CO1, LDC)

  439. 	vmovups	%ymm6 ,  	(%rax)

  440. 	vmovups	%ymm7 ,  	(%rax, LDC)

  441. 

  442. 	prefetcht1	56(CO1)

  443. 	prefetcht1	56(CO1,LDC)

  444. 	prefetcht1	56(%rax)

  445. 	prefetcht1	56(%rax,LDC)

  446. 

  447. #if defined BROADCASTKERNEL

  448.         vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0

  449.         vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1

  450.         vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2

  451.         vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3

  452.         vunpcklpd %ymm1, %ymm0, %ymm4

  453.         vunpckhpd %ymm1, %ymm0, %ymm5

  454.         vunpcklpd %ymm3, %ymm2, %ymm6

  455.         vunpckhpd %ymm3, %ymm2, %ymm7

  456. #else

  457. 	vpermilpd $ 0x05 , %ymm9, %ymm9

  458. 	vpermilpd $ 0x05 , %ymm11, %ymm11

  459. 

  460. 	vblendpd $ 0x0a, %ymm9, %ymm8, %ymm0

  461. 	vblendpd $ 0x05, %ymm9, %ymm8, %ymm1

  462. 	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2

  463. 	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3

  464. 

  465. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  466. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  467. 

  468. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  469. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  470. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  471. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  472. #endif

  473. 

  474. 	leaq	(%rax, LDC, 2), %rax

  475. 	leaq	(%rax, LDC, 2), %rbp

  476. 

  477. #if !defined(TRMMKERNEL)

  478. 

  479. 	vaddpd 	                (%rax), %ymm4, %ymm4

  480. 	vaddpd 	           (%rax, LDC), %ymm5, %ymm5

  481. 	vaddpd 	                (%rbp), %ymm6, %ymm6

  482. 	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7

  483. 

  484. #endif

  485. 

  486. 	vmovups	%ymm4 ,  	(%rax)

  487. 	vmovups	%ymm5 ,  	(%rax, LDC)

  488. 	vmovups	%ymm6 ,  	(%rbp)

  489. 	vmovups	%ymm7 ,  	(%rbp, LDC)

  490. 

  491. 	prefetcht1	56(%rax)

  492. 	prefetcht1	56(%rax,LDC)

  493. 	prefetcht1	56(%rbp)

  494. 	prefetcht1	56(%rbp,LDC)

  495. 

  496. #if defined BROADCASTKERNEL

  497.         vperm2f128 $ 0x20 , %ymm14, %ymm12 , %ymm0

  498.         vperm2f128 $ 0x20 , %ymm15, %ymm13 , %ymm1

  499.         vperm2f128 $ 0x31 , %ymm14, %ymm12 , %ymm2

  500.         vperm2f128 $ 0x31 , %ymm15, %ymm13 , %ymm3

  501.         vunpcklpd %ymm1, %ymm0, %ymm4

  502.         vunpckhpd %ymm1, %ymm0, %ymm5

  503.         vunpcklpd %ymm3, %ymm2, %ymm6

  504.         vunpckhpd %ymm3, %ymm2, %ymm7

  505. #else

  506. 	vpermilpd $ 0x05 , %ymm13, %ymm13

  507. 	vpermilpd $ 0x05 , %ymm15, %ymm15

  508. 

  509. 	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0

  510. 	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1

  511. 	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2

  512. 	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3

  513. 

  514. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  515. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  516. 

  517. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  518. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  519. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  520. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  521. #endif

  522. 

  523. 	leaq	(%rax, LDC, 4), %rax

  524. 	leaq	(%rbp, LDC, 4), %rbp

  525. 

  526. #if !defined(TRMMKERNEL)

  527. 

  528. 	vaddpd 	                (%rax), %ymm4, %ymm4

  529. 	vaddpd 	           (%rax, LDC), %ymm5, %ymm5

  530. 	vaddpd 	                (%rbp), %ymm6, %ymm6

  531. 	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7

  532. 

  533. #endif

  534. 

  535. 	vmovups	%ymm4 ,  	(%rax)

  536. 	vmovups	%ymm5 ,  	(%rax, LDC)

  537. 	vmovups	%ymm6 ,  	(%rbp)

  538. 	vmovups	%ymm7 ,  	(%rbp, LDC)

  539. 

  540. 	prefetcht1	56(%rax)

  541. 	prefetcht1	56(%rax,LDC)

  542. 	prefetcht1	56(%rbp)

  543. 	prefetcht1	56(%rbp,LDC)

  544. 

  545. 	addq	$ 4*SIZE, CO1

  546. .endm

  547. 

  548. /******************************************************************************************/

  549. 

  550. .macro INIT2x12

  551. 

  552. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  553. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  554. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  555. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  556. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  557. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  558. 	vxorpd		%xmm10, %xmm10, %xmm10

  559. 	vxorpd		%xmm11, %xmm11, %xmm11

  560. 	vxorpd		%xmm12, %xmm12, %xmm12

  561. 	vxorpd		%xmm13, %xmm13, %xmm13

  562. 	vxorpd		%xmm14, %xmm14, %xmm14

  563. 	vxorpd		%xmm15, %xmm15, %xmm15

  564. 

  565. .endm

  566. 

  567. .macro KERNEL2x12_SUB

  568. 	vmovups 	-16 * SIZE(AO), %xmm0

  569. 	vmovddup	-12 * SIZE(BO), %xmm1

  570. 	vmovddup	-11 * SIZE(BO), %xmm2

  571. 	vmovddup	-10 * SIZE(BO), %xmm3

  572. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4

  573. 	vmovddup	 -9 * SIZE(BO), %xmm1

  574. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5

  575. 	vmovddup	 -8 * SIZE(BO), %xmm2

  576. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  577. 	vmovddup	 -7 * SIZE(BO), %xmm3

  578. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7

  579. 	vmovddup	 -6 * SIZE(BO), %xmm1

  580. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8

  581. 	vmovddup	 -5 * SIZE(BO), %xmm2

  582. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9

  583. 	vmovddup	 -4 * SIZE(BO), %xmm3

  584. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10

  585. 	vmovddup	 -3 * SIZE(BO), %xmm1

  586. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11

  587. 	vmovddup	 -2 * SIZE(BO), %xmm2

  588. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12

  589. 	vmovddup	 -1 * SIZE(BO), %xmm3

  590. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13

  591. 	addq		$ 12*SIZE, BO

  592. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14

  593. 	addq		$ 2*SIZE, AO

  594. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15

  595. 

  596. .endm

  597. 

  598. .macro SAVE2x12

  599. 

  600. 	vmovddup	ALPHA, %xmm0

  601. 

  602. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  603. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  604. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  605. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  606. 

  607. 	vmulpd	%xmm0 , %xmm8 , %xmm8

  608. 	vmulpd	%xmm0 , %xmm9 , %xmm9

  609. 	vmulpd	%xmm0 , %xmm10, %xmm10

  610. 	vmulpd	%xmm0 , %xmm11, %xmm11

  611. 

  612. 	vmulpd	%xmm0 , %xmm12, %xmm12

  613. 	vmulpd	%xmm0 , %xmm13, %xmm13

  614. 	vmulpd	%xmm0 , %xmm14, %xmm14

  615. 	vmulpd	%xmm0 , %xmm15, %xmm15

  616. 

  617. 

  618.         leaq    (CO1, LDC, 2), %rax     

  619. 	

  620. 

  621. #if !defined(TRMMKERNEL)

  622. 

  623. 	vaddpd 	                (CO1), %xmm4, %xmm4

  624. 	vaddpd 	           (CO1, LDC), %xmm5, %xmm5

  625. 	vaddpd 	               (%rax), %xmm6, %xmm6

  626. 	vaddpd 	          (%rax, LDC), %xmm7, %xmm7

  627. 

  628. #endif

  629. 

  630. 	vmovups	%xmm4 ,  	(CO1)

  631. 	vmovups	%xmm5 ,  	(CO1, LDC)

  632. 	vmovups	%xmm6 ,  	(%rax)

  633. 	vmovups	%xmm7 ,  	(%rax, LDC)

  634. 

  635. 

  636. 	leaq	(%rax, LDC, 2), %rax

  637. 	leaq	(%rax, LDC, 2), %rbp

  638. 

  639. #if !defined(TRMMKERNEL)

  640. 

  641. 	vaddpd 	                (%rax), %xmm8 , %xmm4

  642. 	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5

  643. 	vaddpd 	                (%rbp), %xmm10, %xmm6

  644. 	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7

  645. 

  646. #endif

  647. 

  648. 	vmovups	%xmm4 ,  	(%rax)

  649. 	vmovups	%xmm5 ,  	(%rax, LDC)

  650. 	vmovups	%xmm6 ,  	(%rbp)

  651. 	vmovups	%xmm7 ,  	(%rbp, LDC)

  652. 

  653. 

  654. 	leaq	(%rax, LDC, 4), %rax

  655. 	leaq	(%rbp, LDC, 4), %rbp

  656. 

  657. #if !defined(TRMMKERNEL)

  658. 

  659. 	vaddpd 	                (%rax), %xmm12, %xmm4

  660. 	vaddpd 	           (%rax, LDC), %xmm13, %xmm5

  661. 	vaddpd 	                (%rbp), %xmm14, %xmm6

  662. 	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7

  663. 

  664. #endif

  665. 

  666. 	vmovups	%xmm4 ,  	(%rax)

  667. 	vmovups	%xmm5 ,  	(%rax, LDC)

  668. 	vmovups	%xmm6 ,  	(%rbp)

  669. 	vmovups	%xmm7 ,  	(%rbp, LDC)

  670. 

  671. 	addq	$ 2*SIZE, CO1

  672. .endm

  673. 

  674. 

  675. /******************************************************************************************/

  676. 

  677. .macro INIT1x12

  678. 

  679. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  680. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  681. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  682. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  683. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  684. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  685. 	vxorpd		%xmm10, %xmm10, %xmm10

  686. 	vxorpd		%xmm11, %xmm11, %xmm11

  687. 	vxorpd		%xmm12, %xmm12, %xmm12

  688. 	vxorpd		%xmm13, %xmm13, %xmm13

  689. 	vxorpd		%xmm14, %xmm14, %xmm14

  690. 	vxorpd		%xmm15, %xmm15, %xmm15

  691. 

  692. .endm

  693. 

  694. .macro KERNEL1x12_SUB

  695. 	vmovsd 	-16 * SIZE(AO), %xmm0

  696. 	vmovsd	-12 * SIZE(BO), %xmm1

  697. 	vmovsd	-11 * SIZE(BO), %xmm2

  698. 	vmovsd	-10 * SIZE(BO), %xmm3

  699. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  700. 	vmovsd	 -9 * SIZE(BO), %xmm1

  701. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  702. 	vmovsd	 -8 * SIZE(BO), %xmm2

  703. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6

  704. 	vmovsd	 -7 * SIZE(BO), %xmm3

  705. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7

  706. 	vmovsd	 -6 * SIZE(BO), %xmm1

  707. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8

  708. 	vmovsd	 -5 * SIZE(BO), %xmm2

  709. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9

  710. 	vmovsd	 -4 * SIZE(BO), %xmm3

  711. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10

  712. 	vmovsd	 -3 * SIZE(BO), %xmm1

  713. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11

  714. 	vmovsd	 -2 * SIZE(BO), %xmm2

  715. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12

  716. 	vmovsd	 -1 * SIZE(BO), %xmm3

  717. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13

  718. 	addq		$ 12*SIZE, BO

  719. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14

  720. 	addq		$ 1*SIZE, AO

  721. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15

  722. 

  723. .endm

  724. 

  725. .macro SAVE1x12

  726. 

  727. 	vmovsd	ALPHA, %xmm0

  728. 

  729. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  730. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  731. 	vmulsd	%xmm0 , %xmm6 , %xmm6

  732. 	vmulsd	%xmm0 , %xmm7 , %xmm7

  733. 

  734. 	vmulsd	%xmm0 , %xmm8 , %xmm8

  735. 	vmulsd	%xmm0 , %xmm9 , %xmm9

  736. 	vmulsd	%xmm0 , %xmm10, %xmm10

  737. 	vmulsd	%xmm0 , %xmm11, %xmm11

  738. 

  739. 	vmulsd	%xmm0 , %xmm12, %xmm12

  740. 	vmulsd	%xmm0 , %xmm13, %xmm13

  741. 	vmulsd	%xmm0 , %xmm14, %xmm14

  742. 	vmulsd	%xmm0 , %xmm15, %xmm15

  743. 

  744. 

  745.         leaq    (CO1, LDC, 2), %rax     

  746. 	

  747. 

  748. #if !defined(TRMMKERNEL)

  749. 

  750. 	vaddsd 	                (CO1), %xmm4, %xmm4

  751. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  752. 	vaddsd 	               (%rax), %xmm6, %xmm6

  753. 	vaddsd 	          (%rax, LDC), %xmm7, %xmm7

  754. 

  755. #endif

  756. 

  757. 	vmovsd	%xmm4 ,  	(CO1)

  758. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  759. 	vmovsd	%xmm6 ,  	(%rax)

  760. 	vmovsd	%xmm7 ,  	(%rax, LDC)

  761. 

  762. 

  763. 	leaq	(%rax, LDC, 2), %rax

  764. 	leaq	(%rax, LDC, 2), %rbp

  765. 

  766. #if !defined(TRMMKERNEL)

  767. 

  768. 	vaddsd 	                (%rax), %xmm8 , %xmm4

  769. 	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5

  770. 	vaddsd 	                (%rbp), %xmm10, %xmm6

  771. 	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7

  772. 

  773. #endif

  774. 

  775. 	vmovsd	%xmm4 ,  	(%rax)

  776. 	vmovsd	%xmm5 ,  	(%rax, LDC)

  777. 	vmovsd	%xmm6 ,  	(%rbp)

  778. 	vmovsd	%xmm7 ,  	(%rbp, LDC)

  779. 

  780. 

  781. 	leaq	(%rax, LDC, 4), %rax

  782. 	leaq	(%rbp, LDC, 4), %rbp

  783. 

  784. #if !defined(TRMMKERNEL)

  785. 

  786. 	vaddsd 	                (%rax), %xmm12, %xmm4

  787. 	vaddsd 	           (%rax, LDC), %xmm13, %xmm5

  788. 	vaddsd 	                (%rbp), %xmm14, %xmm6

  789. 	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7

  790. 

  791. #endif

  792. 

  793. 	vmovsd	%xmm4 ,  	(%rax)

  794. 	vmovsd	%xmm5 ,  	(%rax, LDC)

  795. 	vmovsd	%xmm6 ,  	(%rbp)

  796. 	vmovsd	%xmm7 ,  	(%rbp, LDC)

  797. 

  798. 	addq	$ 1*SIZE, CO1

  799. .endm

  800. 

  801. 

  802. 

  803. 

  804. /******************************************************************************************/

  805. 

  806. 

  807. .macro INIT4x8

  808. 

  809. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  810. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  811. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  812. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  813. 	vxorpd		%ymm8 , %ymm8 , %ymm8

  814. 	vxorpd		%ymm9 , %ymm9 , %ymm9

  815. 	vxorpd		%ymm10, %ymm10, %ymm10

  816. 	vxorpd		%ymm11, %ymm11, %ymm11

  817. 

  818. .endm

  819. 

  820. .macro KERNEL4x8_I

  821. 	vmovups		-12 * SIZE(BO), %ymm1

  822. #if defined BROADCASTKERNEL

  823.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  824. #else

  825. 	vmovups 	-16 * SIZE(AO), %ymm0

  826. #endif

  827. 	vmovups		 -8 * SIZE(BO), %ymm2

  828. 	vmulpd  	%ymm0 ,%ymm1  , %ymm4

  829. 	vmulpd  	%ymm0 ,%ymm2  , %ymm8

  830. #if defined BROADCASTKERNEL

  831.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  832. #else

  833. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  834. #endif

  835. 	vmulpd  	%ymm0 ,%ymm1  , %ymm5

  836. 	vmulpd  	%ymm0 ,%ymm2  , %ymm9

  837. #if defined BROADCASTKERNEL

  838.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  839. #else

  840. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  841. #endif

  842. 	vmulpd  	%ymm0 ,%ymm1  , %ymm6

  843. 	vmulpd  	%ymm0 ,%ymm2  , %ymm10

  844. 

  845. 	addq		$  8*SIZE, BO

  846. #if defined BROADCASTKERNEL

  847.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  848. #else

  849. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  850. #endif

  851. 	vmulpd  	%ymm0 ,%ymm1  , %ymm7

  852. 	vmovups		-12 * SIZE(BO), %ymm1

  853. 	vmulpd  	%ymm0 ,%ymm2  , %ymm11

  854. 	vmovups		 -8 * SIZE(BO), %ymm2

  855. 

  856. .endm

  857. 

  858. .macro KERNEL4x8_M1

  859. 	prefetcht0	A_PR1(AO)

  860. #if defined BROADCASTKERNEL

  861.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  862. #else

  863. 	vmovups 	-16 * SIZE(AO), %ymm0

  864. #endif

  865. 	prefetcht0	B_PR1(BO)

  866. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  867. 	prefetcht0	B_PR1+64(BO)

  868. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  869. #if defined BROADCASTKERNEL

  870.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  871. #else

  872. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  873. #endif

  874. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  875. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  876. #if defined BROADCASTKERNEL

  877.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  878. #else

  879. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  880. #endif

  881. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  882. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  883. #if defined BROADCASTKERNEL

  884.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  885. #else

  886. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  887. #endif

  888. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  889. 	vmovups		-12 * SIZE(BO), %ymm1

  890. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  891. 	vmovups		 -8 * SIZE(BO), %ymm2

  892. 

  893. .endm

  894. 

  895. .macro KERNEL4x8_M2

  896. #if defined BROADCASTKERNEL

  897.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  898. #else

  899. 	vmovups 	-12 * SIZE(AO), %ymm0

  900. #endif

  901. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  902. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  903. #if defined BROADCASTKERNEL

  904.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  905. #else

  906. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  907. #endif

  908. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  909. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  910. #if defined BROADCASTKERNEL

  911.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  912. #else

  913. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  914. #endif

  915. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  916. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  917. 

  918. 	addq		$ 8*SIZE, AO

  919. #if defined BROADCASTKERNEL

  920.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  921. #else

  922. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  923. #endif

  924. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  925. 	vmovups		 -4 * SIZE(BO), %ymm1

  926. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  927. 	vmovups		  0 * SIZE(BO), %ymm2

  928. 	addq		$ 16*SIZE, BO

  929. .endm

  930. 

  931. 

  932. .macro KERNEL4x8_E

  933. #if defined BROADCASTKERNEL

  934.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  935. #else

  936. 	vmovups 	-12 * SIZE(AO), %ymm0

  937. #endif

  938. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  939. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  940. #if defined BROADCASTKERNEL

  941.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  942. #else

  943. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  944. #endif

  945. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  946. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  947. #if defined BROADCASTKERNEL

  948.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  949. #else

  950. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  951. #endif

  952. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  953. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  954. 

  955. 	addq		$ 8*SIZE, AO

  956. #if defined BROADCASTKERNEL

  957.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  958. #else

  959. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  960. #endif

  961. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  962. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  963. 	addq		$  8*SIZE, BO

  964. .endm

  965. 

  966. .macro KERNEL4x8_SUB

  967. 	vmovups		-12 * SIZE(BO), %ymm1

  968. #if defined BROADCASTKERNEL

  969.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  970. #else

  971. 	vmovups 	-16 * SIZE(AO), %ymm0

  972. #endif

  973. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  974. 	vmovups		 -8 * SIZE(BO), %ymm2

  975. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8

  976. #if defined BROADCASTKERNEL

  977.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  978. #else

  979. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  980. #endif

  981. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  982. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9

  983. 	addq		$  8*SIZE, BO

  984. #if defined BROADCASTKERNEL

  985.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  986. #else

  987. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  988. #endif

  989. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  990. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10

  991. 	addq		$ 4*SIZE, AO

  992. #if defined BROADCASTKERNEL

  993.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  994. #else

  995. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  996. #endif

  997. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  998. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11

  999. 

  1000. .endm

  1001. 

  1002. 

  1003. .macro SAVE4x8

  1004. 

  1005. 	vbroadcastsd	ALPHA, %ymm0

  1006. 

  1007. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  1008. 	vmulpd	%ymm0 , %ymm5 , %ymm5

  1009. 	vmulpd	%ymm0 , %ymm6 , %ymm6

  1010. 	vmulpd	%ymm0 , %ymm7 , %ymm7

  1011. 

  1012. 	vmulpd	%ymm0 , %ymm8 , %ymm8

  1013. 	vmulpd	%ymm0 , %ymm9 , %ymm9

  1014. 	vmulpd	%ymm0 , %ymm10, %ymm10

  1015. 	vmulpd	%ymm0 , %ymm11, %ymm11

  1016. 

  1017. #if defined BROADCASTKERNEL

  1018.         vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0

  1019.         vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1

  1020.         vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2

  1021.         vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3

  1022.         vunpcklpd %ymm1, %ymm0, %ymm4

  1023.         vunpckhpd %ymm1, %ymm0, %ymm5

  1024.         vunpcklpd %ymm3, %ymm2, %ymm6

  1025.         vunpckhpd %ymm3, %ymm2, %ymm7

  1026. #else

  1027. 	vpermilpd $ 0x05 , %ymm5, %ymm5

  1028. 	vpermilpd $ 0x05 , %ymm7, %ymm7

  1029. 

  1030. 	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0

  1031. 	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1

  1032. 	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2

  1033. 	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3

  1034. 

  1035. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  1036. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  1037. 

  1038. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  1039. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  1040. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  1041. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  1042. #endif

  1043. 

  1044.         leaq    (CO1, LDC, 2), %rax     

  1045. 	

  1046. 

  1047. #if !defined(TRMMKERNEL)

  1048. 

  1049. 	vaddpd 	                (CO1), %ymm4, %ymm4

  1050. 	vaddpd 	           (CO1, LDC), %ymm5, %ymm5

  1051. 	vaddpd 	               (%rax), %ymm6, %ymm6

  1052. 	vaddpd 	          (%rax, LDC), %ymm7, %ymm7

  1053. 

  1054. #endif

  1055. 

  1056. 	vmovups	%ymm4 ,  	(CO1)

  1057. 	vmovups	%ymm5 ,  	(CO1, LDC)

  1058. 	vmovups	%ymm6 ,  	(%rax)

  1059. 	vmovups	%ymm7 ,  	(%rax, LDC)

  1060. 

  1061. 	prefetcht0	56(CO1)

  1062. 	prefetcht0	56(CO1,LDC)

  1063. 	prefetcht0	56(%rax)

  1064. 	prefetcht0	56(%rax,LDC)

  1065. 

  1066. #if defined BROADCASTKERNEL

  1067.         vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0

  1068.         vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1

  1069.         vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2

  1070.         vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3

  1071.         vunpcklpd %ymm1, %ymm0, %ymm4

  1072.         vunpckhpd %ymm1, %ymm0, %ymm5

  1073.         vunpcklpd %ymm3, %ymm2, %ymm6

  1074.         vunpckhpd %ymm3, %ymm2, %ymm7

  1075. #else

  1076. 	vpermilpd $ 0x05 , %ymm9 , %ymm9

  1077. 	vpermilpd $ 0x05 , %ymm11, %ymm11

  1078. 

  1079. 	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0

  1080. 	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1

  1081. 	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2

  1082. 	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3

  1083. 

  1084. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  1085. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  1086. 

  1087. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  1088. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  1089. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  1090. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  1091. #endif

  1092. 

  1093. 	leaq	(%rax, LDC, 2), %rax

  1094. 	leaq	(%rax, LDC, 2), %rbp

  1095. 

  1096. #if !defined(TRMMKERNEL)

  1097. 

  1098. 	vaddpd 	                (%rax), %ymm4, %ymm4

  1099. 	vaddpd 	           (%rax, LDC), %ymm5, %ymm5

  1100. 	vaddpd 	                (%rbp), %ymm6, %ymm6

  1101. 	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7

  1102. 

  1103. #endif

  1104. 

  1105. 	vmovups	%ymm4 ,  	(%rax)

  1106. 	vmovups	%ymm5 ,  	(%rax, LDC)

  1107. 	vmovups	%ymm6 ,  	(%rbp)

  1108. 	vmovups	%ymm7 ,  	(%rbp, LDC)

  1109. 

  1110. 	prefetcht0	56(%rax)

  1111. 	prefetcht0	56(%rax,LDC)

  1112. 	prefetcht0	56(%rbp)

  1113. 	prefetcht0	56(%rbp,LDC)

  1114. 

  1115. 	addq	$ 4*SIZE, CO1

  1116. .endm

  1117. 

  1118. /******************************************************************************************/

  1119. 

  1120. .macro INIT2x8

  1121. 

  1122. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1123. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1124. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1125. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  1126. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  1127. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  1128. 	vxorpd		%xmm10, %xmm10, %xmm10

  1129. 	vxorpd		%xmm11, %xmm11, %xmm11

  1130. 

  1131. .endm

  1132. 

  1133. .macro KERNEL2x8_SUB

  1134. 	vmovups 	-16 * SIZE(AO), %xmm0

  1135. 	vmovddup	-12 * SIZE(BO), %xmm1

  1136. 	vmovddup	-11 * SIZE(BO), %xmm2

  1137. 	vmovddup	-10 * SIZE(BO), %xmm3

  1138. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4

  1139. 	vmovddup	 -9 * SIZE(BO), %xmm1

  1140. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5

  1141. 	vmovddup	 -8 * SIZE(BO), %xmm2

  1142. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  1143. 	vmovddup	 -7 * SIZE(BO), %xmm3

  1144. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7

  1145. 	vmovddup	 -6 * SIZE(BO), %xmm1

  1146. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8

  1147. 	vmovddup	 -5 * SIZE(BO), %xmm2

  1148. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9

  1149. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10

  1150. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11

  1151. 	addq		$  8*SIZE, BO

  1152. 	addq		$ 2*SIZE, AO

  1153. 

  1154. .endm

  1155. 

  1156. .macro SAVE2x8

  1157. 

  1158. 	vmovddup	ALPHA, %xmm0

  1159. 

  1160. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1161. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  1162. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  1163. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  1164. 

  1165. 	vmulpd	%xmm0 , %xmm8 , %xmm8

  1166. 	vmulpd	%xmm0 , %xmm9 , %xmm9

  1167. 	vmulpd	%xmm0 , %xmm10, %xmm10

  1168. 	vmulpd	%xmm0 , %xmm11, %xmm11

  1169. 

  1170.         leaq    (CO1, LDC, 2), %rax     

  1171. 	

  1172. 

  1173. #if !defined(TRMMKERNEL)

  1174. 

  1175. 	vaddpd 	                (CO1), %xmm4, %xmm4

  1176. 	vaddpd 	           (CO1, LDC), %xmm5, %xmm5

  1177. 	vaddpd 	               (%rax), %xmm6, %xmm6

  1178. 	vaddpd 	          (%rax, LDC), %xmm7, %xmm7

  1179. 

  1180. #endif

  1181. 

  1182. 	vmovups	%xmm4 ,  	(CO1)

  1183. 	vmovups	%xmm5 ,  	(CO1, LDC)

  1184. 	vmovups	%xmm6 ,  	(%rax)

  1185. 	vmovups	%xmm7 ,  	(%rax, LDC)

  1186. 

  1187. 

  1188. 	leaq	(%rax, LDC, 2), %rax

  1189. 	leaq	(%rax, LDC, 2), %rbp

  1190. 

  1191. #if !defined(TRMMKERNEL)

  1192. 

  1193. 	vaddpd 	                (%rax), %xmm8 , %xmm4

  1194. 	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5

  1195. 	vaddpd 	                (%rbp), %xmm10, %xmm6

  1196. 	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7

  1197. 

  1198. #endif

  1199. 

  1200. 	vmovups	%xmm4 ,  	(%rax)

  1201. 	vmovups	%xmm5 ,  	(%rax, LDC)

  1202. 	vmovups	%xmm6 ,  	(%rbp)

  1203. 	vmovups	%xmm7 ,  	(%rbp, LDC)

  1204. 

  1205. 	addq	$ 2*SIZE, CO1

  1206. .endm

  1207. 

  1208. 

  1209. /******************************************************************************************/

  1210. 

  1211. .macro INIT1x8

  1212. 

  1213. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1214. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1215. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1216. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  1217. 	vxorpd		%xmm8 , %xmm8 , %xmm8

  1218. 	vxorpd		%xmm9 , %xmm9 , %xmm9

  1219. 	vxorpd		%xmm10, %xmm10, %xmm10

  1220. 	vxorpd		%xmm11, %xmm11, %xmm11

  1221. 

  1222. .endm

  1223. 

  1224. .macro KERNEL1x8_SUB

  1225. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1226. 	vmovsd	-12 * SIZE(BO), %xmm1

  1227. 	vmovsd	-11 * SIZE(BO), %xmm2

  1228. 	vmovsd	-10 * SIZE(BO), %xmm3

  1229. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1230. 	vmovsd	 -9 * SIZE(BO), %xmm1

  1231. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  1232. 	vmovsd	 -8 * SIZE(BO), %xmm2

  1233. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6

  1234. 	vmovsd	 -7 * SIZE(BO), %xmm3

  1235. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7

  1236. 	vmovsd	 -6 * SIZE(BO), %xmm1

  1237. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8

  1238. 	vmovsd	 -5 * SIZE(BO), %xmm2

  1239. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9

  1240. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10

  1241. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11

  1242. 	addq		$  8*SIZE, BO

  1243. 	addq		$ 1*SIZE, AO

  1244. 

  1245. .endm

  1246. 

  1247. .macro SAVE1x8

  1248. 

  1249. 	vmovsd	ALPHA, %xmm0

  1250. 

  1251. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1252. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  1253. 	vmulsd	%xmm0 , %xmm6 , %xmm6

  1254. 	vmulsd	%xmm0 , %xmm7 , %xmm7

  1255. 

  1256. 	vmulsd	%xmm0 , %xmm8 , %xmm8

  1257. 	vmulsd	%xmm0 , %xmm9 , %xmm9

  1258. 	vmulsd	%xmm0 , %xmm10, %xmm10

  1259. 	vmulsd	%xmm0 , %xmm11, %xmm11

  1260. 

  1261.         leaq    (CO1, LDC, 2), %rax     

  1262. 	

  1263. 

  1264. #if !defined(TRMMKERNEL)

  1265. 

  1266. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1267. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  1268. 	vaddsd 	               (%rax), %xmm6, %xmm6

  1269. 	vaddsd 	          (%rax, LDC), %xmm7, %xmm7

  1270. 

  1271. #endif

  1272. 

  1273. 	vmovsd	%xmm4 ,  	(CO1)

  1274. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  1275. 	vmovsd	%xmm6 ,  	(%rax)

  1276. 	vmovsd	%xmm7 ,  	(%rax, LDC)

  1277. 

  1278. 

  1279. 	leaq	(%rax, LDC, 2), %rax

  1280. 	leaq	(%rax, LDC, 2), %rbp

  1281. 

  1282. #if !defined(TRMMKERNEL)

  1283. 

  1284. 	vaddsd 	                (%rax), %xmm8 , %xmm4

  1285. 	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5

  1286. 	vaddsd 	                (%rbp), %xmm10, %xmm6

  1287. 	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7

  1288. 

  1289. #endif

  1290. 

  1291. 	vmovsd	%xmm4 ,  	(%rax)

  1292. 	vmovsd	%xmm5 ,  	(%rax, LDC)

  1293. 	vmovsd	%xmm6 ,  	(%rbp)

  1294. 	vmovsd	%xmm7 ,  	(%rbp, LDC)

  1295. 

  1296. 	addq	$ 1*SIZE, CO1

  1297. .endm

  1298. 

  1299. 

  1300. 

  1301. 

  1302. 

  1303. /******************************************************************************************/

  1304. 

  1305. .macro INIT4x4

  1306. 

  1307. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  1308. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  1309. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  1310. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  1311. 

  1312. .endm

  1313. 

  1314. .macro KERNEL4x4_I

  1315. 	prefetcht0	A_PR1(AO)

  1316. 	vmovups		-12 * SIZE(BO), %ymm1

  1317. #if defined BROADCASTKERNEL

  1318.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  1319. #else

  1320. 	vmovups 	-16 * SIZE(AO), %ymm0

  1321. #endif

  1322. 	vmulpd  	%ymm0 ,%ymm1  , %ymm4

  1323. #if defined BROADCASTKERNEL

  1324.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  1325. #else

  1326. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1327. #endif

  1328. 	vmulpd  	%ymm0 ,%ymm1  , %ymm5

  1329. #if defined BROADCASTKERNEL

  1330.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  1331. #else

  1332. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  1333. #endif

  1334. 	vmulpd  	%ymm0 ,%ymm1  , %ymm6

  1335. 

  1336. 	addq		$ 4*SIZE, BO

  1337. #if defined BROADCASTKERNEL

  1338.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  1339. #else

  1340. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1341. #endif

  1342. 	vmulpd  	%ymm0 ,%ymm1  , %ymm7

  1343. 	vmovups		-12 * SIZE(BO), %ymm1

  1344. 

  1345. .endm

  1346. 

  1347. .macro KERNEL4x4_M1

  1348. 	prefetcht0	A_PR1(AO)

  1349. #if defined BROADCASTKERNEL

  1350.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  1351. #else

  1352. 	vmovups 	-16 * SIZE(AO), %ymm0

  1353. #endif

  1354. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  1355. #if defined BROADCASTKERNEL

  1356.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  1357. #else

  1358. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1359. #endif

  1360. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  1361. #if defined BROADCASTKERNEL

  1362.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  1363. #else

  1364. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  1365. #endif

  1366. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  1367. #if defined BROADCASTKERNEL

  1368.         vbroadcastsd    -13 * SIZE(AO), %ymm0

  1369. #else

  1370. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1371. #endif

  1372. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  1373. 	vmovups		-12 * SIZE(BO), %ymm1

  1374. 

  1375. .endm

  1376. 

  1377. .macro KERNEL4x4_M2

  1378. #if defined BROADCASTKERNEL

  1379.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  1380. #else

  1381. 	vmovups 	-12 * SIZE(AO), %ymm0

  1382. #endif

  1383. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  1384. #if defined BROADCASTKERNEL

  1385.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  1386. #else

  1387. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1388. #endif

  1389. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  1390. #if defined BROADCASTKERNEL

  1391.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  1392. #else

  1393. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  1394. #endif

  1395. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  1396. 

  1397. 	addq		$ 8*SIZE, AO

  1398. #if defined BROADCASTKERNEL

  1399.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  1400. #else

  1401. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1402. #endif

  1403. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  1404. 	vmovups		 -8 * SIZE(BO), %ymm1

  1405. 	addq		$ 8*SIZE, BO

  1406. .endm

  1407. 

  1408. 

  1409. .macro KERNEL4x4_E

  1410. #if defined BROADCASTKERNEL

  1411.         vbroadcastsd    -12 * SIZE(AO), %ymm0

  1412. #else

  1413. 	vmovups 	-12 * SIZE(AO), %ymm0

  1414. #endif

  1415. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  1416. #if defined BROADCASTKERNEL

  1417.         vbroadcastsd    -11 * SIZE(AO), %ymm0

  1418. #else

  1419. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1420. #endif

  1421. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  1422. #if defined BROADCASTKERNEL

  1423.         vbroadcastsd    -10 * SIZE(AO), %ymm0

  1424. #else

  1425. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  1426. #endif

  1427. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  1428. 

  1429. 	addq		$ 8*SIZE, AO

  1430. #if defined BROADCASTKERNEL

  1431.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  1432. #else

  1433. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1434. #endif

  1435. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  1436. 	addq		$ 4*SIZE, BO

  1437. .endm

  1438. 

  1439. .macro KERNEL4x4_SUB

  1440. 	vmovups		-12 * SIZE(BO), %ymm1

  1441. #if defined BROADCASTKERNEL

  1442.         vbroadcastsd    -16 * SIZE(AO), %ymm0

  1443. #else

  1444. 	vmovups 	-16 * SIZE(AO), %ymm0

  1445. #endif

  1446. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4

  1447. #if defined BROADCASTKERNEL

  1448.         vbroadcastsd    -15 * SIZE(AO), %ymm0

  1449. #else

  1450. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1451. #endif

  1452. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5

  1453. 	addq		$ 4*SIZE, BO

  1454. #if defined BROADCASTKERNEL

  1455.         vbroadcastsd    -14 * SIZE(AO), %ymm0

  1456. #else

  1457. 	vpermpd		$ 0x1b, %ymm0  , %ymm0

  1458. #endif

  1459. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6

  1460. 	addq		$ 4*SIZE, AO

  1461. #if defined BROADCASTKERNEL

  1462.         vbroadcastsd    -17 * SIZE(AO), %ymm0

  1463. #else

  1464. 	vpermilpd	$ 0x05, %ymm0  , %ymm0

  1465. #endif

  1466. 	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7

  1467. 

  1468. .endm

  1469. 

  1470. .macro SAVE4x4

  1471. 

  1472. 	vbroadcastsd	ALPHA, %ymm0

  1473. 

  1474. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  1475. 	vmulpd	%ymm0 , %ymm7 , %ymm7

  1476. 	vmulpd	%ymm0 , %ymm5 , %ymm5

  1477. 	vmulpd	%ymm0 , %ymm6 , %ymm6

  1478. 

  1479. #if defined BROADCASTKERNEL

  1480.         vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0

  1481.         vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1

  1482.         vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2

  1483.         vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3

  1484.         vunpcklpd %ymm1, %ymm0, %ymm4

  1485.         vunpckhpd %ymm1, %ymm0, %ymm5

  1486.         vunpcklpd %ymm3, %ymm2, %ymm6

  1487.         vunpckhpd %ymm3, %ymm2, %ymm7

  1488. #else

  1489. 	vpermilpd $ 0x05 , %ymm5, %ymm5

  1490. 	vpermilpd $ 0x05 , %ymm7, %ymm7

  1491. 

  1492. 	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0

  1493. 	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1

  1494. 	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2

  1495. 	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3

  1496. 

  1497. 	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2

  1498. 	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3

  1499. 

  1500. 	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4

  1501. 	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5

  1502. 	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6

  1503. 	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7

  1504. #endif

  1505. 

  1506.         leaq    (CO1, LDC, 2), %rax     

  1507. 	

  1508. 

  1509. #if !defined(TRMMKERNEL)

  1510. 

  1511. 	vaddpd 	                (CO1), %ymm4, %ymm4

  1512. 	vaddpd 	           (CO1, LDC), %ymm5, %ymm5

  1513. 	vaddpd 	               (%rax), %ymm6, %ymm6

  1514. 	vaddpd 	          (%rax, LDC), %ymm7, %ymm7

  1515. 

  1516. #endif

  1517. 

  1518. 	vmovups	%ymm4 ,  	(CO1)

  1519. 	vmovups	%ymm5 ,  	(CO1, LDC)

  1520. 	vmovups	%ymm6 ,  	(%rax)

  1521. 	vmovups	%ymm7 ,  	(%rax, LDC)

  1522. 

  1523. 	addq	$ 4*SIZE, CO1

  1524. .endm

  1525. 

  1526. /******************************************************************************************/

  1527. /******************************************************************************************/

  1528. 

  1529. .macro INIT2x4

  1530. 

  1531. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1532. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1533. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1534. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  1535. 

  1536. .endm

  1537. 

  1538. 

  1539. .macro KERNEL2x4_SUB

  1540. 	vmovddup	-12 * SIZE(BO), %xmm1

  1541. 	vmovups 	-16 * SIZE(AO), %xmm0

  1542. 	vmovddup	-11 * SIZE(BO), %xmm2

  1543. 	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4

  1544. 	vmovddup	-10 * SIZE(BO), %xmm3

  1545. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5

  1546. 	vmovddup	 -9 * SIZE(BO), %xmm8

  1547. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  1548. 	addq		$ 4*SIZE, BO

  1549. 	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7

  1550. 	addq		$ 2*SIZE, AO

  1551. 

  1552. .endm

  1553. 

  1554. 

  1555. .macro SAVE2x4

  1556. 

  1557. 	vmovddup	ALPHA, %xmm0

  1558. 

  1559. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1560. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  1561. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  1562. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  1563. 

  1564.         leaq    (CO1, LDC, 2), %rax     

  1565. 

  1566. #if !defined(TRMMKERNEL)

  1567. 

  1568. 	vaddpd 	                (CO1), %xmm4, %xmm4

  1569. 	vaddpd 	           (CO1, LDC), %xmm5, %xmm5

  1570. 	vaddpd 	               (%rax), %xmm6, %xmm6

  1571. 	vaddpd 	          (%rax, LDC), %xmm7, %xmm7

  1572. 

  1573. #endif

  1574. 

  1575. 	vmovups	%xmm4 ,  	(CO1)

  1576. 	vmovups	%xmm5 ,  	(CO1, LDC)

  1577. 	vmovups	%xmm6 ,  	(%rax)

  1578. 	vmovups	%xmm7 ,  	(%rax, LDC)

  1579. 

  1580. 	addq	$ 2*SIZE, CO1

  1581. .endm

  1582. 

  1583. /******************************************************************************************/

  1584. /******************************************************************************************/

  1585. 

  1586. .macro INIT1x4

  1587. 

  1588. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1589. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1590. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1591. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  1592. 

  1593. .endm

  1594. 

  1595. 

  1596. .macro KERNEL1x4_SUB

  1597. 	vmovsd	-12 * SIZE(BO), %xmm1

  1598. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1599. 	vmovsd	-11 * SIZE(BO), %xmm2

  1600. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1601. 	vmovsd	-10 * SIZE(BO), %xmm3

  1602. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  1603. 	vmovsd	 -9 * SIZE(BO), %xmm8

  1604. 	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6

  1605. 	addq		$ 4*SIZE, BO

  1606. 	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7

  1607. 	addq		$ 1*SIZE, AO

  1608. 

  1609. .endm

  1610. 

  1611. 

  1612. .macro SAVE1x4

  1613. 

  1614. 	vmovsd	ALPHA, %xmm0

  1615. 

  1616. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1617. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  1618. 	vmulsd	%xmm0 , %xmm6 , %xmm6

  1619. 	vmulsd	%xmm0 , %xmm7 , %xmm7

  1620. 

  1621.         leaq    (CO1, LDC, 2), %rax     

  1622. 

  1623. #if !defined(TRMMKERNEL)

  1624. 

  1625. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1626. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  1627. 	vaddsd 	               (%rax), %xmm6, %xmm6

  1628. 	vaddsd 	          (%rax, LDC), %xmm7, %xmm7

  1629. 

  1630. #endif

  1631. 

  1632. 	vmovsd	%xmm4 ,  	(CO1)

  1633. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  1634. 	vmovsd	%xmm6 ,  	(%rax)

  1635. 	vmovsd	%xmm7 ,  	(%rax, LDC)

  1636. 

  1637. 	addq	$ 1*SIZE, CO1

  1638. .endm

  1639. 

  1640. 

  1641. /******************************************************************************************/

  1642. /******************************************************************************************/

  1643. 

  1644. .macro INIT4x2

  1645. 

  1646. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1647. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1648. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1649. 	vxorpd		%xmm7 , %xmm7 , %xmm7

  1650. 

  1651. .endm

  1652. 

  1653. 

  1654. .macro KERNEL4x2_SUB

  1655. 	vmovddup	-12 * SIZE(BO), %xmm2

  1656. 	vmovups 	-16 * SIZE(AO), %xmm0

  1657. 	vmovups 	-14 * SIZE(AO), %xmm1

  1658. 	vmovddup	-11 * SIZE(BO), %xmm3

  1659. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  1660. 	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5

  1661. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  1662. 	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7

  1663. 	addq		$ 2*SIZE, BO

  1664. 	addq		$ 4*SIZE, AO

  1665. 

  1666. .endm

  1667. 

  1668. 

  1669. .macro SAVE4x2

  1670. 

  1671. 	vmovddup	ALPHA, %xmm0

  1672. 

  1673. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1674. 	vmulpd	%xmm0 , %xmm5 , %xmm5

  1675. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  1676. 	vmulpd	%xmm0 , %xmm7 , %xmm7

  1677. 

  1678. 

  1679. #if !defined(TRMMKERNEL)

  1680. 

  1681. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  1682. 	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5

  1683. 	vaddpd 	                (CO1, LDC), %xmm6, %xmm6

  1684. 	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7

  1685. 

  1686. #endif

  1687. 

  1688. 	vmovups	%xmm4 ,  	(CO1)

  1689. 	vmovups	%xmm5 , 2 * SIZE(CO1)

  1690. 	vmovups	%xmm6 ,  	(CO1, LDC)

  1691. 	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)

  1692. 

  1693. 	addq	$ 4*SIZE, CO1

  1694. .endm

  1695. 

  1696. 

  1697. /******************************************************************************************/

  1698. /******************************************************************************************/

  1699. 

  1700. .macro INIT2x2

  1701. 

  1702. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1703. 	vxorpd		%xmm6 , %xmm6 , %xmm6

  1704. 

  1705. .endm

  1706. 

  1707. 

  1708. .macro KERNEL2x2_SUB

  1709. 	vmovddup	-12 * SIZE(BO), %xmm2

  1710. 	vmovups 	-16 * SIZE(AO), %xmm0

  1711. 	vmovddup	-11 * SIZE(BO), %xmm3

  1712. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  1713. 	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6

  1714. 	addq		$ 2*SIZE, BO

  1715. 	addq		$ 2*SIZE, AO

  1716. 

  1717. .endm

  1718. 

  1719. 

  1720. .macro SAVE2x2

  1721. 

  1722. 	vmovddup	ALPHA, %xmm0

  1723. 

  1724. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1725. 	vmulpd	%xmm0 , %xmm6 , %xmm6

  1726. 

  1727. 

  1728. #if !defined(TRMMKERNEL)

  1729. 

  1730. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  1731. 	vaddpd 	                (CO1, LDC), %xmm6, %xmm6

  1732. 

  1733. #endif

  1734. 

  1735. 	vmovups	%xmm4 ,  	(CO1)

  1736. 	vmovups	%xmm6 ,  	(CO1, LDC)

  1737. 

  1738. 	addq	$ 2*SIZE, CO1

  1739. .endm

  1740. 

  1741. /******************************************************************************************/

  1742. /******************************************************************************************/

  1743. 

  1744. .macro INIT1x2

  1745. 

  1746. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1747. 	vxorpd		%xmm5 , %xmm5 , %xmm5

  1748. 

  1749. .endm

  1750. 

  1751. 

  1752. .macro KERNEL1x2_SUB

  1753. 	vmovsd	-12 * SIZE(BO), %xmm1

  1754. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1755. 	vmovsd	-11 * SIZE(BO), %xmm2

  1756. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1757. 	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5

  1758. 	addq		$ 2*SIZE, BO

  1759. 	addq		$ 1*SIZE, AO

  1760. 

  1761. .endm

  1762. 

  1763. 

  1764. .macro SAVE1x2

  1765. 

  1766. 	vmovsd	ALPHA, %xmm0

  1767. 

  1768. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1769. 	vmulsd	%xmm0 , %xmm5 , %xmm5

  1770. 

  1771. 

  1772. #if !defined(TRMMKERNEL)

  1773. 

  1774. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1775. 	vaddsd 	           (CO1, LDC), %xmm5, %xmm5

  1776. 

  1777. #endif

  1778. 

  1779. 	vmovsd	%xmm4 ,  	(CO1)

  1780. 	vmovsd	%xmm5 ,  	(CO1, LDC)

  1781. 

  1782. 	addq	$ 1*SIZE, CO1

  1783. .endm

  1784. 

  1785. 

  1786. /******************************************************************************************/

  1787. /******************************************************************************************/

  1788. 

  1789. .macro INIT4x1

  1790. 

  1791. 	vxorpd		%ymm4 , %ymm4 , %ymm4

  1792. 	vxorpd		%ymm5 , %ymm5 , %ymm5

  1793. 	vxorpd		%ymm6 , %ymm6 , %ymm6

  1794. 	vxorpd		%ymm7 , %ymm7 , %ymm7

  1795. 

  1796. .endm

  1797. 

  1798. 

  1799. .macro KERNEL4x1

  1800. 

  1801. 	vbroadcastsd	-12 * SIZE(BO), %ymm0

  1802. 	vbroadcastsd	-11 * SIZE(BO), %ymm1

  1803. 	vbroadcastsd	-10 * SIZE(BO), %ymm2

  1804. 	vbroadcastsd	-9  * SIZE(BO), %ymm3

  1805. 

  1806. 	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4

  1807. 	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5

  1808. 

  1809. 	vbroadcastsd	-8  * SIZE(BO), %ymm0

  1810. 	vbroadcastsd	-7  * SIZE(BO), %ymm1

  1811. 

  1812. 	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6

  1813. 	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7

  1814. 

  1815. 	vbroadcastsd	-6  * SIZE(BO), %ymm2

  1816. 	vbroadcastsd	-5  * SIZE(BO), %ymm3

  1817. 

  1818. 	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4

  1819. 	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5

  1820. 	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6

  1821. 	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7

  1822. 

  1823. 	addq		$ 8 *SIZE, BO

  1824. 	addq		$ 32*SIZE, AO

  1825. 

  1826. .endm

  1827. 

  1828. 

  1829. .macro KERNEL4x1_SUB

  1830. 	vbroadcastsd	-12 * SIZE(BO), %ymm2

  1831. 	vmovups 	-16 * SIZE(AO), %ymm0

  1832. 	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4

  1833. 	addq		$ 1*SIZE, BO

  1834. 	addq		$ 4*SIZE, AO

  1835. 

  1836. .endm

  1837. 

  1838. 

  1839. .macro SAVE4x1

  1840. 

  1841. 	vbroadcastsd	ALPHA, %ymm0

  1842. 

  1843. 	vaddpd	%ymm4,%ymm5, %ymm4 

  1844. 	vaddpd	%ymm6,%ymm7, %ymm6 

  1845. 	vaddpd	%ymm4,%ymm6, %ymm4 

  1846. 

  1847. 	vmulpd	%ymm0 , %ymm4 , %ymm4

  1848. 

  1849. 

  1850. #if !defined(TRMMKERNEL)

  1851. 

  1852. 	vaddpd 	                (CO1)     , %ymm4, %ymm4

  1853. 

  1854. #endif

  1855. 

  1856. 	vmovups	%ymm4 ,  	(CO1)

  1857. 

  1858. 	addq	$ 4*SIZE, CO1

  1859. .endm

  1860. 

  1861. 

  1862. /******************************************************************************************/

  1863. /******************************************************************************************/

  1864. 

  1865. .macro INIT2x1

  1866. 

  1867. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1868. 

  1869. .endm

  1870. 

  1871. 

  1872. .macro KERNEL2x1_SUB

  1873. 	vmovddup	-12 * SIZE(BO), %xmm2

  1874. 	vmovups 	-16 * SIZE(AO), %xmm0

  1875. 	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4

  1876. 	addq		$ 1*SIZE, BO

  1877. 	addq		$ 2*SIZE, AO

  1878. 

  1879. .endm

  1880. 

  1881. 

  1882. .macro SAVE2x1

  1883. 

  1884. 	vmovddup	ALPHA, %xmm0

  1885. 

  1886. 	vmulpd	%xmm0 , %xmm4 , %xmm4

  1887. 

  1888. 

  1889. #if !defined(TRMMKERNEL)

  1890. 

  1891. 	vaddpd 	                (CO1)     , %xmm4, %xmm4

  1892. 

  1893. #endif

  1894. 

  1895. 	vmovups	%xmm4 ,  	(CO1)

  1896. 

  1897. 	addq	$ 2*SIZE, CO1

  1898. .endm

  1899. 

  1900. 

  1901. /******************************************************************************************/

  1902. /******************************************************************************************/

  1903. 

  1904. .macro INIT1x1

  1905. 

  1906. 	vxorpd		%xmm4 , %xmm4 , %xmm4

  1907. 

  1908. .endm

  1909. 

  1910. 

  1911. .macro KERNEL1x1_SUB

  1912. 	vmovsd	-12 * SIZE(BO), %xmm1

  1913. 	vmovsd 	-16 * SIZE(AO), %xmm0

  1914. 	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4

  1915. 	addq		$ 1*SIZE, BO

  1916. 	addq		$ 1*SIZE, AO

  1917. 

  1918. .endm

  1919. 

  1920. 

  1921. .macro SAVE1x1

  1922. 

  1923. 	vmovsd	ALPHA, %xmm0

  1924. 

  1925. 	vmulsd	%xmm0 , %xmm4 , %xmm4

  1926. 

  1927. 

  1928. #if !defined(TRMMKERNEL)

  1929. 

  1930. 	vaddsd 	                (CO1), %xmm4, %xmm4

  1931. 

  1932. #endif

  1933. 

  1934. 	vmovsd	%xmm4 ,  	(CO1)

  1935. 

  1936. 	addq	$ 1*SIZE, CO1

  1937. .endm

  1938. 

  1939. 

  1940. .macro PREFETCHT0_C

  1941.         prefetcht0 (CO1)

  1942.         prefetcht0 24(CO1)

  1943.         prefetcht0 (CO1,LDC,4)

  1944.         prefetcht0 24(CO1,LDC,4)

  1945.         prefetcht0 (CO1,LDC,8)

  1946.         prefetcht0 24(CO1,LDC,8)

  1947. .endm

  1948. /*******************************************************************************************/

  1949. 

  1950. #if !defined(TRMMKERNEL)

  1951. 

  1952. 

  1953. 	PROLOGUE

  1954. 	PROFCODE

  1955. 	

  1956. 	subq	$STACKSIZE, %rsp

  1957. 	movq	%rbx,   (%rsp)

  1958. 	movq	%rbp,  8(%rsp)

  1959. 	movq	%r12, 16(%rsp)

  1960. 	movq	%r13, 24(%rsp)

  1961. 	movq	%r14, 32(%rsp)

  1962. 	movq	%r15, 40(%rsp)

  1963. 

  1964. 	vzeroupper

  1965. 

  1966. #ifdef WINDOWS_ABI

  1967. 	movq	%rdi,    48(%rsp)

  1968. 	movq	%rsi,    56(%rsp)

  1969. 	vmovups	%xmm6,   64(%rsp)

  1970. 	vmovups	%xmm7,   80(%rsp)

  1971. 	vmovups	%xmm8,   96(%rsp)

  1972. 	vmovups	%xmm9,  112(%rsp)

  1973. 	vmovups	%xmm10, 128(%rsp)

  1974. 	vmovups	%xmm11, 144(%rsp)

  1975. 	vmovups	%xmm12, 160(%rsp)

  1976. 	vmovups	%xmm13, 176(%rsp)

  1977. 	vmovups	%xmm14, 192(%rsp)

  1978. 	vmovups	%xmm15, 208(%rsp)

  1979. 

  1980. 	movq	ARG1,      OLD_M

  1981. 	movq	ARG2,      OLD_N

  1982. 	movq	ARG3,      OLD_K

  1983. 	movq	OLD_A,     A

  1984. 	movq	OLD_B,     B

  1985. 	movq	OLD_C,     C

  1986. 	movq	OLD_LDC,   LDC

  1987. 

  1988. 	vmovups	%xmm3, %xmm0

  1989. 

  1990. #else

  1991. 	movq	STACKSIZE +  8(%rsp), LDC

  1992. 

  1993. #endif

  1994. 

  1995. 	movq    %rsp, SP      # save old stack

  1996.         subq    $128 + L_BUFFER_SIZE, %rsp

  1997.         andq    $-4096, %rsp    # align stack

  1998. 

  1999.         STACK_TOUCH

  2000. 

  2001. 	cmpq	$ 0, OLD_M

  2002. 	je	.L999

  2003. 

  2004. 	cmpq	$ 0, OLD_N

  2005. 	je	.L999

  2006. 

  2007. 	cmpq	$ 0, OLD_K

  2008. 	je	.L999

  2009. 

  2010. 	movq	OLD_M, M

  2011. 	movq	OLD_N, N

  2012. 	movq	OLD_K, K

  2013. 

  2014. 	vmovsd	 %xmm0, ALPHA

  2015. 

  2016. 	salq	$BASE_SHIFT, LDC

  2017. 

  2018. 	movq    N, %rax

  2019.         xorq    %rdx, %rdx

  2020.         movq    $24,  %rdi

  2021.         divq    %rdi                     //    N / 24

  2022.         movq    %rax, Ndiv12             //    N / 24

  2023.         movq    %rdx, Nmod12             //    N % 24

  2024. 

  2025. 

  2026. 	movq	Ndiv12,  J

  2027. 	cmpq	$ 0, J

  2028. 	je	.L8_0

  2029. 	ALIGN_4

  2030. 

  2031. .L12_01:

  2032.         // copy to sub buffer

  2033.         movq    K, %rax

  2034.         salq    $3,%rax                 // K * 8 ; read 8 values from BO1

  2035.         movq    B, BO1

  2036.         leaq    (B,%rax, SIZE), BO2     // next offset to BO2

  2037. 	movq	BO2 , B			

  2038. 

  2039.         leaq    BUFFER1, BO             // first buffer to BO

  2040.         movq    K, %rax

  2041. 

  2042.         ALIGN_4

  2043. 

  2044. .L12_02b:

  2045. 

  2046. 	vmovups	0 * SIZE(BO1), %ymm1

  2047. 	vmovups	4 * SIZE(BO1), %ymm2

  2048. 	vmovups	0 * SIZE(BO2), %ymm3

  2049. 	vmovups	%ymm1, 0 * SIZE(BO)

  2050. 	vmovups	%ymm2, 4 * SIZE(BO)

  2051. 	vmovups	%ymm3, 8 * SIZE(BO)

  2052. 	addq	$ 8*SIZE,BO1

  2053. 	addq	$ 8*SIZE,BO2

  2054. 	addq	$ 12*SIZE,BO

  2055. 	decq	%rax

  2056. 	jnz	.L12_02b

  2057. 

  2058. .L12_03c:

  2059. 

  2060. 

  2061. .L12_10:

  2062. 	movq	C, CO1

  2063. 	leaq	(C, LDC, 8), C		 

  2064. 	leaq	(C, LDC, 4), C		// c += 12 * ldc

  2065. 	

  2066. 	movq	A, AO		 	// aoffset = a

  2067. 	addq	$16 * SIZE, AO

  2068. 

  2069. 	movq	M,  I

  2070. 	sarq	$2, I			// i = m / 4

  2071. 	je	.L12_20

  2072. 

  2073. 	ALIGN_4

  2074. 

  2075. .L12_11:

  2076.         leaq    BUFFER1, BO             // first buffer to BO

  2077.         addq    $12 * SIZE, BO

  2078. 

  2079.         movq    K, %rax

  2080. 

  2081. 	sarq $3, %rax			//  K / 8

  2082. 	cmpq $2, %rax

  2083. 

  2084. 	jl	.L12_13

  2085. 

  2086. 

  2087. 	KERNEL4x12_I

  2088. 	KERNEL4x12_M2

  2089. 	KERNEL4x12_M1

  2090. 	KERNEL4x12_M2

  2091. 

  2092. 	KERNEL4x12_M1

  2093. 	KERNEL4x12_M2

  2094. 	KERNEL4x12_M1

  2095. 	KERNEL4x12_M2

  2096. 

  2097. 	subq $2, %rax

  2098. 	je	.L12_12a

  2099. 

  2100. 	ALIGN_5

  2101. .L12_12:

  2102. 

  2103. 	KERNEL4x12_M1

  2104. 	KERNEL4x12_M2

  2105. 	KERNEL4x12_M1

  2106. 	KERNEL4x12_M2

  2107. 

  2108. 	KERNEL4x12_M1

  2109. 	KERNEL4x12_M2

  2110. 	KERNEL4x12_M1

  2111. 	KERNEL4x12_M2

  2112. 

  2113. 	dec	%rax

  2114. 	jne	.L12_12

  2115. 	

  2116. .L12_12a:

  2117.         prefetcht0 ALPHA

  2118.         PREFETCHT0_C

  2119.         addq  LDC,CO1

  2120. 	KERNEL4x12_M1

  2121.         PREFETCHT0_C

  2122.         leaq  (CO1,LDC,2),CO1

  2123. 	KERNEL4x12_M2

  2124.         PREFETCHT0_C

  2125.         subq  LDC,CO1

  2126. 	KERNEL4x12_M1

  2127.         PREFETCHT0_C

  2128.         subq  LDC,CO1

  2129.         subq  LDC,CO1

  2130. 	KERNEL4x12_M2

  2131. 

  2132. 	KERNEL4x12_M1

  2133. 	KERNEL4x12_M2

  2134. 	KERNEL4x12_M1

  2135. 	KERNEL4x12_E

  2136. 

  2137. 	jmp .L12_16

  2138. 

  2139. 

  2140. .L12_13:

  2141. 

  2142. 	test $1, %rax

  2143. 	jz .L12_14

  2144. 

  2145. 	KERNEL4x12_I

  2146. 	KERNEL4x12_M2

  2147. 	KERNEL4x12_M1

  2148. 	KERNEL4x12_M2

  2149. 

  2150. 	KERNEL4x12_M1

  2151. 	KERNEL4x12_M2

  2152. 	KERNEL4x12_M1

  2153. 	KERNEL4x12_E

  2154. 

  2155. 	jmp .L12_16

  2156. 

  2157. 

  2158. .L12_14:

  2159. 

  2160. 	INIT4x12

  2161. 

  2162. 

  2163. .L12_16:

  2164.         movq    K, %rax

  2165. 

  2166. 	andq	$7, %rax		# if (k & 1)

  2167. 	je .L12_19

  2168. 

  2169. 	ALIGN_4

  2170. 

  2171. .L12_17:

  2172. 

  2173. 	KERNEL4x12_SUB

  2174. 

  2175. 	dec	%rax

  2176. 	jne	.L12_17

  2177. 	ALIGN_4

  2178. 

  2179. 

  2180. .L12_19:

  2181. 

  2182. 	SAVE4x12

  2183. 

  2184.         /* here for the prefetch of next b source block */

  2185. 	/* the increment should be proportional to GEMM_Q/GEMM_P */

  2186. 

  2187.         salq    $3, K

  2188. #ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */

  2189.         prefetcht2 32(B)

  2190.         prefetcht2 32(B, K, 8)

  2191.         addq    $64, B /* increment */

  2192. #else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */

  2193.         prefetcht2 32(B)

  2194.         prefetcht2 32(B, K, 8)

  2195.         prefetcht2 96(B)

  2196.         prefetcht2 96(B, K, 8)

  2197.         addq    $128, B /* increment */

  2198. #endif

  2199.         sarq    $3, K

  2200. 

  2201. 	decq	I			# i --

  2202. 	jne	.L12_11

  2203. 	ALIGN_4	

  2204. 

  2205. /**************************************************************************

  2206. * Rest of M 

  2207. ***************************************************************************/

  2208. 

  2209.         /* recover the original value of pointer B after prefetch */

  2210.         movq    M, I

  2211.         sarq    $2, I

  2212. #ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */

  2213.         salq    $6, I

  2214. #else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */

  2215.         salq    $7, I

  2216. #endif

  2217.         subq    I, B

  2218. 

  2219. .L12_20:

  2220. 	// Test rest of M

  2221. 

  2222. 	testq	$3, M

  2223. 	jz	.L12_100			// to next 16 lines of N

  2224. 

  2225. 

  2226. .L12_30:

  2227. 	testq	$2, M		

  2228. 	jz	.L12_40

  2229. 

  2230. 	ALIGN_4

  2231. 

  2232. .L12_31:

  2233.         leaq    BUFFER1, BO             // first buffer to BO

  2234.         addq    $12 * SIZE, BO

  2235. 

  2236. 	INIT2x12

  2237. 

  2238.         movq    K, %rax

  2239. 

  2240. 	sarq	$3, %rax

  2241. 	je	.L12_36

  2242. 	ALIGN_4

  2243. 

  2244. .L12_32:

  2245. 

  2246. 	KERNEL2x12_SUB

  2247. 	KERNEL2x12_SUB

  2248. 	KERNEL2x12_SUB

  2249. 	KERNEL2x12_SUB

  2250. 

  2251. 	KERNEL2x12_SUB

  2252. 	KERNEL2x12_SUB

  2253. 	KERNEL2x12_SUB

  2254. 	KERNEL2x12_SUB

  2255. 

  2256. 	dec %rax

  2257. 	jne	.L12_32

  2258. 	ALIGN_4

  2259. 

  2260. .L12_36:

  2261.         movq    K, %rax

  2262. 

  2263. 	andq	$7, %rax		# if (k & 1)

  2264. 	je .L12_39

  2265. 

  2266. 	ALIGN_4

  2267. 

  2268. .L12_37:

  2269. 

  2270. 	KERNEL2x12_SUB

  2271. 

  2272. 	dec %rax

  2273. 	jne	.L12_37

  2274. 	ALIGN_4

  2275. 

  2276. 

  2277. .L12_39:

  2278. 

  2279. 	SAVE2x12

  2280. 

  2281. 	ALIGN_4

  2282. 

  2283. .L12_40:

  2284. 	testq	$1, M		

  2285. 	jz	.L12_100		// to next 3 lines of N

  2286. 

  2287. 	ALIGN_4

  2288. 

  2289. .L12_41:

  2290.         leaq    BUFFER1, BO             // first buffer to BO

  2291.         addq    $12 * SIZE, BO

  2292. 

  2293. 	INIT1x12

  2294. 

  2295.         movq    K, %rax

  2296. 

  2297. 	sarq	$3,%rax

  2298. 	je	.L12_46

  2299. 

  2300. 	ALIGN_4

  2301. 

  2302. .L12_42:

  2303. 

  2304. 	KERNEL1x12_SUB

  2305. 	KERNEL1x12_SUB

  2306. 	KERNEL1x12_SUB

  2307. 	KERNEL1x12_SUB

  2308. 

  2309. 	KERNEL1x12_SUB

  2310. 	KERNEL1x12_SUB

  2311. 	KERNEL1x12_SUB

  2312. 	KERNEL1x12_SUB

  2313. 

  2314. 

  2315. 	dec %rax

  2316. 	jne	.L12_42

  2317. 	ALIGN_4

  2318. 

  2319. .L12_46:

  2320.         movq    K, %rax

  2321. 

  2322. 	andq	$7, %rax		# if (k & 1)

  2323. 	je .L12_49

  2324. 

  2325. 	ALIGN_4

  2326. 

  2327. .L12_47:

  2328. 

  2329. 	KERNEL1x12_SUB

  2330. 

  2331. 	dec	%rax

  2332. 	jne	.L12_47

  2333. 	ALIGN_4

  2334. 

  2335. 

  2336. .L12_49:

  2337. 

  2338. 	SAVE1x12

  2339. 

  2340. 	ALIGN_4

  2341. 	

  2342. .L12_100:

  2343. 

  2344. 

  2345. 

  2346. /**************************************************************************************************/

  2347. 

  2348. .L13_01:

  2349.         // copy to sub buffer

  2350.         movq    K, %rax

  2351.         salq    $3,%rax                 // K * 8 ; read 8 values

  2352.         movq    B, BO2

  2353.         leaq    (B,%rax, SIZE), BO3     // next offset to BO2

  2354.         leaq    (BO3,%rax, SIZE), B     // next offset to B

  2355. 

  2356. 

  2357.         leaq    BUFFER1, BO             // first buffer to BO

  2358.         movq    K, %rax

  2359. 

  2360.         ALIGN_4

  2361. 

  2362. 

  2363. .L13_02b:

  2364. 

  2365. 	vmovups	4 * SIZE(BO2), %ymm1

  2366. 	vmovups	0 * SIZE(BO3), %ymm2

  2367. 	vmovups	4 * SIZE(BO3), %ymm3

  2368. 	vmovups	%ymm1, 0 * SIZE(BO)

  2369. 	vmovups	%ymm2, 4 * SIZE(BO)

  2370. 	vmovups	%ymm3, 8 * SIZE(BO)

  2371. 	addq	$ 8*SIZE,BO2

  2372. 	addq	$ 8*SIZE,BO3

  2373. 	addq	$ 12*SIZE,BO

  2374. 	decq	%rax

  2375. 	jnz	.L13_02b

  2376. 

  2377. 

  2378. 

  2379. .L13_10:

  2380. 	movq	C, CO1

  2381. 	leaq	(C, LDC, 8), C		 

  2382. 	leaq	(C, LDC, 4), C		// c += 12 * ldc

  2383. 

  2384. 	

  2385. 	movq	A, AO		 	// aoffset = a

  2386. 	addq	$16 * SIZE, AO

  2387. 

  2388. 	movq	M,  I

  2389. 	sarq	$2, I			// i = m / 4

  2390. 	je	.L13_20

  2391. 

  2392. 	ALIGN_4

  2393. 

  2394. .L13_11:

  2395.         leaq    BUFFER1, BO             // first buffer to BO

  2396.         addq    $12 * SIZE, BO

  2397. 

  2398.         movq    K, %rax

  2399. 

  2400. 	sarq $3, %rax			//  K / 8

  2401. 	cmpq $2, %rax

  2402. 

  2403. 	jl	.L13_13

  2404. 

  2405. 

  2406. 	KERNEL4x12_I

  2407. 	KERNEL4x12_M2

  2408. 	KERNEL4x12_M1

  2409. 	KERNEL4x12_M2

  2410. 

  2411. 	KERNEL4x12_M1

  2412. 	KERNEL4x12_M2

  2413. 	KERNEL4x12_M1

  2414. 	KERNEL4x12_M2

  2415. 

  2416. 	subq $2, %rax

  2417. 	je	.L13_12a

  2418. 

  2419. 	ALIGN_5

  2420. .L13_12:

  2421. 

  2422. 	KERNEL4x12_M1

  2423. 	KERNEL4x12_M2

  2424. 	KERNEL4x12_M1

  2425. 	KERNEL4x12_M2

  2426. 

  2427. 	KERNEL4x12_M1

  2428. 	KERNEL4x12_M2

  2429. 	KERNEL4x12_M1

  2430. 	KERNEL4x12_M2

  2431. 

  2432. 	dec	%rax

  2433. 	jne	.L13_12

  2434. 

  2435. .L13_12a:

  2436.         prefetcht0 ALPHA

  2437.         PREFETCHT0_C

  2438.         addq  LDC,CO1

  2439. 	KERNEL4x12_M1

  2440.         PREFETCHT0_C

  2441.         leaq  (CO1,LDC,2),CO1

  2442. 	KERNEL4x12_M2

  2443.         PREFETCHT0_C

  2444.         subq  LDC,CO1

  2445. 	KERNEL4x12_M1

  2446.         PREFETCHT0_C

  2447.         subq  LDC,CO1

  2448.         subq  LDC,CO1

  2449. 	KERNEL4x12_M2

  2450. 

  2451. 	KERNEL4x12_M1

  2452. 	KERNEL4x12_M2

  2453. 	KERNEL4x12_M1

  2454. 	KERNEL4x12_E

  2455. 

  2456. 	jmp .L13_16

  2457. 

  2458. .L13_13:

  2459. 

  2460. 	test $1, %rax

  2461. 	jz .L13_14

  2462. 

  2463. 	KERNEL4x12_I

  2464. 	KERNEL4x12_M2

  2465. 	KERNEL4x12_M1

  2466. 	KERNEL4x12_M2

  2467. 

  2468. 	KERNEL4x12_M1

  2469. 	KERNEL4x12_M2

  2470. 	KERNEL4x12_M1

  2471. 	KERNEL4x12_E

  2472. 

  2473. 	jmp .L13_16

  2474. 

  2475. 

  2476. .L13_14:

  2477. 

  2478. 	INIT4x12

  2479. 

  2480. 

  2481. .L13_16:

  2482.         movq    K, %rax

  2483. 

  2484. 	andq	$7, %rax		# if (k & 1)

  2485. 	je .L13_19

  2486. 

  2487. 	ALIGN_4

  2488. 

  2489. .L13_17:

  2490. 

  2491. 	KERNEL4x12_SUB

  2492. 

  2493. 	dec	%rax

  2494. 	jne	.L13_17

  2495. 	ALIGN_4

  2496. 

  2497. 

  2498. .L13_19:

  2499. 

  2500. 	SAVE4x12

  2501. 

  2502.         /* here for the prefetch of next b source block */

  2503. 	/* the increment should be proportional to GEMM_Q/GEMM_P */

  2504. 

  2505.         salq    $3, K

  2506. #ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */

  2507.         prefetcht2 (B)

  2508.         prefetcht2 (B, K, 8)

  2509.         addq    $64, B /* increment */

  2510. #else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */

  2511.         prefetcht2 (B)

  2512.         prefetcht2 (B, K, 8)

  2513.         prefetcht2 64(B)

  2514.         prefetcht2 64(B, K, 8)

  2515.         addq    $128, B /* increment */

  2516. #endif

  2517.         sarq    $3, K

  2518. 

  2519. 	decq	I			# i --

  2520. 	jne	.L13_11

  2521. 	ALIGN_4	

  2522. 

  2523. /**************************************************************************

  2524. * Rest of M 

  2525. ***************************************************************************/

  2526.         /* recover the original value of pointer B */

  2527.         movq    M, I

  2528.         sarq    $2, I

  2529. #ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */

  2530.         salq    $6, I

  2531. #else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */

  2532.         salq    $7, I

  2533. #endif

  2534.         subq    I, B

  2535. 

  2536. .L13_20:

  2537. 	// Test rest of M

  2538. 

  2539. 	testq	$3, M

  2540. 	jz	.L13_100			// to next 16 lines of N

  2541. 

  2542. 

  2543. .L13_30:

  2544. 	testq	$2, M		

  2545. 	jz	.L13_40

  2546. 

  2547. 	ALIGN_4

  2548. 

  2549. .L13_31:

  2550.         leaq    BUFFER1, BO             // first buffer to BO

  2551.         addq    $12 * SIZE, BO

  2552. 

  2553. 	INIT2x12

  2554. 

  2555.         movq    K, %rax

  2556. 

  2557. 	sarq	$3, %rax

  2558. 	je	.L13_36

  2559. 	ALIGN_4

  2560. 

  2561. .L13_32:

  2562. 

  2563. 	KERNEL2x12_SUB

  2564. 	KERNEL2x12_SUB

  2565. 	KERNEL2x12_SUB

  2566. 	KERNEL2x12_SUB

  2567. 

  2568. 	KERNEL2x12_SUB

  2569. 	KERNEL2x12_SUB

  2570. 	KERNEL2x12_SUB

  2571. 	KERNEL2x12_SUB

  2572. 

  2573. 	dec %rax

  2574. 	jne	.L13_32

  2575. 	ALIGN_4

  2576. 

  2577. .L13_36:

  2578.         movq    K, %rax

  2579. 

  2580. 	andq	$7, %rax		# if (k & 1)

  2581. 	je .L13_39

  2582. 

  2583. 	ALIGN_4

  2584. 

  2585. .L13_37:

  2586. 

  2587. 	KERNEL2x12_SUB

  2588. 

  2589. 	dec %rax

  2590. 	jne	.L13_37

  2591. 	ALIGN_4

  2592. 

  2593. 

  2594. .L13_39:

  2595. 

  2596. 	SAVE2x12

  2597. 

  2598. 	ALIGN_4

  2599. 

  2600. .L13_40:

  2601. 	testq	$1, M		

  2602. 	jz	.L13_100		// to next 3 lines of N

  2603. 

  2604. 	ALIGN_4

  2605. 

  2606. .L13_41:

  2607.         leaq    BUFFER1, BO             // first buffer to BO

  2608.         addq    $12 * SIZE, BO

  2609. 

  2610. 	INIT1x12

  2611. 

  2612.         movq    K, %rax

  2613. 

  2614. 	sarq	$3,%rax

  2615. 	je	.L13_46

  2616. 

  2617. 	ALIGN_4

  2618. 

  2619. .L13_42:

  2620. 

  2621. 	KERNEL1x12_SUB

  2622. 	KERNEL1x12_SUB

  2623. 	KERNEL1x12_SUB

  2624. 	KERNEL1x12_SUB

  2625. 

  2626. 	KERNEL1x12_SUB

  2627. 	KERNEL1x12_SUB

  2628. 	KERNEL1x12_SUB

  2629. 	KERNEL1x12_SUB

  2630. 

  2631. 

  2632. 	dec %rax

  2633. 	jne	.L13_42

  2634. 	ALIGN_4

  2635. 

  2636. .L13_46:

  2637.         movq    K, %rax

  2638. 

  2639. 	andq	$7, %rax		# if (k & 1)

  2640. 	je .L13_49

  2641. 

  2642. 	ALIGN_4

  2643. 

  2644. .L13_47:

  2645. 

  2646. 	KERNEL1x12_SUB

  2647. 

  2648. 	dec	%rax

  2649. 	jne	.L13_47

  2650. 	ALIGN_4

  2651. 

  2652. 

  2653. .L13_49:

  2654. 

  2655. 	SAVE1x12

  2656. 

  2657. 	ALIGN_4

  2658. 	

  2659. .L13_100:

  2660. 

  2661. 	decq	J			// j --

  2662. 	jg	.L12_01

  2663. 

  2664. 

  2665. 

  2666. 

  2667. /**************************************************************************************************/

  2668. 

  2669. .L8_0:

  2670. 

  2671. 	cmpq	$ 0, Nmod12		// N % 12 == 0

  2672. 	je	.L999

  2673. 

  2674. 	movq	Nmod12, J		

  2675. 	sarq	$3, J			// j = j / 8

  2676. 	je	.L4_0

  2677. 

  2678. .L8_10:

  2679. 	movq	C, CO1

  2680. 	leaq	(C, LDC, 8), C		// c += 4 * ldc

  2681. 

  2682. 	

  2683. 	movq	A, AO		 	// aoffset = a

  2684. 	addq	$16 * SIZE, AO

  2685. 

  2686. 	movq	M,  I

  2687. 	sarq	$2, I			// i = m / 4

  2688. 	je	.L8_20

  2689. 

  2690. 	ALIGN_4

  2691. 

  2692. .L8_11:

  2693.         movq    B, BO        

  2694.         addq    $12 * SIZE, BO

  2695. 

  2696.         movq    K, %rax

  2697. 

  2698. 	sarq	$3, %rax			//  K / 8

  2699. 	cmpq    $2, %rax

  2700. 	jl	.L8_13

  2701. 

  2702. 

  2703. 	KERNEL4x8_I

  2704. 	KERNEL4x8_M2

  2705. 	KERNEL4x8_M1

  2706. 	KERNEL4x8_M2

  2707. 

  2708. 	KERNEL4x8_M1

  2709. 	KERNEL4x8_M2

  2710. 	KERNEL4x8_M1

  2711. 	KERNEL4x8_M2

  2712. 

  2713. 	subq $2, %rax

  2714. 	je	.L8_12a

  2715. 

  2716. 	ALIGN_5

  2717. 

  2718. .L8_12:

  2719. 

  2720. 	KERNEL4x8_M1

  2721. 	KERNEL4x8_M2

  2722. 	KERNEL4x8_M1

  2723. 	KERNEL4x8_M2

  2724. 

  2725. 	KERNEL4x8_M1

  2726. 	KERNEL4x8_M2

  2727. 	KERNEL4x8_M1

  2728. 	KERNEL4x8_M2

  2729. 

  2730. 	dec	%rax

  2731. 	jne	.L8_12

  2732. 

  2733. .L8_12a:

  2734. 

  2735. 	KERNEL4x8_M1

  2736. 	KERNEL4x8_M2

  2737. 	KERNEL4x8_M1

  2738. 	KERNEL4x8_M2

  2739. 

  2740. 	KERNEL4x8_M1

  2741. 	KERNEL4x8_M2

  2742. 	KERNEL4x8_M1

  2743. 	KERNEL4x8_E

  2744. 

  2745. 	jmp .L8_16

  2746. 

  2747. 

  2748. .L8_13:

  2749. 

  2750. 	test $1, %rax

  2751. 	jz .L8_14

  2752. 

  2753. 	KERNEL4x8_I

  2754. 	KERNEL4x8_M2

  2755. 	KERNEL4x8_M1

  2756. 	KERNEL4x8_M2

  2757. 

  2758. 	KERNEL4x8_M1

  2759. 	KERNEL4x8_M2

  2760. 	KERNEL4x8_M1

  2761. 	KERNEL4x8_E

  2762. 

  2763. 	jmp .L8_16

  2764. 

  2765. 

  2766. .L8_14:

  2767. 

  2768. 	INIT4x8

  2769. 

  2770. 

  2771. .L8_16:

  2772.         movq    K, %rax

  2773. 

  2774. 	andq	$7, %rax		# if (k & 1)

  2775. 	je .L8_19

  2776. 

  2777. 	ALIGN_4

  2778. 

  2779. .L8_17:

  2780. 

  2781. 	KERNEL4x8_SUB

  2782. 

  2783. 	dec	%rax

  2784. 	jne	.L8_17

  2785. 	ALIGN_4

  2786. 

  2787. 

  2788. .L8_19:

  2789. 

  2790. 	SAVE4x8

  2791. 

  2792. 	decq	I			# i --

  2793. 	jg	.L8_11

  2794. 	ALIGN_4	

  2795. 

  2796. /**************************************************************************

  2797. * Rest of M 

  2798. ***************************************************************************/

  2799. .L8_20:

  2800. 	// Test rest of M

  2801. 

  2802. 	testq	$3, M

  2803. 	jz	.L8_100			// to next 16 lines of N

  2804. 

  2805. 

  2806. .L8_30:

  2807. 	testq	$2, M		

  2808. 	jz	.L8_40

  2809. 

  2810. 	ALIGN_4

  2811. 

  2812. .L8_31:

  2813.         movq    B, BO             // first buffer to BO

  2814.         addq    $12 * SIZE, BO

  2815. 

  2816. 	INIT2x8

  2817. 

  2818.         movq    K, %rax

  2819. 

  2820. 	sarq	$3, %rax

  2821. 	je	.L8_36

  2822. 	ALIGN_4

  2823. 

  2824. .L8_32:

  2825. 

  2826. 	KERNEL2x8_SUB

  2827. 	KERNEL2x8_SUB

  2828. 	KERNEL2x8_SUB

  2829. 	KERNEL2x8_SUB

  2830. 

  2831. 	KERNEL2x8_SUB

  2832. 	KERNEL2x8_SUB

  2833. 	KERNEL2x8_SUB

  2834. 	KERNEL2x8_SUB

  2835. 

  2836. 	dec %rax

  2837. 	jne	.L8_32

  2838. 	ALIGN_4

  2839. 

  2840. .L8_36:

  2841.         movq    K, %rax

  2842. 

  2843. 	andq	$7, %rax		# if (k & 1)

  2844. 	je .L8_39

  2845. 

  2846. 	ALIGN_4

  2847. 

  2848. .L8_37:

  2849. 

  2850. 	KERNEL2x8_SUB

  2851. 

  2852. 	dec %rax

  2853. 	jne	.L8_37

  2854. 

  2855. 

  2856. .L8_39:

  2857. 

  2858. 	SAVE2x8

  2859. 

  2860. .L8_40:

  2861. 	testq	$1, M		

  2862. 	jz	.L8_100		// to next 3 lines of N

  2863. 

  2864. 	ALIGN_4

  2865. 

  2866. .L8_41:

  2867.         movq    B, BO             // first buffer to BO

  2868.         addq    $12 * SIZE, BO

  2869. 

  2870. 	INIT1x8

  2871. 

  2872.         movq    K, %rax

  2873. 

  2874. 	sarq	$3,%rax

  2875. 	je	.L8_46

  2876. 

  2877. 	ALIGN_4

  2878. 

  2879. .L8_42:

  2880. 

  2881. 	KERNEL1x8_SUB

  2882. 	KERNEL1x8_SUB

  2883. 	KERNEL1x8_SUB

  2884. 	KERNEL1x8_SUB

  2885. 

  2886. 	KERNEL1x8_SUB

  2887. 	KERNEL1x8_SUB

  2888. 	KERNEL1x8_SUB

  2889. 	KERNEL1x8_SUB

  2890. 

  2891. 	dec %rax

  2892. 	jne	.L8_42

  2893. 	ALIGN_4

  2894. 

  2895. .L8_46:

  2896.         movq    K, %rax

  2897. 

  2898. 	andq	$7, %rax		# if (k & 1)

  2899. 	je .L8_49

  2900. 

  2901. 	ALIGN_4

  2902. 

  2903. .L8_47:

  2904. 

  2905. 	KERNEL1x8_SUB

  2906. 

  2907. 	dec	%rax

  2908. 	jne	.L8_47

  2909. 	ALIGN_4

  2910. 

  2911. 

  2912. .L8_49:

  2913. 

  2914. 	SAVE1x8

  2915. 

  2916. 	ALIGN_4

  2917. 	

  2918. .L8_100:

  2919. 

  2920. 	movq	K, %rax

  2921. 	salq	$3, %rax		// * 8

  2922. 	leaq	(B , %rax, SIZE), B

  2923. 	decq	J			// j --

  2924. 	jg	.L8_10

  2925. 

  2926. 

  2927. 

  2928. /**************************************************************************************************/

  2929. 

  2930. .L4_0:

  2931. 

  2932. 	cmpq	$ 0, Nmod12		// N % 12 == 0

  2933. 	je	.L999

  2934. 

  2935. 	movq	Nmod12, J		

  2936. 	testq   $4, J			// j = j / 4

  2937. 	je	.L2_0

  2938. 

  2939. .L4_10:

  2940. 	movq	C, CO1

  2941. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  2942. 

  2943. 	

  2944. 	movq	A, AO		 	// aoffset = a

  2945. 	addq	$16 * SIZE, AO

  2946. 

  2947. 	movq	M,  I

  2948. 	sarq	$2, I			// i = m / 4

  2949. 	je	.L4_20

  2950. 

  2951. 	ALIGN_4

  2952. 

  2953. .L4_11:

  2954.         movq    B, BO        

  2955.         addq    $12 * SIZE, BO

  2956. 

  2957.         movq    K, %rax

  2958. 

  2959. 	sarq	$3, %rax			//  K / 8

  2960. 	cmpq    $2, %rax

  2961. 	jl	.L4_13

  2962. 

  2963. 

  2964. 	KERNEL4x4_I

  2965. 	KERNEL4x4_M2

  2966. 	KERNEL4x4_M1

  2967. 	KERNEL4x4_M2

  2968. 

  2969. 	KERNEL4x4_M1

  2970. 	KERNEL4x4_M2

  2971. 	KERNEL4x4_M1

  2972. 	KERNEL4x4_M2

  2973. 

  2974. 	subq $2, %rax

  2975. 	je	.L4_12a

  2976. 

  2977. 	ALIGN_5

  2978. 

  2979. .L4_12:

  2980. 

  2981. 	KERNEL4x4_M1

  2982. 	KERNEL4x4_M2

  2983. 	KERNEL4x4_M1

  2984. 	KERNEL4x4_M2

  2985. 

  2986. 	KERNEL4x4_M1

  2987. 	KERNEL4x4_M2

  2988. 	KERNEL4x4_M1

  2989. 	KERNEL4x4_M2

  2990. 

  2991. 	dec	%rax

  2992. 	jne	.L4_12

  2993. 

  2994. .L4_12a:

  2995. 

  2996. 	KERNEL4x4_M1

  2997. 	KERNEL4x4_M2

  2998. 	KERNEL4x4_M1

  2999. 	KERNEL4x4_M2

  3000. 

  3001. 	KERNEL4x4_M1

  3002. 	KERNEL4x4_M2

  3003. 	KERNEL4x4_M1

  3004. 	KERNEL4x4_E

  3005. 

  3006. 	jmp .L4_16

  3007. 

  3008. 

  3009. .L4_13:

  3010. 

  3011. 	test $1, %rax

  3012. 	jz .L4_14

  3013. 

  3014. 	KERNEL4x4_I

  3015. 	KERNEL4x4_M2

  3016. 	KERNEL4x4_M1

  3017. 	KERNEL4x4_M2

  3018. 

  3019. 	KERNEL4x4_M1

  3020. 	KERNEL4x4_M2

  3021. 	KERNEL4x4_M1

  3022. 	KERNEL4x4_E

  3023. 

  3024. 	jmp .L4_16

  3025. 

  3026. 

  3027. .L4_14:

  3028. 

  3029. 	INIT4x4

  3030. 

  3031. 

  3032. .L4_16:

  3033.         movq    K, %rax

  3034. 

  3035. 	andq	$7, %rax		# if (k & 1)

  3036. 	je .L4_19

  3037. 

  3038. 	ALIGN_4

  3039. 

  3040. .L4_17:

  3041. 

  3042. 	KERNEL4x4_SUB

  3043. 

  3044. 	dec	%rax

  3045. 	jne	.L4_17

  3046. 	ALIGN_4

  3047. 

  3048. 

  3049. .L4_19:

  3050. 

  3051. 	SAVE4x4

  3052. 

  3053. 	decq	I			# i --

  3054. 	jg	.L4_11

  3055. 

  3056. 	ALIGN_4	

  3057. 

  3058. /**************************************************************************

  3059. * Rest of M 

  3060. ***************************************************************************/

  3061. .L4_20:

  3062. 	// Test rest of M

  3063. 

  3064. 	testq	$3, M

  3065. 	jz	.L4_100			// to next 16 lines of N

  3066. 

  3067. 

  3068. .L4_30:

  3069. 	testq	$2, M		

  3070. 	jz	.L4_40

  3071. 

  3072. 	ALIGN_4

  3073. 

  3074. .L4_31:

  3075.         movq    B, BO             // first buffer to BO

  3076.         addq    $12 * SIZE, BO

  3077. 

  3078. 	INIT2x4

  3079. 

  3080.         movq    K, %rax

  3081. 

  3082. 	sarq	$3, %rax

  3083. 	je	.L4_36

  3084. 	ALIGN_4

  3085. 

  3086. .L4_32:

  3087. 

  3088. 	KERNEL2x4_SUB

  3089. 	KERNEL2x4_SUB

  3090. 	KERNEL2x4_SUB

  3091. 	KERNEL2x4_SUB

  3092. 

  3093. 	KERNEL2x4_SUB

  3094. 	KERNEL2x4_SUB

  3095. 	KERNEL2x4_SUB

  3096. 	KERNEL2x4_SUB

  3097. 

  3098. 	dec %rax

  3099. 	jne	.L4_32

  3100. 	ALIGN_4

  3101. 

  3102. .L4_36:

  3103.         movq    K, %rax

  3104. 

  3105. 	andq	$7, %rax		# if (k & 1)

  3106. 	je .L4_39

  3107. 

  3108. 	ALIGN_4

  3109. 

  3110. .L4_37:

  3111. 

  3112. 	KERNEL2x4_SUB

  3113. 

  3114. 	dec %rax

  3115. 	jne	.L4_37

  3116. 

  3117. 

  3118. .L4_39:

  3119. 

  3120. 	SAVE2x4

  3121. 

  3122. .L4_40:

  3123. 	testq	$1, M		

  3124. 	jz	.L4_100		// to next 3 lines of N

  3125. 

  3126. 	ALIGN_4

  3127. 

  3128. .L4_41:

  3129.         movq    B, BO             // first buffer to BO

  3130.         addq    $12 * SIZE, BO

  3131. 

  3132. 	INIT1x4

  3133. 

  3134.         movq    K, %rax

  3135. 

  3136. 	sarq	$3,%rax

  3137. 	je	.L4_46

  3138. 

  3139. 	ALIGN_4

  3140. 

  3141. .L4_42:

  3142. 

  3143. 	KERNEL1x4_SUB

  3144. 	KERNEL1x4_SUB

  3145. 	KERNEL1x4_SUB

  3146. 	KERNEL1x4_SUB

  3147. 

  3148. 	KERNEL1x4_SUB

  3149. 	KERNEL1x4_SUB

  3150. 	KERNEL1x4_SUB

  3151. 	KERNEL1x4_SUB

  3152. 

  3153. 	dec %rax

  3154. 	jne	.L4_42

  3155. 	ALIGN_4

  3156. 

  3157. .L4_46:

  3158.         movq    K, %rax

  3159. 

  3160. 	andq	$7, %rax		# if (k & 1)

  3161. 	je .L4_49

  3162. 

  3163. 	ALIGN_4

  3164. 

  3165. .L4_47:

  3166. 

  3167. 	KERNEL1x4_SUB

  3168. 

  3169. 	dec	%rax

  3170. 	jne	.L4_47

  3171. 	ALIGN_4

  3172. 

  3173. 

  3174. .L4_49:

  3175. 

  3176. 	SAVE1x4

  3177. 

  3178. 	ALIGN_4

  3179. 	

  3180. .L4_100:

  3181. 

  3182. 	movq	K, %rax

  3183. 	salq	$2, %rax		// * 4

  3184. 	leaq	(B , %rax, SIZE), B

  3185. 

  3186. 

  3187. 

  3188. 

  3189. /***************************************************************************************************************/

  3190. 

  3191. .L2_0:

  3192. 

  3193. 	movq	Nmod12, J		

  3194. 	testq	$2, J

  3195. 	je	.L1_0

  3196. 

  3197. .L2_10:

  3198. 	movq	C, CO1

  3199. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  3200. 

  3201. 	

  3202. 	movq	A, AO		 	// aoffset = a

  3203. 	addq	$16 * SIZE, AO

  3204. 

  3205. 	movq	M,  I

  3206. 	sarq	$2, I			// i = m / 4

  3207. 	je	.L2_20

  3208. 

  3209. 	ALIGN_4

  3210. 

  3211. .L2_11:

  3212.         movq    B, BO        

  3213.         addq    $12 * SIZE, BO

  3214. 

  3215. 	INIT4x2

  3216. 

  3217.         movq    K, %rax

  3218. 	sarq $3, %rax			//  K / 8

  3219. 

  3220. 	je	.L2_16

  3221. 

  3222. 	ALIGN_5

  3223. 

  3224. .L2_12:

  3225. 

  3226. 	KERNEL4x2_SUB

  3227. 	KERNEL4x2_SUB

  3228. 	KERNEL4x2_SUB

  3229. 	KERNEL4x2_SUB

  3230. 

  3231. 	KERNEL4x2_SUB

  3232. 	KERNEL4x2_SUB

  3233. 	KERNEL4x2_SUB

  3234. 	KERNEL4x2_SUB

  3235. 

  3236. 	dec	%rax

  3237. 	jne	.L2_12

  3238. 

  3239. 

  3240. .L2_16:

  3241.         movq    K, %rax

  3242. 

  3243. 	andq	$7, %rax		# if (k & 1)

  3244. 	je .L2_19

  3245. 

  3246. 	ALIGN_4

  3247. 

  3248. .L2_17:

  3249. 

  3250. 	KERNEL4x2_SUB

  3251. 

  3252. 	dec	%rax

  3253. 	jne	.L2_17

  3254. 	ALIGN_4

  3255. 

  3256. 

  3257. .L2_19:

  3258. 

  3259. 	SAVE4x2

  3260. 

  3261. 	decq	I			# i --

  3262. 	jg	.L2_11

  3263. 	ALIGN_4	

  3264. 

  3265. /**************************************************************************

  3266. * Rest of M 

  3267. ***************************************************************************/

  3268. .L2_20:

  3269. 	// Test rest of M

  3270. 

  3271. 	testq	$3, M

  3272. 	jz	.L2_100			// to next 16 lines of N

  3273. 

  3274. 

  3275. .L2_30:

  3276. 	testq	$2, M		

  3277. 	jz	.L2_40

  3278. 

  3279. 	ALIGN_4

  3280. 

  3281. .L2_31:

  3282.         movq    B, BO             // first buffer to BO

  3283.         addq    $12 * SIZE, BO

  3284. 

  3285. 	INIT2x2

  3286. 

  3287.         movq    K, %rax

  3288. 

  3289. 	sarq	$3, %rax

  3290. 	je	.L2_36

  3291. 	ALIGN_4

  3292. 

  3293. .L2_32:

  3294. 

  3295. 	KERNEL2x2_SUB

  3296. 	KERNEL2x2_SUB

  3297. 	KERNEL2x2_SUB

  3298. 	KERNEL2x2_SUB

  3299. 

  3300. 	KERNEL2x2_SUB

  3301. 	KERNEL2x2_SUB

  3302. 	KERNEL2x2_SUB

  3303. 	KERNEL2x2_SUB

  3304. 

  3305. 	dec %rax

  3306. 	jne	.L2_32

  3307. 

  3308. .L2_36:

  3309.         movq    K, %rax

  3310. 

  3311. 	andq	$7, %rax		# if (k & 1)

  3312. 	je .L2_39

  3313. 

  3314. 	ALIGN_4

  3315. 

  3316. .L2_37:

  3317. 

  3318. 	KERNEL2x2_SUB

  3319. 

  3320. 	dec %rax

  3321. 	jne	.L2_37

  3322. 

  3323. 

  3324. .L2_39:

  3325. 

  3326. 	SAVE2x2

  3327. 

  3328. .L2_40:

  3329. 	testq	$1, M		

  3330. 	jz	.L2_100		// to next 3 lines of N

  3331. 

  3332. .L2_41:

  3333.         movq    B, BO             // first buffer to BO

  3334.         addq    $12 * SIZE, BO

  3335. 

  3336. 	INIT1x2

  3337. 

  3338.         movq    K, %rax

  3339. 

  3340. 	sarq	$3,%rax

  3341. 	je	.L2_46

  3342. 

  3343. 	ALIGN_4

  3344. 

  3345. .L2_42:

  3346. 

  3347. 	KERNEL1x2_SUB

  3348. 	KERNEL1x2_SUB

  3349. 	KERNEL1x2_SUB

  3350. 	KERNEL1x2_SUB

  3351. 

  3352. 	KERNEL1x2_SUB

  3353. 	KERNEL1x2_SUB

  3354. 	KERNEL1x2_SUB

  3355. 	KERNEL1x2_SUB

  3356. 

  3357. 	dec %rax

  3358. 	jne	.L2_42

  3359. 

  3360. .L2_46:

  3361.         movq    K, %rax

  3362. 

  3363. 	andq	$7, %rax		# if (k & 1)

  3364. 	je .L2_49

  3365. 

  3366. 	ALIGN_4

  3367. 

  3368. .L2_47:

  3369. 

  3370. 	KERNEL1x2_SUB

  3371. 

  3372. 	dec	%rax

  3373. 	jne	.L2_47

  3374. 

  3375. .L2_49:

  3376. 

  3377. 	SAVE1x2

  3378. 

  3379. .L2_100:

  3380. 

  3381. 	movq	K, %rax

  3382. 	salq	$1, %rax		// * 2

  3383. 	leaq	(B , %rax, SIZE), B

  3384. 

  3385. /***************************************************************************************************************/

  3386. 

  3387. .L1_0:

  3388. 

  3389. 	movq	Nmod12, J		

  3390. 	testq	$1, J

  3391. 	je	.L999

  3392. 

  3393. .L1_10:

  3394. 	movq	C, CO1

  3395. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  3396. 

  3397. 	

  3398. 	movq	A, AO		 	// aoffset = a

  3399. 	addq	$16 * SIZE, AO

  3400. 

  3401. 	movq	M,  I

  3402. 	sarq	$2, I			// i = m / 4

  3403. 	je	.L1_20

  3404. 

  3405. 	ALIGN_4

  3406. 

  3407. .L1_11:

  3408.         movq    B, BO        

  3409.         addq    $12 * SIZE, BO

  3410. 

  3411. 	INIT4x1

  3412. 

  3413.         movq    K, %rax

  3414. 

  3415. 	sarq	$3, %rax			//  K / 8

  3416. 	je	.L1_16

  3417. 

  3418. 	ALIGN_5

  3419. 

  3420. .L1_12:

  3421. 

  3422. 	KERNEL4x1

  3423. 

  3424. 	dec	%rax

  3425. 	jne	.L1_12

  3426. 

  3427. 

  3428. .L1_16:

  3429.         movq    K, %rax

  3430. 

  3431. 	andq	$7, %rax		# if (k & 1)

  3432. 	je .L1_19

  3433. 

  3434. 	ALIGN_4

  3435. 

  3436. .L1_17:

  3437. 

  3438. 	KERNEL4x1_SUB

  3439. 

  3440. 	dec	%rax

  3441. 	jne	.L1_17

  3442. 	ALIGN_4

  3443. 

  3444. 

  3445. .L1_19:

  3446. 

  3447. 	SAVE4x1

  3448. 

  3449. 	decq	I			# i --

  3450. 	jg	.L1_11

  3451. 

  3452. /**************************************************************************

  3453. * Rest of M 

  3454. ***************************************************************************/

  3455. .L1_20:

  3456. 	// Test rest of M

  3457. 

  3458. 	testq	$3, M

  3459. 	jz	.L1_100	

  3460. 

  3461. 

  3462. .L1_30:

  3463. 	testq	$2, M		

  3464. 	jz	.L1_40

  3465. 

  3466. 	ALIGN_4

  3467. 

  3468. .L1_31:

  3469.         movq    B, BO             // first buffer to BO

  3470.         addq    $12 * SIZE, BO

  3471. 

  3472. 	INIT2x1

  3473. 

  3474.         movq    K, %rax

  3475. 

  3476. 	sarq	$3, %rax

  3477. 	je	.L1_36

  3478. 	ALIGN_4

  3479. 

  3480. .L1_32:

  3481. 

  3482. 	KERNEL2x1_SUB

  3483. 	KERNEL2x1_SUB

  3484. 	KERNEL2x1_SUB

  3485. 	KERNEL2x1_SUB

  3486. 

  3487. 	KERNEL2x1_SUB

  3488. 	KERNEL2x1_SUB

  3489. 	KERNEL2x1_SUB

  3490. 	KERNEL2x1_SUB

  3491. 

  3492. 

  3493. 	dec %rax

  3494. 	jne	.L1_32

  3495. 

  3496. .L1_36:

  3497.         movq    K, %rax

  3498. 

  3499. 	andq	$7, %rax		# if (k & 1)

  3500. 	je .L1_39

  3501. 

  3502. 	ALIGN_4

  3503. 

  3504. .L1_37:

  3505. 

  3506. 	KERNEL2x1_SUB

  3507. 

  3508. 	dec %rax

  3509. 	jne	.L1_37

  3510. 

  3511. .L1_39:

  3512. 

  3513. 	SAVE2x1

  3514. 

  3515. .L1_40:

  3516. 	testq	$1, M		

  3517. 	jz	.L1_100		// to next 3 lines of N

  3518. 

  3519. 

  3520. .L1_41:

  3521.         movq    B, BO             // first buffer to BO

  3522.         addq    $12 * SIZE, BO

  3523. 

  3524. 	INIT1x1

  3525. 

  3526.         movq    K, %rax

  3527. 

  3528. 	sarq	$3,%rax

  3529. 	je	.L1_46

  3530. 

  3531. 	ALIGN_4

  3532. 

  3533. .L1_42:

  3534. 

  3535. 	KERNEL1x1_SUB

  3536. 	KERNEL1x1_SUB

  3537. 	KERNEL1x1_SUB

  3538. 	KERNEL1x1_SUB

  3539. 

  3540. 	KERNEL1x1_SUB

  3541. 	KERNEL1x1_SUB

  3542. 	KERNEL1x1_SUB

  3543. 	KERNEL1x1_SUB

  3544. 

  3545. 	dec %rax

  3546. 	jne	.L1_42

  3547. 

  3548. .L1_46:

  3549.         movq    K, %rax

  3550. 

  3551. 	andq	$7, %rax		# if (k & 1)

  3552. 	je .L1_49

  3553. 

  3554. 	ALIGN_4

  3555. 

  3556. .L1_47:

  3557. 

  3558. 	KERNEL1x1_SUB

  3559. 

  3560. 	dec	%rax

  3561. 	jne	.L1_47

  3562. 

  3563. 

  3564. .L1_49:

  3565. 

  3566. 	SAVE1x1

  3567. 

  3568. .L1_100:

  3569. 

  3570. 

  3571. 

  3572. 

  3573. .L999:

  3574. 	vzeroupper

  3575. 

  3576. 	movq   		SP, %rsp

  3577. 	movq	   (%rsp), %rbx

  3578. 	movq	  8(%rsp), %rbp

  3579. 	movq	 16(%rsp), %r12

  3580. 	movq	 24(%rsp), %r13

  3581. 	movq	 32(%rsp), %r14

  3582. 	movq	 40(%rsp), %r15

  3583. 

  3584. #ifdef WINDOWS_ABI

  3585. 	movq	 48(%rsp), %rdi

  3586. 	movq	 56(%rsp), %rsi

  3587. 	vmovups	 64(%rsp), %xmm6

  3588. 	vmovups	 80(%rsp), %xmm7

  3589. 	vmovups	 96(%rsp), %xmm8

  3590. 	vmovups	112(%rsp), %xmm9

  3591. 	vmovups	128(%rsp), %xmm10

  3592. 	vmovups	144(%rsp), %xmm11

  3593. 	vmovups	160(%rsp), %xmm12

  3594. 	vmovups	176(%rsp), %xmm13

  3595. 	vmovups	192(%rsp), %xmm14

  3596. 	vmovups	208(%rsp), %xmm15

  3597. #endif

  3598. 

  3599. 	addq	$STACKSIZE, %rsp

  3600. 	ret

  3601. 

  3602. 	EPILOGUE

  3603. 

  3604. 

  3605. #else

  3606. /*************************************************************************************

  3607. * TRMM Kernel

  3608. *************************************************************************************/

  3609. 

  3610. 

  3611. 	PROLOGUE

  3612. 	PROFCODE

  3613. 	

  3614. 	subq	$STACKSIZE, %rsp

  3615. 	movq	%rbx,   (%rsp)

  3616. 	movq	%rbp,  8(%rsp)

  3617. 	movq	%r12, 16(%rsp)

  3618. 	movq	%r13, 24(%rsp)

  3619. 	movq	%r14, 32(%rsp)

  3620. 	movq	%r15, 40(%rsp)

  3621. 

  3622. 	vzeroupper

  3623. 

  3624. #ifdef WINDOWS_ABI

  3625. 	movq	%rdi,    48(%rsp)

  3626. 	movq	%rsi,    56(%rsp)

  3627. 	vmovups	%xmm6,   64(%rsp)

  3628. 	vmovups	%xmm7,   80(%rsp)

  3629. 	vmovups	%xmm8,   96(%rsp)

  3630. 	vmovups	%xmm9,  112(%rsp)

  3631. 	vmovups	%xmm10, 128(%rsp)

  3632. 	vmovups	%xmm11, 144(%rsp)

  3633. 	vmovups	%xmm12, 160(%rsp)

  3634. 	vmovups	%xmm13, 176(%rsp)

  3635. 	vmovups	%xmm14, 192(%rsp)

  3636. 	vmovups	%xmm15, 208(%rsp)

  3637. 

  3638. 	movq	ARG1,      OLD_M

  3639. 	movq	ARG2,      OLD_N

  3640. 	movq	ARG3,      OLD_K

  3641. 	movq	OLD_A,     A

  3642. 	movq	OLD_B,     B

  3643. 	movq	OLD_C,     C

  3644. 	movq	OLD_LDC,   LDC

  3645. #ifdef TRMMKERNEL

  3646. 	vmovsd	OLD_OFFSET, %xmm12

  3647. #endif

  3648. 	vmovups	%xmm3, %xmm0

  3649. 

  3650. #else

  3651. 	movq	STACKSIZE +  8(%rsp), LDC

  3652. #ifdef TRMMKERNEL

  3653. 	vmovsd	STACKSIZE + 16(%rsp), %xmm12

  3654. #endif

  3655. 

  3656. #endif

  3657. 

  3658. 	movq    %rsp, SP      # save old stack

  3659.         subq    $128 + L_BUFFER_SIZE, %rsp

  3660.         andq    $-4096, %rsp    # align stack

  3661. 

  3662.         STACK_TOUCH

  3663. 

  3664. 	cmpq	$ 0, OLD_M

  3665. 	je	.L999

  3666. 

  3667. 	cmpq	$ 0, OLD_N

  3668. 	je	.L999

  3669. 

  3670. 	cmpq	$ 0, OLD_K

  3671. 	je	.L999

  3672. 

  3673. 	movq	OLD_M, M

  3674. 	movq	OLD_N, N

  3675. 	movq	OLD_K, K

  3676. 

  3677. 	vmovsd	 %xmm0, ALPHA

  3678. 

  3679. 	salq	$BASE_SHIFT, LDC

  3680. 

  3681. 	movq    N, %rax

  3682.         xorq    %rdx, %rdx

  3683.         movq    $8,  %rdi

  3684.         divq    %rdi                     //    N / 8

  3685.         movq    %rax, Ndiv12             //    N / 8

  3686.         movq    %rdx, Nmod12             //    N % 8

  3687. 

  3688. #ifdef TRMMKERNEL

  3689.         vmovsd  %xmm12, OFFSET

  3690.         vmovsd  %xmm12, KK

  3691. #ifndef LEFT

  3692.         negq    KK

  3693. #endif  

  3694. #endif

  3695. 

  3696. /*************************************************************************************************/

  3697. .L8_0:

  3698. 	movq	Ndiv12,  J

  3699. 	cmpq	$ 0, J

  3700. 	je	.L4_0

  3701. 	ALIGN_4

  3702. 

  3703. .L8_10:

  3704. 	movq	C, CO1

  3705. 	leaq	(C, LDC, 8), C		// c += 8 * ldc

  3706. 

  3707. #if defined(TRMMKERNEL) && defined(LEFT)

  3708.         movq    OFFSET, %rax

  3709.         movq    %rax, KK

  3710. #endif

  3711. 

  3712. 	

  3713. 	movq	A, AO		 	// aoffset = a

  3714. 	addq	$16 * SIZE, AO

  3715. 

  3716. 	movq	M,  I

  3717. 	sarq	$2, I			// i = m / 4

  3718. 	je	.L8_20

  3719. 

  3720. 	ALIGN_4

  3721. 

  3722. .L8_11:

  3723. 

  3724. #if !defined(TRMMKERNEL) || \

  3725.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3726.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3727.         movq    B, BO        

  3728.         addq    $12 * SIZE, BO

  3729. #else

  3730.         movq    B, BO        

  3731.         addq    $12 * SIZE, BO

  3732.         movq    KK, %rax

  3733. 	salq	$3, %rax		// rax * SIZE

  3734. 	leaq	(BO,%rax,8), BO		// add number of values in B

  3735. 	leaq	(AO,%rax,4), AO		// add number of values in A

  3736. #endif

  3737. 

  3738. 

  3739. #ifndef TRMMKERNEL

  3740.         movq    K, %rax

  3741. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3742.         movq    K, %rax

  3743.         subq    KK, %rax

  3744.         movq    %rax, KKK

  3745. #else

  3746.         movq    KK, %rax

  3747. #ifdef LEFT

  3748.         addq    $4, %rax        // number of values in AO

  3749. #else

  3750.         addq    $8, %rax        // number of values in BO

  3751. #endif

  3752.         movq    %rax, KKK

  3753. #endif

  3754. 

  3755. 	sarq	$3, %rax			//  K / 8

  3756. 	cmpq    $2, %rax

  3757. 	jl	.L8_13

  3758. 

  3759. 

  3760. 	KERNEL4x8_I

  3761. 	KERNEL4x8_M2

  3762. 	KERNEL4x8_M1

  3763. 	KERNEL4x8_M2

  3764. 

  3765. 	KERNEL4x8_M1

  3766. 	KERNEL4x8_M2

  3767. 	KERNEL4x8_M1

  3768. 	KERNEL4x8_M2

  3769. 

  3770. 	subq $2, %rax

  3771. 	je	.L8_12a

  3772. 

  3773. 	ALIGN_5

  3774. 

  3775. .L8_12:

  3776. 

  3777. 	KERNEL4x8_M1

  3778. 	KERNEL4x8_M2

  3779. 	KERNEL4x8_M1

  3780. 	KERNEL4x8_M2

  3781. 

  3782. 	KERNEL4x8_M1

  3783. 	KERNEL4x8_M2

  3784. 	KERNEL4x8_M1

  3785. 	KERNEL4x8_M2

  3786. 

  3787. 	dec	%rax

  3788. 	jne	.L8_12

  3789. 

  3790. .L8_12a:

  3791. 

  3792. 	KERNEL4x8_M1

  3793. 	KERNEL4x8_M2

  3794. 	KERNEL4x8_M1

  3795. 	KERNEL4x8_M2

  3796. 

  3797. 	KERNEL4x8_M1

  3798. 	KERNEL4x8_M2

  3799. 	KERNEL4x8_M1

  3800. 	KERNEL4x8_E

  3801. 

  3802. 	jmp .L8_16

  3803. 

  3804. 

  3805. .L8_13:

  3806. 

  3807. 	test $1, %rax

  3808. 	jz .L8_14

  3809. 

  3810. 	KERNEL4x8_I

  3811. 	KERNEL4x8_M2

  3812. 	KERNEL4x8_M1

  3813. 	KERNEL4x8_M2

  3814. 

  3815. 	KERNEL4x8_M1

  3816. 	KERNEL4x8_M2

  3817. 	KERNEL4x8_M1

  3818. 	KERNEL4x8_E

  3819. 

  3820. 	jmp .L8_16

  3821. 

  3822. 

  3823. .L8_14:

  3824. 

  3825. 	INIT4x8

  3826. 

  3827. 

  3828. .L8_16:

  3829.         movq    KKK, %rax

  3830. 

  3831. 	andq	$7, %rax		# if (k & 1)

  3832. 	je .L8_19

  3833. 

  3834. 	ALIGN_4

  3835. 

  3836. .L8_17:

  3837. 

  3838. 	KERNEL4x8_SUB

  3839. 

  3840. 	dec	%rax

  3841. 	jne	.L8_17

  3842. 	ALIGN_4

  3843. 

  3844. 

  3845. .L8_19:

  3846. 

  3847. 	SAVE4x8

  3848. 

  3849. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3850.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3851.         movq    K, %rax

  3852.         subq    KKK, %rax

  3853. 	salq	$3, %rax			// rax + SIZE

  3854.         leaq    (BO, %rax, 8), BO		// number of values in B

  3855.         leaq    (AO, %rax, 4), AO		// number of values in A

  3856. #endif

  3857. 

  3858. #if defined(TRMMKERNEL) && defined(LEFT)

  3859.         addq    $4, KK				// number of values in A

  3860. #endif

  3861. 

  3862. 	decq	I			# i --

  3863. 	jg	.L8_11

  3864. 	ALIGN_4	

  3865. 

  3866. /**************************************************************************

  3867. * Rest of M 

  3868. ***************************************************************************/

  3869. .L8_20:

  3870. 	// Test rest of M

  3871. 

  3872. 	testq	$3, M

  3873. 	jz	.L8_100			// to next 16 lines of N

  3874. 

  3875. 

  3876. .L8_30:

  3877. 	testq	$2, M		

  3878. 	jz	.L8_40

  3879. 

  3880. 	ALIGN_4

  3881. 

  3882. .L8_31:

  3883. 

  3884. #if !defined(TRMMKERNEL) || \

  3885.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3886.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3887.         movq    B, BO        

  3888.         addq    $12 * SIZE, BO

  3889. #else

  3890.         movq    B, BO        

  3891.         addq    $12 * SIZE, BO

  3892.         movq    KK, %rax

  3893. 	salq	$3, %rax		// rax * SIZE

  3894. 	leaq	(BO,%rax,8), BO		// add number of values in B

  3895. 	leaq	(AO,%rax,2), AO		// add number of values in A

  3896. #endif

  3897. 

  3898. 

  3899. #ifndef TRMMKERNEL

  3900.         movq    K, %rax

  3901. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3902.         movq    K, %rax

  3903.         subq    KK, %rax

  3904.         movq    %rax, KKK

  3905. #else

  3906.         movq    KK, %rax

  3907. #ifdef LEFT

  3908.         addq    $2, %rax        // number of values in AO

  3909. #else

  3910.         addq    $8, %rax        // number of values in BO

  3911. #endif

  3912.         movq    %rax, KKK

  3913. #endif

  3914. 

  3915. 	INIT2x8

  3916. 

  3917. 	sarq	$3, %rax

  3918. 	je	.L8_36

  3919. 	ALIGN_4

  3920. 

  3921. .L8_32:

  3922. 

  3923. 	KERNEL2x8_SUB

  3924. 	KERNEL2x8_SUB

  3925. 	KERNEL2x8_SUB

  3926. 	KERNEL2x8_SUB

  3927. 

  3928. 	KERNEL2x8_SUB

  3929. 	KERNEL2x8_SUB

  3930. 	KERNEL2x8_SUB

  3931. 	KERNEL2x8_SUB

  3932. 

  3933. 	dec %rax

  3934. 	jne	.L8_32

  3935. 	ALIGN_4

  3936. 

  3937. .L8_36:

  3938.         movq    KKK, %rax

  3939. 

  3940. 	andq	$7, %rax		# if (k & 1)

  3941. 	je .L8_39

  3942. 

  3943. 	ALIGN_4

  3944. 

  3945. .L8_37:

  3946. 

  3947. 	KERNEL2x8_SUB

  3948. 

  3949. 	dec %rax

  3950. 	jne	.L8_37

  3951. 

  3952. 

  3953. .L8_39:

  3954. 

  3955. 	SAVE2x8

  3956. 

  3957. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3958.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3959.         movq    K, %rax

  3960.         subq    KKK, %rax

  3961. 	salq	$3, %rax			// rax + SIZE

  3962.         leaq    (BO, %rax, 8), BO		// number of values in B

  3963.         leaq    (AO, %rax, 2), AO		// number of values in A

  3964. #endif

  3965. 

  3966. #if defined(TRMMKERNEL) && defined(LEFT)

  3967.         addq    $2, KK				// number of values in A

  3968. #endif

  3969. 

  3970. 

  3971. .L8_40:

  3972. 	testq	$1, M		

  3973. 	jz	.L8_100		// to next 3 lines of N

  3974. 

  3975. 	ALIGN_4

  3976. 

  3977. .L8_41:

  3978. 

  3979. #if !defined(TRMMKERNEL) || \

  3980.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3981.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3982.         movq    B, BO        

  3983.         addq    $12 * SIZE, BO

  3984. #else

  3985.         movq    B, BO        

  3986.         addq    $12 * SIZE, BO

  3987.         movq    KK, %rax

  3988. 	salq	$3, %rax		// rax * SIZE

  3989. 	leaq	(BO,%rax,8), BO		// add number of values in B

  3990. 	leaq	(AO,%rax,1), AO		// add number of values in A

  3991. #endif

  3992. 

  3993. 

  3994. #ifndef TRMMKERNEL

  3995.         movq    K, %rax

  3996. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3997.         movq    K, %rax

  3998.         subq    KK, %rax

  3999.         movq    %rax, KKK

  4000. #else

  4001.         movq    KK, %rax

  4002. #ifdef LEFT

  4003.         addq    $1, %rax        // number of values in AO

  4004. #else

  4005.         addq    $8, %rax        // number of values in BO

  4006. #endif

  4007.         movq    %rax, KKK

  4008. #endif

  4009. 

  4010. 	INIT1x8

  4011. 

  4012. 	sarq	$3,%rax

  4013. 	je	.L8_46

  4014. 

  4015. 	ALIGN_4

  4016. 

  4017. .L8_42:

  4018. 

  4019. 	KERNEL1x8_SUB

  4020. 	KERNEL1x8_SUB

  4021. 	KERNEL1x8_SUB

  4022. 	KERNEL1x8_SUB

  4023. 

  4024. 	KERNEL1x8_SUB

  4025. 	KERNEL1x8_SUB

  4026. 	KERNEL1x8_SUB

  4027. 	KERNEL1x8_SUB

  4028. 

  4029. 	dec %rax

  4030. 	jne	.L8_42

  4031. 	ALIGN_4

  4032. 

  4033. .L8_46:

  4034.         movq    KKK, %rax

  4035. 

  4036. 	andq	$7, %rax		# if (k & 1)

  4037. 	je .L8_49

  4038. 

  4039. 	ALIGN_4

  4040. 

  4041. .L8_47:

  4042. 

  4043. 	KERNEL1x8_SUB

  4044. 

  4045. 	dec	%rax

  4046. 	jne	.L8_47

  4047. 	ALIGN_4

  4048. 

  4049. 

  4050. .L8_49:

  4051. 

  4052. 	SAVE1x8

  4053. 

  4054. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4055.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4056.         movq    K, %rax

  4057.         subq    KKK, %rax

  4058. 	salq	$3, %rax			// rax + SIZE

  4059.         leaq    (BO, %rax, 8), BO		// number of values in B

  4060.         leaq    (AO, %rax, 1), AO		// number of values in A

  4061. #endif

  4062. 

  4063. #if defined(TRMMKERNEL) && defined(LEFT)

  4064.         addq    $1, KK				// number of values in A

  4065. #endif

  4066. 

  4067. .L8_100:

  4068. 

  4069. #if defined(TRMMKERNEL) && !defined(LEFT)

  4070.         addq    $8, KK				// number of values in B

  4071. #endif

  4072. 

  4073. 

  4074. 	decq	J			// j --

  4075. 	jg	.L8_10

  4076. 

  4077. 

  4078. 

  4079. 

  4080. 

  4081. /*************************************************************************************************/

  4082. .L4_0:

  4083. 	movq	Nmod12, J		

  4084. 	testq	$4, J

  4085. 	je	.L2_0

  4086. 	ALIGN_4

  4087. 

  4088. .L4_10:

  4089. 	movq	C, CO1

  4090. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  4091. 

  4092. #if defined(TRMMKERNEL) && defined(LEFT)

  4093.         movq    OFFSET, %rax

  4094.         movq    %rax, KK

  4095. #endif

  4096. 

  4097. 	

  4098. 	movq	A, AO		 	// aoffset = a

  4099. 	addq	$16 * SIZE, AO

  4100. 

  4101. 	movq	M,  I

  4102. 	sarq	$2, I			// i = m / 4

  4103. 	je	.L4_20

  4104. 

  4105. 	ALIGN_4

  4106. 

  4107. .L4_11:

  4108. 

  4109. #if !defined(TRMMKERNEL) || \

  4110.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4111.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4112.         movq    B, BO        

  4113.         addq    $12 * SIZE, BO

  4114. #else

  4115.         movq    B, BO        

  4116.         addq    $12 * SIZE, BO

  4117.         movq    KK, %rax

  4118. 	salq	$3, %rax		// rax * SIZE

  4119. 	leaq	(BO,%rax,4), BO		// add number of values in B

  4120. 	leaq	(AO,%rax,4), AO		// add number of values in A

  4121. #endif

  4122. 

  4123. 

  4124. #ifndef TRMMKERNEL

  4125.         movq    K, %rax

  4126. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4127.         movq    K, %rax

  4128.         subq    KK, %rax

  4129.         movq    %rax, KKK

  4130. #else

  4131.         movq    KK, %rax

  4132. #ifdef LEFT

  4133.         addq    $4, %rax        // number of values in AO

  4134. #else

  4135.         addq    $4, %rax        // number of values in BO

  4136. #endif

  4137.         movq    %rax, KKK

  4138. #endif

  4139. 

  4140. 	sarq	$3, %rax			//  K / 8

  4141. 	cmpq    $2, %rax

  4142. 	jl	.L4_13

  4143. 

  4144. 

  4145. 	KERNEL4x4_I

  4146. 	KERNEL4x4_M2

  4147. 	KERNEL4x4_M1

  4148. 	KERNEL4x4_M2

  4149. 

  4150. 	KERNEL4x4_M1

  4151. 	KERNEL4x4_M2

  4152. 	KERNEL4x4_M1

  4153. 	KERNEL4x4_M2

  4154. 

  4155. 	subq $2, %rax

  4156. 	je	.L4_12a

  4157. 

  4158. 	ALIGN_5

  4159. 

  4160. .L4_12:

  4161. 

  4162. 	KERNEL4x4_M1

  4163. 	KERNEL4x4_M2

  4164. 	KERNEL4x4_M1

  4165. 	KERNEL4x4_M2

  4166. 

  4167. 	KERNEL4x4_M1

  4168. 	KERNEL4x4_M2

  4169. 	KERNEL4x4_M1

  4170. 	KERNEL4x4_M2

  4171. 

  4172. 	dec	%rax

  4173. 	jne	.L4_12

  4174. 

  4175. .L4_12a:

  4176. 

  4177. 	KERNEL4x4_M1

  4178. 	KERNEL4x4_M2

  4179. 	KERNEL4x4_M1

  4180. 	KERNEL4x4_M2

  4181. 

  4182. 	KERNEL4x4_M1

  4183. 	KERNEL4x4_M2

  4184. 	KERNEL4x4_M1

  4185. 	KERNEL4x4_E

  4186. 

  4187. 	jmp .L4_16

  4188. 

  4189. 

  4190. .L4_13:

  4191. 

  4192. 	test $1, %rax

  4193. 	jz .L4_14

  4194. 

  4195. 	KERNEL4x4_I

  4196. 	KERNEL4x4_M2

  4197. 	KERNEL4x4_M1

  4198. 	KERNEL4x4_M2

  4199. 

  4200. 	KERNEL4x4_M1

  4201. 	KERNEL4x4_M2

  4202. 	KERNEL4x4_M1

  4203. 	KERNEL4x4_E

  4204. 

  4205. 	jmp .L4_16

  4206. 

  4207. 

  4208. .L4_14:

  4209. 

  4210. 	INIT4x4

  4211. 

  4212. 

  4213. .L4_16:

  4214.         movq    KKK, %rax

  4215. 

  4216. 	andq	$7, %rax		# if (k & 1)

  4217. 	je .L4_19

  4218. 

  4219. 	ALIGN_4

  4220. 

  4221. .L4_17:

  4222. 

  4223. 	KERNEL4x4_SUB

  4224. 

  4225. 	dec	%rax

  4226. 	jne	.L4_17

  4227. 	ALIGN_4

  4228. 

  4229. 

  4230. .L4_19:

  4231. 

  4232. 	SAVE4x4

  4233. 

  4234. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4235.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4236.         movq    K, %rax

  4237.         subq    KKK, %rax

  4238. 	salq	$3, %rax			// rax + SIZE

  4239.         leaq    (BO, %rax, 4), BO		// number of values in B

  4240.         leaq    (AO, %rax, 4), AO		// number of values in A

  4241. #endif

  4242. 

  4243. #if defined(TRMMKERNEL) && defined(LEFT)

  4244.         addq    $4, KK				// number of values in A

  4245. #endif

  4246. 

  4247. 	decq	I			# i --

  4248. 	jg	.L4_11

  4249. 	ALIGN_4	

  4250. 

  4251. /**************************************************************************

  4252. * Rest of M 

  4253. ***************************************************************************/

  4254. .L4_20:

  4255. 	// Test rest of M

  4256. 

  4257. 	testq	$3, M

  4258. 	jz	.L4_100			// to next 16 lines of N

  4259. 

  4260. 

  4261. .L4_30:

  4262. 	testq	$2, M		

  4263. 	jz	.L4_40

  4264. 

  4265. 	ALIGN_4

  4266. 

  4267. .L4_31:

  4268. 

  4269. #if !defined(TRMMKERNEL) || \

  4270.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4271.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4272.         movq    B, BO        

  4273.         addq    $12 * SIZE, BO

  4274. #else

  4275.         movq    B, BO        

  4276.         addq    $12 * SIZE, BO

  4277.         movq    KK, %rax

  4278. 	salq	$3, %rax		// rax * SIZE

  4279. 	leaq	(BO,%rax,4), BO		// add number of values in B

  4280. 	leaq	(AO,%rax,2), AO		// add number of values in A

  4281. #endif

  4282. 

  4283. 

  4284. #ifndef TRMMKERNEL

  4285.         movq    K, %rax

  4286. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4287.         movq    K, %rax

  4288.         subq    KK, %rax

  4289.         movq    %rax, KKK

  4290. #else

  4291.         movq    KK, %rax

  4292. #ifdef LEFT

  4293.         addq    $2, %rax        // number of values in AO

  4294. #else

  4295.         addq    $4, %rax        // number of values in BO

  4296. #endif

  4297.         movq    %rax, KKK

  4298. #endif

  4299. 

  4300. 	INIT2x4

  4301. 

  4302. 	sarq	$3, %rax

  4303. 	je	.L4_36

  4304. 	ALIGN_4

  4305. 

  4306. .L4_32:

  4307. 

  4308. 	KERNEL2x4_SUB

  4309. 	KERNEL2x4_SUB

  4310. 	KERNEL2x4_SUB

  4311. 	KERNEL2x4_SUB

  4312. 

  4313. 	KERNEL2x4_SUB

  4314. 	KERNEL2x4_SUB

  4315. 	KERNEL2x4_SUB

  4316. 	KERNEL2x4_SUB

  4317. 

  4318. 	dec %rax

  4319. 	jne	.L4_32

  4320. 	ALIGN_4

  4321. 

  4322. .L4_36:

  4323.         movq    KKK, %rax

  4324. 

  4325. 	andq	$7, %rax		# if (k & 1)

  4326. 	je .L4_39

  4327. 

  4328. 	ALIGN_4

  4329. 

  4330. .L4_37:

  4331. 

  4332. 	KERNEL2x4_SUB

  4333. 

  4334. 	dec %rax

  4335. 	jne	.L4_37

  4336. 

  4337. 

  4338. .L4_39:

  4339. 

  4340. 	SAVE2x4

  4341. 

  4342. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4343.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4344.         movq    K, %rax

  4345.         subq    KKK, %rax

  4346. 	salq	$3, %rax			// rax + SIZE

  4347.         leaq    (BO, %rax, 4), BO		// number of values in B

  4348.         leaq    (AO, %rax, 2), AO		// number of values in A

  4349. #endif

  4350. 

  4351. #if defined(TRMMKERNEL) && defined(LEFT)

  4352.         addq    $2, KK				// number of values in A

  4353. #endif

  4354. 

  4355. 

  4356. .L4_40:

  4357. 	testq	$1, M		

  4358. 	jz	.L4_100		// to next 3 lines of N

  4359. 

  4360. 	ALIGN_4

  4361. 

  4362. .L4_41:

  4363. 

  4364. #if !defined(TRMMKERNEL) || \

  4365.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4366.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4367.         movq    B, BO        

  4368.         addq    $12 * SIZE, BO

  4369. #else

  4370.         movq    B, BO        

  4371.         addq    $12 * SIZE, BO

  4372.         movq    KK, %rax

  4373. 	salq	$3, %rax		// rax * SIZE

  4374. 	leaq	(BO,%rax,4), BO		// add number of values in B

  4375. 	leaq	(AO,%rax,1), AO		// add number of values in A

  4376. #endif

  4377. 

  4378. 

  4379. #ifndef TRMMKERNEL

  4380.         movq    K, %rax

  4381. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4382.         movq    K, %rax

  4383.         subq    KK, %rax

  4384.         movq    %rax, KKK

  4385. #else

  4386.         movq    KK, %rax

  4387. #ifdef LEFT

  4388.         addq    $1, %rax        // number of values in AO

  4389. #else

  4390.         addq    $4, %rax        // number of values in BO

  4391. #endif

  4392.         movq    %rax, KKK

  4393. #endif

  4394. 

  4395. 	INIT1x4

  4396. 

  4397. 	sarq	$3,%rax

  4398. 	je	.L4_46

  4399. 

  4400. 	ALIGN_4

  4401. 

  4402. .L4_42:

  4403. 

  4404. 	KERNEL1x4_SUB

  4405. 	KERNEL1x4_SUB

  4406. 	KERNEL1x4_SUB

  4407. 	KERNEL1x4_SUB

  4408. 

  4409. 	KERNEL1x4_SUB

  4410. 	KERNEL1x4_SUB

  4411. 	KERNEL1x4_SUB

  4412. 	KERNEL1x4_SUB

  4413. 

  4414. 	dec %rax

  4415. 	jne	.L4_42

  4416. 	ALIGN_4

  4417. 

  4418. .L4_46:

  4419.         movq    KKK, %rax

  4420. 

  4421. 	andq	$7, %rax		# if (k & 1)

  4422. 	je .L4_49

  4423. 

  4424. 	ALIGN_4

  4425. 

  4426. .L4_47:

  4427. 

  4428. 	KERNEL1x4_SUB

  4429. 

  4430. 	dec	%rax

  4431. 	jne	.L4_47

  4432. 	ALIGN_4

  4433. 

  4434. 

  4435. .L4_49:

  4436. 

  4437. 	SAVE1x4

  4438. 

  4439. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4440.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4441.         movq    K, %rax

  4442.         subq    KKK, %rax

  4443. 	salq	$3, %rax			// rax + SIZE

  4444.         leaq    (BO, %rax, 4), BO		// number of values in B

  4445.         leaq    (AO, %rax, 1), AO		// number of values in A

  4446. #endif

  4447. 

  4448. #if defined(TRMMKERNEL) && defined(LEFT)

  4449.         addq    $1, KK				// number of values in A

  4450. #endif

  4451. 

  4452. .L4_100:

  4453. 

  4454. #if defined(TRMMKERNEL) && !defined(LEFT)

  4455.         addq    $4, KK				// number of values in B

  4456. #endif

  4457. 

  4458. 

  4459. 	movq	K, %rax

  4460. 	salq	$2, %rax		// * 4

  4461. 	leaq	(B , %rax, SIZE), B

  4462. 

  4463. 

  4464. 

  4465. 

  4466. /***************************************************************************************************************/

  4467. 

  4468. .L2_0:

  4469. 

  4470. 	movq	Nmod12, J		

  4471. 	testq	$2, J

  4472. 	je	.L1_0

  4473. 

  4474. .L2_10:

  4475. 	movq	C, CO1

  4476. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  4477. 

  4478. #if defined(TRMMKERNEL) && defined(LEFT)

  4479.         movq    OFFSET, %rax

  4480.         movq    %rax, KK

  4481. #endif

  4482. 

  4483. 

  4484. 	

  4485. 	movq	A, AO		 	// aoffset = a

  4486. 	addq	$16 * SIZE, AO

  4487. 

  4488. 	movq	M,  I

  4489. 	sarq	$2, I			// i = m / 4

  4490. 	je	.L2_20

  4491. 

  4492. 	ALIGN_4

  4493. 

  4494. .L2_11:

  4495. 

  4496. #if !defined(TRMMKERNEL) || \

  4497.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4498.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4499.         movq    B, BO        

  4500.         addq    $12 * SIZE, BO

  4501. #else

  4502.         movq    B, BO        

  4503.         addq    $12 * SIZE, BO

  4504.         movq    KK, %rax

  4505. 	salq	$3, %rax		// rax * SIZE

  4506. 	leaq	(BO,%rax,2), BO		// add number of values in B

  4507. 	leaq	(AO,%rax,4), AO		// add number of values in A

  4508. #endif

  4509. 

  4510. 

  4511. #ifndef TRMMKERNEL

  4512.         movq    K, %rax

  4513. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4514.         movq    K, %rax

  4515.         subq    KK, %rax

  4516.         movq    %rax, KKK

  4517. #else

  4518.         movq    KK, %rax

  4519. #ifdef LEFT

  4520.         addq    $4, %rax        // number of values in AO

  4521. #else

  4522.         addq    $2, %rax        // number of values in BO

  4523. #endif

  4524.         movq    %rax, KKK

  4525. #endif

  4526. 

  4527. 	INIT4x2

  4528. 

  4529. 	sarq $3, %rax			//  K / 8

  4530. 

  4531. 	je	.L2_16

  4532. 

  4533. 	ALIGN_5

  4534. 

  4535. .L2_12:

  4536. 

  4537. 	KERNEL4x2_SUB

  4538. 	KERNEL4x2_SUB

  4539. 	KERNEL4x2_SUB

  4540. 	KERNEL4x2_SUB

  4541. 

  4542. 	KERNEL4x2_SUB

  4543. 	KERNEL4x2_SUB

  4544. 	KERNEL4x2_SUB

  4545. 	KERNEL4x2_SUB

  4546. 

  4547. 	dec	%rax

  4548. 	jne	.L2_12

  4549. 

  4550. 

  4551. .L2_16:

  4552.         movq    KKK, %rax

  4553. 

  4554. 	andq	$7, %rax		# if (k & 1)

  4555. 	je .L2_19

  4556. 

  4557. 	ALIGN_4

  4558. 

  4559. .L2_17:

  4560. 

  4561. 	KERNEL4x2_SUB

  4562. 

  4563. 	dec	%rax

  4564. 	jne	.L2_17

  4565. 	ALIGN_4

  4566. 

  4567. 

  4568. .L2_19:

  4569. 

  4570. 	SAVE4x2

  4571. 

  4572. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4573.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4574.         movq    K, %rax

  4575.         subq    KKK, %rax

  4576. 	salq	$3, %rax			// rax + SIZE

  4577.         leaq    (BO, %rax, 2), BO		// number of values in B

  4578.         leaq    (AO, %rax, 4), AO		// number of values in A

  4579. #endif

  4580. 

  4581. #if defined(TRMMKERNEL) && defined(LEFT)

  4582.         addq    $4, KK				// number of values in A

  4583. #endif

  4584. 

  4585. 

  4586. 	decq	I			# i --

  4587. 	jg	.L2_11

  4588. 	ALIGN_4	

  4589. 

  4590. /**************************************************************************

  4591. * Rest of M 

  4592. ***************************************************************************/

  4593. .L2_20:

  4594. 	// Test rest of M

  4595. 

  4596. 	testq	$3, M

  4597. 	jz	.L2_100			// to next 16 lines of N

  4598. 

  4599. 

  4600. .L2_30:

  4601. 	testq	$2, M		

  4602. 	jz	.L2_40

  4603. 

  4604. 	ALIGN_4

  4605. 

  4606. .L2_31:

  4607. 

  4608. #if !defined(TRMMKERNEL) || \

  4609.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4610.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4611.         movq    B, BO        

  4612.         addq    $12 * SIZE, BO

  4613. #else

  4614.         movq    B, BO        

  4615.         addq    $12 * SIZE, BO

  4616.         movq    KK, %rax

  4617. 	salq	$3, %rax		// rax * SIZE

  4618. 	leaq	(BO,%rax,2), BO		// add number of values in B

  4619. 	leaq	(AO,%rax,2), AO		// add number of values in A

  4620. #endif

  4621. 

  4622. 

  4623. #ifndef TRMMKERNEL

  4624.         movq    K, %rax

  4625. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4626.         movq    K, %rax

  4627.         subq    KK, %rax

  4628.         movq    %rax, KKK

  4629. #else

  4630.         movq    KK, %rax

  4631. #ifdef LEFT

  4632.         addq    $2, %rax        // number of values in AO

  4633. #else

  4634.         addq    $2, %rax        // number of values in BO

  4635. #endif

  4636.         movq    %rax, KKK

  4637. #endif

  4638. 

  4639. 	INIT2x2

  4640. 

  4641. 	sarq	$3, %rax

  4642. 	je	.L2_36

  4643. 	ALIGN_4

  4644. 

  4645. .L2_32:

  4646. 

  4647. 	KERNEL2x2_SUB

  4648. 	KERNEL2x2_SUB

  4649. 	KERNEL2x2_SUB

  4650. 	KERNEL2x2_SUB

  4651. 

  4652. 	KERNEL2x2_SUB

  4653. 	KERNEL2x2_SUB

  4654. 	KERNEL2x2_SUB

  4655. 	KERNEL2x2_SUB

  4656. 

  4657. 	dec %rax

  4658. 	jne	.L2_32

  4659. 

  4660. .L2_36:

  4661.         movq    KKK, %rax

  4662. 

  4663. 	andq	$7, %rax		# if (k & 1)

  4664. 	je .L2_39

  4665. 

  4666. 	ALIGN_4

  4667. 

  4668. .L2_37:

  4669. 

  4670. 	KERNEL2x2_SUB

  4671. 

  4672. 	dec %rax

  4673. 	jne	.L2_37

  4674. 

  4675. 

  4676. .L2_39:

  4677. 

  4678. 	SAVE2x2

  4679. 

  4680. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4681.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4682.         movq    K, %rax

  4683.         subq    KKK, %rax

  4684. 	salq	$3, %rax			// rax + SIZE

  4685.         leaq    (BO, %rax, 2), BO		// number of values in B

  4686.         leaq    (AO, %rax, 2), AO		// number of values in A

  4687. #endif

  4688. 

  4689. #if defined(TRMMKERNEL) && defined(LEFT)

  4690.         addq    $2, KK				// number of values in A

  4691. #endif

  4692. 

  4693. 

  4694. .L2_40:

  4695. 	testq	$1, M		

  4696. 	jz	.L2_100		// to next 3 lines of N

  4697. 

  4698. .L2_41:

  4699. 

  4700. #if !defined(TRMMKERNEL) || \

  4701.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4702.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4703.         movq    B, BO        

  4704.         addq    $12 * SIZE, BO

  4705. #else

  4706.         movq    B, BO        

  4707.         addq    $12 * SIZE, BO

  4708.         movq    KK, %rax

  4709. 	salq	$3, %rax		// rax * SIZE

  4710. 	leaq	(BO,%rax,2), BO		// add number of values in B

  4711. 	leaq	(AO,%rax,1), AO		// add number of values in A

  4712. #endif

  4713. 

  4714. 

  4715. #ifndef TRMMKERNEL

  4716.         movq    K, %rax

  4717. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4718.         movq    K, %rax

  4719.         subq    KK, %rax

  4720.         movq    %rax, KKK

  4721. #else

  4722.         movq    KK, %rax

  4723. #ifdef LEFT

  4724.         addq    $1, %rax        // number of values in AO

  4725. #else

  4726.         addq    $2, %rax        // number of values in BO

  4727. #endif

  4728.         movq    %rax, KKK

  4729. #endif

  4730. 

  4731. 	INIT1x2

  4732. 

  4733. 	sarq	$3,%rax

  4734. 	je	.L2_46

  4735. 

  4736. 	ALIGN_4

  4737. 

  4738. .L2_42:

  4739. 

  4740. 	KERNEL1x2_SUB

  4741. 	KERNEL1x2_SUB

  4742. 	KERNEL1x2_SUB

  4743. 	KERNEL1x2_SUB

  4744. 

  4745. 	KERNEL1x2_SUB

  4746. 	KERNEL1x2_SUB

  4747. 	KERNEL1x2_SUB

  4748. 	KERNEL1x2_SUB

  4749. 

  4750. 	dec %rax

  4751. 	jne	.L2_42

  4752. 

  4753. .L2_46:

  4754.         movq    KKK, %rax

  4755. 

  4756. 	andq	$7, %rax		# if (k & 1)

  4757. 	je .L2_49

  4758. 

  4759. 	ALIGN_4

  4760. 

  4761. .L2_47:

  4762. 

  4763. 	KERNEL1x2_SUB

  4764. 

  4765. 	dec	%rax

  4766. 	jne	.L2_47

  4767. 

  4768. .L2_49:

  4769. 

  4770. 	SAVE1x2

  4771. 

  4772. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4773.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4774.         movq    K, %rax

  4775.         subq    KKK, %rax

  4776. 	salq	$3, %rax			// rax * SIZE

  4777.         leaq    (BO, %rax, 2), BO		// number of values in B

  4778.         leaq    (AO, %rax, 1), AO		// number of values in A

  4779. #endif

  4780. 

  4781. #if defined(TRMMKERNEL) && defined(LEFT)

  4782.         addq    $1, KK				// number of values in A

  4783. #endif

  4784. 

  4785. 

  4786. .L2_100:

  4787. 

  4788. 

  4789. #if defined(TRMMKERNEL) && !defined(LEFT)

  4790.         addq    $2, KK				// number of values in B

  4791. #endif

  4792. 

  4793. 	movq	K, %rax

  4794. 	salq	$1, %rax		// * 2

  4795. 	leaq	(B , %rax, SIZE), B

  4796. 

  4797. /***************************************************************************************************************/

  4798. 

  4799. .L1_0:

  4800. 

  4801. 	movq	Nmod12, J		

  4802. 	testq	$1, J

  4803. 	je	.L999

  4804. 

  4805. .L1_10:

  4806. 	movq	C, CO1

  4807. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  4808. 

  4809. #if defined(TRMMKERNEL) && defined(LEFT)

  4810.         movq    OFFSET, %rax

  4811.         movq    %rax, KK

  4812. #endif

  4813. 	

  4814. 	movq	A, AO		 	// aoffset = a

  4815. 	addq	$16 * SIZE, AO

  4816. 

  4817. 	movq	M,  I

  4818. 	sarq	$2, I			// i = m / 4

  4819. 	je	.L1_20

  4820. 

  4821. 	ALIGN_4

  4822. 

  4823. .L1_11:

  4824. 

  4825. #if !defined(TRMMKERNEL) || \

  4826.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4827.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4828.         movq    B, BO        

  4829.         addq    $12 * SIZE, BO

  4830. #else

  4831.         movq    B, BO        

  4832.         addq    $12 * SIZE, BO

  4833.         movq    KK, %rax

  4834. 	salq	$3, %rax		// rax * SIZE

  4835. 	leaq	(BO,%rax,1), BO		// add number of values in B

  4836. 	leaq	(AO,%rax,4), AO		// add number of values in A

  4837. #endif

  4838. 

  4839. 

  4840. #ifndef TRMMKERNEL

  4841.         movq    K, %rax

  4842. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4843.         movq    K, %rax

  4844.         subq    KK, %rax

  4845.         movq    %rax, KKK

  4846. #else

  4847.         movq    KK, %rax

  4848. #ifdef LEFT

  4849.         addq    $4, %rax        // number of values in AO

  4850. #else

  4851.         addq    $1, %rax        // number of values in BO

  4852. #endif

  4853.         movq    %rax, KKK

  4854. #endif

  4855. 

  4856. 	INIT4x1

  4857. 

  4858. 	sarq	$3, %rax			//  K / 8

  4859. 	je	.L1_16

  4860. 

  4861. 	ALIGN_5

  4862. 

  4863. .L1_12:

  4864. 

  4865. 	KERNEL4x1

  4866. 

  4867. 	dec	%rax

  4868. 	jne	.L1_12

  4869. 

  4870. 

  4871. .L1_16:

  4872.         movq    KKK, %rax

  4873. 

  4874. 	andq	$7, %rax		# if (k & 1)

  4875. 	je .L1_19

  4876. 

  4877. 	ALIGN_4

  4878. 

  4879. .L1_17:

  4880. 

  4881. 	KERNEL4x1_SUB

  4882. 

  4883. 	dec	%rax

  4884. 	jne	.L1_17

  4885. 	ALIGN_4

  4886. 

  4887. 

  4888. .L1_19:

  4889. 

  4890. 	SAVE4x1

  4891. 

  4892. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4893.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4894.         movq    K, %rax

  4895.         subq    KKK, %rax

  4896. 	salq	$3, %rax			// rax * SIZE

  4897.         leaq    (BO, %rax, 1), BO		// number of values in B

  4898.         leaq    (AO, %rax, 4), AO		// number of values in A

  4899. #endif

  4900. 

  4901. #if defined(TRMMKERNEL) && defined(LEFT)

  4902.         addq    $4, KK				// number of values in A

  4903. #endif

  4904. 

  4905. 

  4906. 	decq	I			# i --

  4907. 	jg	.L1_11

  4908. 

  4909. /**************************************************************************

  4910. * Rest of M 

  4911. ***************************************************************************/

  4912. .L1_20:

  4913. 	// Test rest of M

  4914. 

  4915. 	testq	$3, M

  4916. 	jz	.L1_100	

  4917. 

  4918. 

  4919. .L1_30:

  4920. 	testq	$2, M		

  4921. 	jz	.L1_40

  4922. 

  4923. 	ALIGN_4

  4924. 

  4925. .L1_31:

  4926. 

  4927. #if !defined(TRMMKERNEL) || \

  4928.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4929.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4930.         movq    B, BO        

  4931.         addq    $12 * SIZE, BO

  4932. #else

  4933.         movq    B, BO        

  4934.         addq    $12 * SIZE, BO

  4935.         movq    KK, %rax

  4936. 	salq	$3, %rax		// rax * SIZE

  4937. 	leaq	(BO,%rax,1), BO		// add number of values in B

  4938. 	leaq	(AO,%rax,2), AO		// add number of values in A

  4939. #endif

  4940. 

  4941. 

  4942. #ifndef TRMMKERNEL

  4943.         movq    K, %rax

  4944. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4945.         movq    K, %rax

  4946.         subq    KK, %rax

  4947.         movq    %rax, KKK

  4948. #else

  4949.         movq    KK, %rax

  4950. #ifdef LEFT

  4951.         addq    $2, %rax        // number of values in AO

  4952. #else

  4953.         addq    $1, %rax        // number of values in BO

  4954. #endif

  4955.         movq    %rax, KKK

  4956. #endif

  4957. 

  4958. 	INIT2x1

  4959. 

  4960. 	sarq	$3, %rax

  4961. 	je	.L1_36

  4962. 	ALIGN_4

  4963. 

  4964. .L1_32:

  4965. 

  4966. 	KERNEL2x1_SUB

  4967. 	KERNEL2x1_SUB

  4968. 	KERNEL2x1_SUB

  4969. 	KERNEL2x1_SUB

  4970. 

  4971. 	KERNEL2x1_SUB

  4972. 	KERNEL2x1_SUB

  4973. 	KERNEL2x1_SUB

  4974. 	KERNEL2x1_SUB

  4975. 

  4976. 

  4977. 	dec %rax

  4978. 	jne	.L1_32

  4979. 

  4980. .L1_36:

  4981.         movq    KKK, %rax

  4982. 

  4983. 	andq	$7, %rax		# if (k & 1)

  4984. 	je .L1_39

  4985. 

  4986. 	ALIGN_4

  4987. 

  4988. .L1_37:

  4989. 

  4990. 	KERNEL2x1_SUB

  4991. 

  4992. 	dec %rax

  4993. 	jne	.L1_37

  4994. 

  4995. .L1_39:

  4996. 

  4997. 	SAVE2x1

  4998. 

  4999. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5000.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5001.         movq    K, %rax

  5002.         subq    KKK, %rax

  5003. 	salq	$3, %rax			// rax * SIZE

  5004.         leaq    (BO, %rax, 1), BO		// number of values in B

  5005.         leaq    (AO, %rax, 2), AO		// number of values in A

  5006. #endif

  5007. 

  5008. #if defined(TRMMKERNEL) && defined(LEFT)

  5009.         addq    $2, KK				// number of values in A

  5010. #endif

  5011. 

  5012. 

  5013. .L1_40:

  5014. 	testq	$1, M		

  5015. 	jz	.L1_100		// to next 3 lines of N

  5016. 

  5017. 

  5018. .L1_41:

  5019. 

  5020. #if !defined(TRMMKERNEL) || \

  5021.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5022.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5023.         movq    B, BO        

  5024.         addq    $12 * SIZE, BO

  5025. #else

  5026.         movq    B, BO        

  5027.         addq    $12 * SIZE, BO

  5028.         movq    KK, %rax

  5029. 	salq	$3, %rax		// rax * SIZE

  5030. 	leaq	(BO,%rax,1), BO		// add number of values in B

  5031. 	leaq	(AO,%rax,1), AO		// add number of values in A

  5032. #endif

  5033. 

  5034. 

  5035. #ifndef TRMMKERNEL

  5036.         movq    K, %rax

  5037. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5038.         movq    K, %rax

  5039.         subq    KK, %rax

  5040.         movq    %rax, KKK

  5041. #else

  5042.         movq    KK, %rax

  5043. #ifdef LEFT

  5044.         addq    $1, %rax        // number of values in AO

  5045. #else

  5046.         addq    $1, %rax        // number of values in BO

  5047. #endif

  5048.         movq    %rax, KKK

  5049. #endif

  5050. 

  5051. 	INIT1x1

  5052. 

  5053. 	sarq	$3,%rax

  5054. 	je	.L1_46

  5055. 

  5056. 	ALIGN_4

  5057. 

  5058. .L1_42:

  5059. 

  5060. 	KERNEL1x1_SUB

  5061. 	KERNEL1x1_SUB

  5062. 	KERNEL1x1_SUB

  5063. 	KERNEL1x1_SUB

  5064. 

  5065. 	KERNEL1x1_SUB

  5066. 	KERNEL1x1_SUB

  5067. 	KERNEL1x1_SUB

  5068. 	KERNEL1x1_SUB

  5069. 

  5070. 	dec %rax

  5071. 	jne	.L1_42

  5072. 

  5073. .L1_46:

  5074.         movq    KKK, %rax

  5075. 

  5076. 	andq	$7, %rax		# if (k & 1)

  5077. 	je .L1_49

  5078. 

  5079. 	ALIGN_4

  5080. 

  5081. .L1_47:

  5082. 

  5083. 	KERNEL1x1_SUB

  5084. 

  5085. 	dec	%rax

  5086. 	jne	.L1_47

  5087. 

  5088. 

  5089. .L1_49:

  5090. 

  5091. 	SAVE1x1

  5092. 

  5093. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5094.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5095.         movq    K, %rax

  5096.         subq    KKK, %rax

  5097. 	salq	$3, %rax			// rax * SIZE

  5098.         leaq    (BO, %rax, 1), BO		// number of values in B

  5099.         leaq    (AO, %rax, 1), AO		// number of values in A

  5100. #endif

  5101. 

  5102. #if defined(TRMMKERNEL) && defined(LEFT)

  5103.         addq    $1, KK				// number of values in A

  5104. #endif

  5105. 

  5106. 

  5107. 

  5108. .L1_100:

  5109. 

  5110. 

  5111. #if defined(TRMMKERNEL) && !defined(LEFT)

  5112.         addq    $1, KK				// number of values in B

  5113. #endif

  5114. 

  5115. 

  5116. 

  5117. .L999:

  5118. 

  5119. 	vzeroupper

  5120. 

  5121. 	movq   		SP, %rsp

  5122. 	movq	   (%rsp), %rbx

  5123. 	movq	  8(%rsp), %rbp

  5124. 	movq	 16(%rsp), %r12

  5125. 	movq	 24(%rsp), %r13

  5126. 	movq	 32(%rsp), %r14

  5127. 	movq	 40(%rsp), %r15

  5128. 

  5129. #ifdef WINDOWS_ABI

  5130. 	movq	 48(%rsp), %rdi

  5131. 	movq	 56(%rsp), %rsi

  5132. 	vmovups	 64(%rsp), %xmm6

  5133. 	vmovups	 80(%rsp), %xmm7

  5134. 	vmovups	 96(%rsp), %xmm8

  5135. 	vmovups	112(%rsp), %xmm9

  5136. 	vmovups	128(%rsp), %xmm10

  5137. 	vmovups	144(%rsp), %xmm11

  5138. 	vmovups	160(%rsp), %xmm12

  5139. 	vmovups	176(%rsp), %xmm13

  5140. 	vmovups	192(%rsp), %xmm14

  5141. 	vmovups	208(%rsp), %xmm15

  5142. #endif

  5143. 

  5144. 	addq	$STACKSIZE, %rsp

  5145. 	ret

  5146. 

  5147. 	EPILOGUE

  5148. 

  5149. 

  5150. 

  5151. 

  5152. 

  5153. #endif