OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 490a2e5d23
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '490a2e5d23'
${ noResults }
OpenBLAS/kernel/x86_64/sgemm_kernel_16x4_skylakex.c

1642 lines
42 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2013, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. 

  29. /* comment below left for history, data does not represent the implementation in this file */

  30. 

  31. /*********************************************************************

  32. * 2014/07/28 Saar

  33. *        BLASTEST               : OK

  34. *        CTEST                  : OK

  35. *        TEST                   : OK

  36. *

  37. * 2013/10/28 Saar

  38. * Parameter:

  39. *	SGEMM_DEFAULT_UNROLL_N	4

  40. *	SGEMM_DEFAULT_UNROLL_M	16

  41. *	SGEMM_DEFAULT_P		768

  42. *	SGEMM_DEFAULT_Q		384

  43. *	A_PR1			512

  44. *	B_PR1			512

  45. *	

  46. * 

  47. * 2014/07/28 Saar

  48. * Performance at 9216x9216x9216:

  49. *       1 thread:      102 GFLOPS       (SANDYBRIDGE:  59)      (MKL:   83)

  50. *       2 threads:     195 GFLOPS       (SANDYBRIDGE: 116)      (MKL:  155)

  51. *       3 threads:     281 GFLOPS       (SANDYBRIDGE: 165)      (MKL:  230)

  52. *       4 threads:     366 GFLOPS       (SANDYBRIDGE: 223)      (MKL:  267)

  53. *

  54. *********************************************************************/

  55. 

  56. #include "common.h"

  57. #include <immintrin.h>

  58. 

  59. 

  60. 

  61. /*******************************************************************************************

  62. * 8 lines of N

  63. *******************************************************************************************/

  64.  

  65. 

  66. 

  67. 

  68. 

  69. 

  70. /*******************************************************************************************

  71. * 4 lines of N

  72. *******************************************************************************************/

  73. 

  74. #define INIT64x4()	\

  75. 	row0 = _mm512_setzero_ps();					\

  76. 	row1 = _mm512_setzero_ps();					\

  77. 	row2 = _mm512_setzero_ps();					\

  78. 	row3 = _mm512_setzero_ps();					\

  79. 	row0b = _mm512_setzero_ps();					\

  80. 	row1b = _mm512_setzero_ps();					\

  81. 	row2b = _mm512_setzero_ps();					\

  82. 	row3b = _mm512_setzero_ps();					\

  83. 	row0c = _mm512_setzero_ps();					\

  84. 	row1c = _mm512_setzero_ps();					\

  85. 	row2c = _mm512_setzero_ps();					\

  86. 	row3c = _mm512_setzero_ps();					\

  87. 	row0d = _mm512_setzero_ps();					\

  88. 	row1d = _mm512_setzero_ps();					\

  89. 	row2d = _mm512_setzero_ps();					\

  90. 	row3d = _mm512_setzero_ps();					\

  91. 

  92. #define KERNEL64x4_SUB() 						\

  93. 	zmm0   = _mm512_loadu_ps(AO);					\

  94. 	zmm1   = _mm512_loadu_ps(A1);					\

  95. 	zmm5   = _mm512_loadu_ps(A2);					\

  96. 	zmm7   = _mm512_loadu_ps(A3);					\

  97. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  98. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+1));		\

  99. 	row0  += zmm0 * zmm2;						\

  100. 	row1  += zmm0 * zmm3;						\

  101. 	row0b += zmm1 * zmm2;						\

  102. 	row1b += zmm1 * zmm3;						\

  103. 	row0c += zmm5 * zmm2;						\

  104. 	row1c += zmm5 * zmm3;						\

  105. 	row0d += zmm7 * zmm2;						\

  106. 	row1d += zmm7 * zmm3;						\

  107. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO+2));		\

  108. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+3));		\

  109. 	row2  += zmm0 * zmm2;						\

  110. 	row3 += zmm0 * zmm3;						\

  111. 	row2b += zmm1 * zmm2;						\

  112. 	row3b += zmm1 * zmm3;						\

  113. 	row2c += zmm5 * zmm2;						\

  114. 	row3c += zmm5 * zmm3;						\

  115. 	row2d += zmm7 * zmm2;						\

  116. 	row3d += zmm7 * zmm3;						\

  117. 	BO += 4;							\

  118. 	AO += 16;							\

  119. 	A1 += 16;							\

  120. 	A2 += 16;							\

  121. 	A3 += 16;							\

  122. 

  123. 

  124. #define SAVE64x4(ALPHA)							\

  125. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  126. 	row0  *= zmm0;							\

  127. 	row1  *= zmm0;							\

  128. 	row2  *= zmm0;							\

  129. 	row3 *= zmm0;							\

  130. 	row0b *= zmm0;							\

  131. 	row1b *= zmm0;							\

  132. 	row2b *= zmm0;							\

  133. 	row3b *= zmm0;							\

  134. 	row0c *= zmm0;							\

  135. 	row1c *= zmm0;							\

  136. 	row2c *= zmm0;							\

  137. 	row3c *= zmm0;							\

  138. 	row0d *= zmm0;							\

  139. 	row1d *= zmm0;							\

  140. 	row2d *= zmm0;							\

  141. 	row3d *= zmm0;							\

  142. 	row0  += _mm512_loadu_ps(CO1 + 0*ldc);				\

  143. 	row1  += _mm512_loadu_ps(CO1 + 1*ldc);				\

  144. 	row2  += _mm512_loadu_ps(CO1 + 2*ldc);				\

  145. 	row3 += _mm512_loadu_ps(CO1 + 3*ldc);				\

  146. 	_mm512_storeu_ps(CO1 + 0*ldc, row0);				\

  147. 	_mm512_storeu_ps(CO1 + 1*ldc, row1);				\

  148. 	_mm512_storeu_ps(CO1 + 2*ldc, row2);				\

  149. 	_mm512_storeu_ps(CO1 + 3*ldc, row3);				\

  150. 	row0b  += _mm512_loadu_ps(CO1 + 0*ldc + 16);			\

  151. 	row1b  += _mm512_loadu_ps(CO1 + 1*ldc + 16);			\

  152. 	row2b  += _mm512_loadu_ps(CO1 + 2*ldc + 16);			\

  153. 	row3b += _mm512_loadu_ps(CO1 + 3*ldc + 16);			\

  154. 	_mm512_storeu_ps(CO1 + 0*ldc + 16, row0b);			\

  155. 	_mm512_storeu_ps(CO1 + 1*ldc + 16, row1b);			\

  156. 	_mm512_storeu_ps(CO1 + 2*ldc + 16, row2b);			\

  157. 	_mm512_storeu_ps(CO1 + 3*ldc + 16, row3b);			\

  158. 	row0c  += _mm512_loadu_ps(CO1 + 0*ldc + 32);			\

  159. 	row1c  += _mm512_loadu_ps(CO1 + 1*ldc + 32);			\

  160. 	row2c  += _mm512_loadu_ps(CO1 + 2*ldc + 32);			\

  161. 	row3c  += _mm512_loadu_ps(CO1 + 3*ldc + 32);			\

  162. 	_mm512_storeu_ps(CO1 + 0*ldc + 32, row0c);			\

  163. 	_mm512_storeu_ps(CO1 + 1*ldc + 32, row1c);			\

  164. 	_mm512_storeu_ps(CO1 + 2*ldc + 32, row2c);			\

  165. 	_mm512_storeu_ps(CO1 + 3*ldc + 32, row3c);			\

  166. 	row0d  += _mm512_loadu_ps(CO1 + 0*ldc + 48);			\

  167. 	row1d  += _mm512_loadu_ps(CO1 + 1*ldc + 48);			\

  168. 	row2d  += _mm512_loadu_ps(CO1 + 2*ldc + 48);			\

  169. 	row3d  += _mm512_loadu_ps(CO1 + 3*ldc + 48);			\

  170. 	_mm512_storeu_ps(CO1 + 0*ldc + 48, row0d);			\

  171. 	_mm512_storeu_ps(CO1 + 1*ldc + 48, row1d);			\

  172. 	_mm512_storeu_ps(CO1 + 2*ldc + 48, row2d);			\

  173. 	_mm512_storeu_ps(CO1 + 3*ldc + 48, row3d);		

  174. 

  175. 

  176. #define INIT48x4()	\

  177. 	row0 = _mm512_setzero_ps();					\

  178. 	row1 = _mm512_setzero_ps();					\

  179. 	row2 = _mm512_setzero_ps();					\

  180. 	row3 = _mm512_setzero_ps();					\

  181. 	row0b = _mm512_setzero_ps();					\

  182. 	row1b = _mm512_setzero_ps();					\

  183. 	row2b = _mm512_setzero_ps();					\

  184. 	row3b = _mm512_setzero_ps();					\

  185. 	row0c = _mm512_setzero_ps();					\

  186. 	row1c = _mm512_setzero_ps();					\

  187. 	row2c = _mm512_setzero_ps();					\

  188. 	row3c = _mm512_setzero_ps();					\

  189. 

  190. #define KERNEL48x4_SUB() 						\

  191. 	zmm0   = _mm512_loadu_ps(AO);					\

  192. 	zmm1   = _mm512_loadu_ps(A1);					\

  193. 	zmm5   = _mm512_loadu_ps(A2);					\

  194. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  195. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+1));		\

  196. 	row0  += zmm0 * zmm2;						\

  197. 	row1  += zmm0 * zmm3;						\

  198. 	row0b += zmm1 * zmm2;						\

  199. 	row1b += zmm1 * zmm3;						\

  200. 	row0c += zmm5 * zmm2;						\

  201. 	row1c += zmm5 * zmm3;						\

  202. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO+2));		\

  203. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+3));		\

  204. 	row2  += zmm0 * zmm2;						\

  205. 	row3 += zmm0 * zmm3;						\

  206. 	row2b += zmm1 * zmm2;						\

  207. 	row3b += zmm1 * zmm3;						\

  208. 	row2c += zmm5 * zmm2;						\

  209. 	row3c += zmm5 * zmm3;						\

  210. 	BO += 4;							\

  211. 	AO += 16;							\

  212. 	A1 += 16;							\

  213. 	A2 += 16;

  214. 

  215. 

  216. #define SAVE48x4(ALPHA)							\

  217. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  218. 	row0  *= zmm0;							\

  219. 	row1  *= zmm0;							\

  220. 	row2  *= zmm0;							\

  221. 	row3 *= zmm0;							\

  222. 	row0b *= zmm0;							\

  223. 	row1b *= zmm0;							\

  224. 	row2b *= zmm0;							\

  225. 	row3b *= zmm0;							\

  226. 	row0c *= zmm0;							\

  227. 	row1c *= zmm0;							\

  228. 	row2c *= zmm0;							\

  229. 	row3c *= zmm0;							\

  230. 	row0  += _mm512_loadu_ps(CO1 + 0*ldc);				\

  231. 	row1  += _mm512_loadu_ps(CO1 + 1*ldc);				\

  232. 	row2  += _mm512_loadu_ps(CO1 + 2*ldc);				\

  233. 	row3 += _mm512_loadu_ps(CO1 + 3*ldc);				\

  234. 	_mm512_storeu_ps(CO1 + 0*ldc, row0);				\

  235. 	_mm512_storeu_ps(CO1 + 1*ldc, row1);				\

  236. 	_mm512_storeu_ps(CO1 + 2*ldc, row2);				\

  237. 	_mm512_storeu_ps(CO1 + 3*ldc, row3);				\

  238. 	row0b  += _mm512_loadu_ps(CO1 + 0*ldc + 16);			\

  239. 	row1b  += _mm512_loadu_ps(CO1 + 1*ldc + 16);			\

  240. 	row2b  += _mm512_loadu_ps(CO1 + 2*ldc + 16);			\

  241. 	row3b += _mm512_loadu_ps(CO1 + 3*ldc + 16);			\

  242. 	_mm512_storeu_ps(CO1 + 0*ldc + 16, row0b);			\

  243. 	_mm512_storeu_ps(CO1 + 1*ldc + 16, row1b);			\

  244. 	_mm512_storeu_ps(CO1 + 2*ldc + 16, row2b);			\

  245. 	_mm512_storeu_ps(CO1 + 3*ldc + 16, row3b);			\

  246. 	row0c  += _mm512_loadu_ps(CO1 + 0*ldc + 32);			\

  247. 	row1c  += _mm512_loadu_ps(CO1 + 1*ldc + 32);			\

  248. 	row2c  += _mm512_loadu_ps(CO1 + 2*ldc + 32);			\

  249. 	row3c  += _mm512_loadu_ps(CO1 + 3*ldc + 32);			\

  250. 	_mm512_storeu_ps(CO1 + 0*ldc + 32, row0c);			\

  251. 	_mm512_storeu_ps(CO1 + 1*ldc + 32, row1c);			\

  252. 	_mm512_storeu_ps(CO1 + 2*ldc + 32, row2c);			\

  253. 	_mm512_storeu_ps(CO1 + 3*ldc + 32, row3c);		

  254. 

  255. 

  256. #define INIT32x4()	\

  257. 	row0 = _mm512_setzero_ps();					\

  258. 	row1 = _mm512_setzero_ps();					\

  259. 	row2 = _mm512_setzero_ps();					\

  260. 	row3 = _mm512_setzero_ps();					\

  261. 	row0b = _mm512_setzero_ps();					\

  262. 	row1b = _mm512_setzero_ps();					\

  263. 	row2b = _mm512_setzero_ps();					\

  264. 	row3b = _mm512_setzero_ps();					\

  265. 

  266. #define KERNEL32x4_SUB() 						\

  267. 	zmm0   = _mm512_loadu_ps(AO);					\

  268. 	zmm1   = _mm512_loadu_ps(A1);					\

  269. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  270. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+1));		\

  271. 	row0  += zmm0 * zmm2;						\

  272. 	row1  += zmm0 * zmm3;						\

  273. 	row0b += zmm1 * zmm2;						\

  274. 	row1b += zmm1 * zmm3;						\

  275. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO+2));		\

  276. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+3));		\

  277. 	row2  += zmm0 * zmm2;						\

  278. 	row3  += zmm0 * zmm3;						\

  279. 	row2b += zmm1 * zmm2;						\

  280. 	row3b += zmm1 * zmm3;						\

  281. 	BO += 4;							\

  282. 	AO += 16;							\

  283. 	A1 += 16;

  284. 

  285. 

  286. #define SAVE32x4(ALPHA)							\

  287. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  288. 	row0  *= zmm0;							\

  289. 	row1  *= zmm0;							\

  290. 	row2  *= zmm0;							\

  291. 	row3 *= zmm0;							\

  292. 	row0b *= zmm0;							\

  293. 	row1b *= zmm0;							\

  294. 	row2b *= zmm0;							\

  295. 	row3b *= zmm0;							\

  296. 	row0  += _mm512_loadu_ps(CO1 + 0*ldc);				\

  297. 	row1  += _mm512_loadu_ps(CO1 + 1*ldc);				\

  298. 	row2  += _mm512_loadu_ps(CO1 + 2*ldc);				\

  299. 	row3 += _mm512_loadu_ps(CO1 + 3*ldc);				\

  300. 	_mm512_storeu_ps(CO1 + 0*ldc, row0);				\

  301. 	_mm512_storeu_ps(CO1 + 1*ldc, row1);				\

  302. 	_mm512_storeu_ps(CO1 + 2*ldc, row2);				\

  303. 	_mm512_storeu_ps(CO1 + 3*ldc, row3);				\

  304. 	row0b  += _mm512_loadu_ps(CO1 + 0*ldc + 16);			\

  305. 	row1b  += _mm512_loadu_ps(CO1 + 1*ldc + 16);			\

  306. 	row2b  += _mm512_loadu_ps(CO1 + 2*ldc + 16);			\

  307. 	row3b += _mm512_loadu_ps(CO1 + 3*ldc + 16);			\

  308. 	_mm512_storeu_ps(CO1 + 0*ldc + 16, row0b);			\

  309. 	_mm512_storeu_ps(CO1 + 1*ldc + 16, row1b);			\

  310. 	_mm512_storeu_ps(CO1 + 2*ldc + 16, row2b);			\

  311. 	_mm512_storeu_ps(CO1 + 3*ldc + 16, row3b);		

  312. 

  313. 

  314. 

  315. #define INIT16x4()	\

  316. 	row0 = _mm512_setzero_ps();					\

  317. 	row1 = _mm512_setzero_ps();					\

  318. 	row2 = _mm512_setzero_ps();					\

  319. 	row3 = _mm512_setzero_ps();					\

  320. 

  321. #define KERNEL16x4_SUB() 						\

  322. 	zmm0   = _mm512_loadu_ps(AO);					\

  323. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  324. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+1));		\

  325. 	row0  += zmm0 * zmm2;						\

  326. 	row1  += zmm0 * zmm3;						\

  327. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO+2));		\

  328. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO+3));		\

  329. 	row2  += zmm0 * zmm2;						\

  330. 	row3 += zmm0 * zmm3;						\

  331. 	BO += 4;							\

  332. 	AO += 16;

  333. 

  334. 

  335. #define SAVE16x4(ALPHA)							\

  336. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  337. 	row0  *= zmm0;							\

  338. 	row1  *= zmm0;							\

  339. 	row2  *= zmm0;							\

  340. 	row3  *= zmm0;							\

  341. 	row0  += _mm512_loadu_ps(CO1 + 0 * ldc);			\

  342. 	row1  += _mm512_loadu_ps(CO1 + 1 * ldc);			\

  343. 	row2  += _mm512_loadu_ps(CO1 + 2 * ldc);			\

  344. 	row3  += _mm512_loadu_ps(CO1 + 3 * ldc);			\

  345. 	_mm512_storeu_ps(CO1 + 0 * ldc, row0);				\

  346. 	_mm512_storeu_ps(CO1 + 1 * ldc, row1);				\

  347. 	_mm512_storeu_ps(CO1 + 2 * ldc, row2);				\

  348. 	_mm512_storeu_ps(CO1 + 3 * ldc, row3);			

  349. 

  350. 

  351. 

  352. /*******************************************************************************************/

  353. 

  354. #define INIT8x4()							\

  355. 	ymm4 = _mm256_setzero_ps();					\

  356. 	ymm6 = _mm256_setzero_ps();					\

  357. 	ymm8 = _mm256_setzero_ps();					\

  358. 	ymm10 = _mm256_setzero_ps();					\

  359. 

  360. #define KERNEL8x4_SUB() 						\

  361. 	ymm0   = _mm256_loadu_ps(AO);					\

  362. 	ymm2   =  _mm256_broadcastss_ps(_mm_load_ss(BO + 0));		\

  363. 	ymm3   =  _mm256_broadcastss_ps(_mm_load_ss(BO + 1));		\

  364. 	ymm4  += ymm0 * ymm2;						\

  365. 	ymm6  += ymm0 * ymm3;						\

  366. 	ymm2   =  _mm256_broadcastss_ps(_mm_load_ss(BO + 2));		\

  367. 	ymm3   =  _mm256_broadcastss_ps(_mm_load_ss(BO + 3));		\

  368. 	ymm8  += ymm0 * ymm2;						\

  369. 	ymm10 += ymm0 * ymm3;						\

  370. 	BO  += 4;							\

  371. 	AO  += 8;

  372. 

  373. 

  374. #define SAVE8x4(ALPHA)							\

  375. 	ymm0   = _mm256_set1_ps(ALPHA);					\

  376. 	ymm4  *= ymm0;							\

  377. 	ymm6  *= ymm0;							\

  378. 	ymm8  *= ymm0;							\

  379. 	ymm10 *= ymm0;							\

  380. 	ymm4  += _mm256_loadu_ps(CO1 + 0 * ldc);			\

  381. 	ymm6  += _mm256_loadu_ps(CO1 + 1 * ldc);			\

  382. 	ymm8  += _mm256_loadu_ps(CO1 + 2 * ldc);			\

  383. 	ymm10 += _mm256_loadu_ps(CO1 + 3 * ldc);			\

  384. 	_mm256_storeu_ps(CO1 + 0 * ldc, ymm4);				\

  385. 	_mm256_storeu_ps(CO1 + 1 * ldc, ymm6);				\

  386. 	_mm256_storeu_ps(CO1 + 2 * ldc, ymm8);				\

  387. 	_mm256_storeu_ps(CO1 + 3 * ldc, ymm10);				\

  388. 

  389. 

  390. 

  391. /*******************************************************************************************/

  392. 

  393. #define INIT4x4()							\

  394. 	row0 = _mm_setzero_ps();					\

  395. 	row1 = _mm_setzero_ps();					\

  396. 	row2 = _mm_setzero_ps();					\

  397. 	row3 = _mm_setzero_ps();					\

  398. 

  399. 

  400. #define KERNEL4x4_SUB() 						\

  401. 	xmm0   = _mm_loadu_ps(AO);					\

  402. 	xmm2   =  _mm_broadcastss_ps(_mm_load_ss(BO + 0));		\

  403. 	xmm3   =  _mm_broadcastss_ps(_mm_load_ss(BO + 1));		\

  404. 	row0  += xmm0 * xmm2;						\

  405. 	row1  += xmm0 * xmm3;						\

  406. 	xmm2   =  _mm_broadcastss_ps(_mm_load_ss(BO + 2));		\

  407. 	xmm3   =  _mm_broadcastss_ps(_mm_load_ss(BO + 3));		\

  408. 	row2  += xmm0 * xmm2;						\

  409. 	row3  += xmm0 * xmm3;						\

  410. 	BO  += 4;							\

  411. 	AO  += 4;

  412. 

  413. 

  414. #define SAVE4x4(ALPHA)							\

  415. 	xmm0   = _mm_set1_ps(ALPHA);					\

  416. 	row0  *= xmm0;							\

  417. 	row1  *= xmm0;							\

  418. 	row2  *= xmm0;							\

  419. 	row3  *= xmm0;							\

  420. 	row0  += _mm_loadu_ps(CO1 + 0 * ldc);				\

  421. 	row1  += _mm_loadu_ps(CO1 + 1 * ldc);				\

  422. 	row2  += _mm_loadu_ps(CO1 + 2 * ldc);				\

  423. 	row3  += _mm_loadu_ps(CO1 + 3 * ldc);				\

  424. 	_mm_storeu_ps(CO1 + 0 * ldc, row0);				\

  425. 	_mm_storeu_ps(CO1 + 1 * ldc, row1);				\

  426. 	_mm_storeu_ps(CO1 + 2 * ldc, row2);				\

  427. 	_mm_storeu_ps(CO1 + 3 * ldc, row3);				\

  428. 

  429. 

  430. /*******************************************************************************************/

  431. 

  432. #define INIT2x4() 	\

  433. 	row0 = 0; row0b = 0; row1 = 0; row1b = 0; 			\

  434. 	row2 = 0; row2b = 0; row3 = 0; row3b = 0;

  435. 

  436. #define KERNEL2x4_SUB()							\

  437. 	xmm0  = *(AO);							\

  438. 	xmm1  = *(AO + 1);						\

  439. 	xmm2  = *(BO + 0);						\

  440. 	xmm3  = *(BO + 1);						\

  441. 	row0 += xmm0 * xmm2;						\

  442. 	row0b += xmm1 * xmm2;						\

  443. 	row1 += xmm0 * xmm3;						\

  444. 	row1b += xmm1 * xmm3;						\

  445. 	xmm2 = *(BO + 2);						\

  446. 	xmm3 = *(BO + 3);						\

  447. 	row2 += xmm0 * xmm2;						\

  448. 	row2b += xmm1 * xmm2;						\

  449. 	row3 += xmm0 * xmm3;						\

  450. 	row3b += xmm1 * xmm3;						\

  451. 	BO += 4;							\

  452. 	AO += 2;

  453. 

  454. 

  455. #define SAVE2x4(ALPHA)							\

  456. 	xmm0   = ALPHA;							\

  457. 	row0  *= xmm0;							\

  458. 	row0b *= xmm0;							\

  459. 	row1  *= xmm0;							\

  460. 	row1b *= xmm0;							\

  461. 	row2  *= xmm0;							\

  462. 	row2b *= xmm0;							\

  463. 	row3  *= xmm0;							\

  464. 	row3b *= xmm0;							\

  465. 	*(CO1 + 0 * ldc + 0) += row0;					\

  466. 	*(CO1 + 0 * ldc + 1) += row0b;					\

  467. 	*(CO1 + 1 * ldc + 0) += row1;					\

  468. 	*(CO1 + 1 * ldc + 1) += row1b;					\

  469. 	*(CO1 + 2 * ldc + 0) += row2;					\

  470. 	*(CO1 + 2 * ldc + 1) += row2b;					\

  471. 	*(CO1 + 3 * ldc + 0) += row3;					\

  472. 	*(CO1 + 3 * ldc + 1) += row3b;					\

  473. 

  474. 

  475. 

  476. /*******************************************************************************************/

  477. 

  478. #define INIT1x4() \

  479. 	row0 = 0; row1 = 0; row2 = 0; row3 = 0;

  480. #define KERNEL1x4_SUB()							\

  481. 	xmm0  = *(AO );							\

  482. 	xmm2  = *(BO + 0);						\

  483. 	xmm3  = *(BO + 1);						\

  484. 	row0 += xmm0 * xmm2;						\

  485. 	row1 += xmm0 * xmm3;						\

  486. 	xmm2   = *(BO + 2);						\

  487. 	xmm3   = *(BO + 3);						\

  488. 	row2  += xmm0 * xmm2;						\

  489. 	row3 += xmm0 * xmm3;						\

  490. 	BO += 4;							\

  491. 	AO += 1;

  492. 

  493. 

  494. #define SAVE1x4(ALPHA)							\

  495. 	xmm0   = ALPHA;							\

  496. 	row0  *= xmm0;							\

  497. 	row1  *= xmm0;							\

  498. 	row2  *= xmm0;							\

  499. 	row3  *= xmm0;							\

  500. 	*(CO1 + 0 * ldc) += row0;					\

  501. 	*(CO1 + 1 * ldc) += row1;					\

  502. 	*(CO1 + 2 * ldc) += row2;					\

  503. 	*(CO1 + 3 * ldc) += row3;					\

  504. 

  505. 

  506. 

  507. /*******************************************************************************************/

  508. 

  509. /*******************************************************************************************

  510. * 2 lines of N

  511. *******************************************************************************************/

  512. 

  513. #define INIT16x2()							\

  514. 	row0 = _mm512_setzero_ps();					\

  515. 	row1 = _mm512_setzero_ps();					\

  516. 

  517. 

  518. #define KERNEL16x2_SUB() 						\

  519. 	zmm0   = _mm512_loadu_ps(AO);					\

  520. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  521. 	zmm3   =  _mm512_broadcastss_ps(_mm_load_ss(BO + 1));		\

  522. 	row0  += zmm0 * zmm2;						\

  523. 	row1  += zmm0 * zmm3;						\

  524. 	BO += 2;							\

  525. 	AO += 16;

  526. 

  527. 

  528. #define SAVE16x2(ALPHA)							\

  529. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  530. 	row0  *= zmm0;							\

  531. 	row1  *= zmm0;							\

  532. 	row0  += _mm512_loadu_ps(CO1);					\

  533. 	row1  += _mm512_loadu_ps(CO1 + ldc);				\

  534. 	_mm512_storeu_ps(CO1      , row0);				\

  535. 	_mm512_storeu_ps(CO1 + ldc, row1);				\

  536. 

  537. 

  538. 

  539. 

  540. /*******************************************************************************************/

  541. 

  542. #define INIT8x2()	\

  543. 	ymm4 = _mm256_setzero_ps();					\

  544. 	ymm6 = _mm256_setzero_ps();					\

  545. 

  546. #define KERNEL8x2_SUB() 						\

  547. 	ymm0   = _mm256_loadu_ps(AO);					\

  548. 	ymm2   =  _mm256_broadcastss_ps(_mm_load_ss(BO));		\

  549. 	ymm3   =  _mm256_broadcastss_ps(_mm_load_ss(BO + 1));		\

  550. 	ymm4  += ymm0 * ymm2;						\

  551. 	ymm6  += ymm0 * ymm3;						\

  552. 	BO  += 2;							\

  553. 	AO  += 8;

  554. 

  555. 

  556. #define SAVE8x2(ALPHA)							\

  557. 	ymm0   = _mm256_set1_ps(ALPHA);					\

  558. 	ymm4  *= ymm0;							\

  559. 	ymm6  *= ymm0;							\

  560. 	ymm4  += _mm256_loadu_ps(CO1);					\

  561. 	ymm6  += _mm256_loadu_ps(CO1 + ldc);				\

  562. 	_mm256_storeu_ps(CO1      , ymm4);				\

  563. 	_mm256_storeu_ps(CO1 + ldc, ymm6);				\

  564. 

  565. 

  566. 

  567. /*******************************************************************************************/

  568. 

  569. #define INIT4x2()	\

  570. 	row0 = _mm_setzero_ps(); 					\

  571. 	row1 = _mm_setzero_ps(); 					\

  572. 

  573. #define KERNEL4x2_SUB() 						\

  574. 	xmm0   = _mm_loadu_ps(AO);					\

  575. 	xmm2   =  _mm_broadcastss_ps(_mm_load_ss(BO));			\

  576. 	xmm3   =  _mm_broadcastss_ps(_mm_load_ss(BO + 1));		\

  577. 	row0  += xmm0 * xmm2;						\

  578. 	row1  += xmm0 * xmm3;						\

  579. 	BO  += 2;							\

  580. 	AO  += 4;

  581. 

  582. 

  583. #define SAVE4x2(ALPHA)							\

  584. 	xmm0   = _mm_set1_ps(ALPHA);					\

  585. 	row0  *= xmm0;							\

  586. 	row1  *= xmm0;							\

  587. 	row0  += _mm_loadu_ps(CO1);					\

  588. 	row1  += _mm_loadu_ps(CO1 + ldc);				\

  589. 	_mm_storeu_ps(CO1      , row0);					\

  590. 	_mm_storeu_ps(CO1 + ldc, row1);					\

  591. 

  592. 

  593. 

  594. /*******************************************************************************************/

  595. 

  596. 

  597. #define INIT2x2() 	\

  598. 	row0 = 0; row0b = 0; row1 = 0; row1b = 0; 			\

  599. 

  600. #define KERNEL2x2_SUB()							\

  601. 	xmm0  = *(AO + 0);						\

  602. 	xmm1  = *(AO + 1);						\

  603. 	xmm2  = *(BO + 0);						\

  604. 	xmm3  = *(BO + 1);						\

  605. 	row0 += xmm0 * xmm2;						\

  606. 	row0b += xmm1 * xmm2;						\

  607. 	row1 += xmm0 * xmm3;						\

  608. 	row1b += xmm1 * xmm3;						\

  609. 	BO += 2;							\

  610. 	AO += 2;							\

  611. 

  612. 

  613. #define SAVE2x2(ALPHA)							\

  614. 	xmm0   = ALPHA;							\

  615. 	row0  *= xmm0;							\

  616. 	row0b  *= xmm0;							\

  617. 	row1  *= xmm0;							\

  618. 	row1b  *= xmm0;							\

  619. 	*(CO1         ) += row0;					\

  620. 	*(CO1 +1      ) += row0b;					\

  621. 	*(CO1 + ldc   ) += row1;					\

  622. 	*(CO1 + ldc +1) += row1b;					\

  623. 

  624. 

  625. /*******************************************************************************************/

  626. 

  627. #define INIT1x2()	\

  628. 	row0 = 0; row1 = 0;

  629. 

  630. #define KERNEL1x2_SUB()							\

  631. 	xmm0  = *(AO);							\

  632. 	xmm2  = *(BO + 0);						\

  633. 	xmm3  = *(BO + 1);						\

  634. 	row0 += xmm0 * xmm2;						\

  635. 	row1 += xmm0 * xmm3;						\

  636. 	BO += 2;							\

  637. 	AO += 1;

  638. 

  639. 

  640. #define SAVE1x2(ALPHA)							\

  641. 	xmm0   = ALPHA;							\

  642. 	row0  *= xmm0;							\

  643. 	row1  *= xmm0;							\

  644. 	*(CO1         ) += row0;					\

  645. 	*(CO1 + ldc   ) += row1;					\

  646. 

  647. 

  648. /*******************************************************************************************/

  649. 

  650. /*******************************************************************************************

  651. * 1 line of N

  652. *******************************************************************************************/

  653. 

  654. #define INIT16x1() \

  655. 	row0 = _mm512_setzero_ps();				\

  656. 

  657. #define KERNEL16x1_SUB() 						\

  658. 	zmm0   = _mm512_loadu_ps(AO);			\

  659. 	zmm2   =  _mm512_broadcastss_ps(_mm_load_ss(BO));		\

  660. 	row0  += zmm0 * zmm2;						\

  661. 	BO += 1;							\

  662. 	AO += 16;

  663. 

  664. 

  665. #define SAVE16x1(ALPHA)							\

  666. 	zmm0   = _mm512_set1_ps(ALPHA);					\

  667. 	row0  *= zmm0;							\

  668. 	row0  += _mm512_loadu_ps(CO1);					\

  669. 	_mm512_storeu_ps(CO1      , row0);				\

  670. 

  671. 

  672. /*******************************************************************************************/

  673. 

  674. #define INIT8x1()							\

  675. 	ymm4 = _mm256_setzero_ps();					

  676. 

  677. #define KERNEL8x1_SUB() 						\

  678. 	ymm0   = _mm256_loadu_ps(AO);					\

  679. 	ymm2   =  _mm256_broadcastss_ps(_mm_load_ss(BO));		\

  680. 	ymm4  += ymm0 * ymm2;						\

  681. 	BO  += 1;							\

  682. 	AO  += 8;

  683. 

  684. 

  685. #define SAVE8x1(ALPHA)							\

  686. 	ymm0   = _mm256_set1_ps(ALPHA);					\

  687. 	ymm4  *= ymm0;							\

  688. 	ymm4  += _mm256_loadu_ps(CO1);					\

  689. 	_mm256_storeu_ps(CO1      , ymm4);				\

  690. 

  691. 

  692. /*******************************************************************************************/

  693. 

  694. #define INIT4x1()							\

  695. 	row0 = _mm_setzero_ps();					\

  696. 

  697. #define KERNEL4x1_SUB() 						\

  698. 	xmm0   = _mm_loadu_ps(AO);					\

  699. 	xmm2   =  _mm_broadcastss_ps(_mm_load_ss(BO));			\

  700. 	row0  += xmm0 * xmm2;						\

  701. 	BO    += 1;							\

  702. 	AO    += 4;

  703. 

  704. 

  705. #define SAVE4x1(ALPHA)							\

  706. 	xmm0   = _mm_set1_ps(ALPHA);					\

  707. 	row0  *= xmm0;							\

  708. 	row0  += _mm_loadu_ps(CO1);					\

  709. 	_mm_storeu_ps(CO1      , row0);					\

  710. 

  711. 

  712. 

  713. /*******************************************************************************************/

  714. 

  715. #define INIT2x1()							\

  716. 	row0 = 0; row0b = 0;

  717. 

  718. #define KERNEL2x1_SUB()							\

  719. 	xmm0  = *(AO + 0);						\

  720. 	xmm1  = *(AO + 1);						\

  721. 	xmm2  = *(BO);							\

  722. 	row0 += xmm0 * xmm2;						\

  723. 	row0b += xmm1 * xmm2;						\

  724. 	BO += 1;							\

  725. 	AO += 2;

  726. 

  727. 

  728. #define SAVE2x1(ALPHA)							\

  729. 	xmm0   = ALPHA;							\

  730. 	row0  *= xmm0;							\

  731. 	row0b  *= xmm0;							\

  732. 	*(CO1         ) += row0;					\

  733. 	*(CO1 +1      ) += row0b;					\

  734. 

  735. 

  736. /*******************************************************************************************/

  737. 

  738. #define INIT1x1()							\

  739. 	row0 = 0;

  740. 

  741. #define KERNEL1x1_SUB()							\

  742. 	xmm0  = *(AO);							\

  743. 	xmm2  = *(BO);							\

  744. 	row0 += xmm0 * xmm2;						\

  745. 	BO += 1;							\

  746. 	AO += 1;

  747. 

  748. 

  749. #define SAVE1x1(ALPHA)							\

  750. 	xmm0   = ALPHA;							\

  751. 	row0  *= xmm0;							\

  752. 	*(CO1         ) += row0;					\

  753. 

  754. 

  755. /*******************************************************************************************/

  756. 

  757. 

  758. /*************************************************************************************

  759. * GEMM Kernel

  760. *************************************************************************************/

  761. 

  762. int __attribute__ ((noinline))

  763. CNAME(BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float * __restrict A, float * __restrict B, float * __restrict C, BLASLONG ldc)

  764. {

  765. 	unsigned long M = m, N = n, K = k;

  766. 	if (M == 0)

  767. 		return 0;

  768. 	if (N == 0)

  769. 		return 0;

  770. 	if (K == 0)

  771. 		return 0;

  772. 

  773. 

  774. 	while (N >= 4) {

  775. 		float *CO1;

  776. 		float *AO;

  777. 		int i;

  778. 		// L8_10

  779. 		CO1 = C;

  780. 		C += 4 * ldc;

  781. 

  782. 		AO = A;

  783. 

  784. 		i = m;

  785. 		while (i >= 64) {

  786. 			float *BO;

  787. 			float *A1, *A2, *A3;

  788. 			// L8_11

  789. 			__m512 zmm0, zmm1, zmm2, zmm3, row0, zmm5, row1, zmm7, row2, row3, row0b, row1b, row2b, row3b, row0c, row1c, row2c, row3c, row0d, row1d, row2d, row3d;

  790. 			BO = B;

  791. 			int kloop = K;

  792. 

  793. 			A1 = AO + 16 * K;

  794. 			A2 = A1 + 16 * K;

  795. 			A3 = A2 + 16 * K;

  796. 	

  797. 			INIT64x4()

  798. 

  799. 			while (kloop > 0) {

  800. 				// L12_17

  801. 				KERNEL64x4_SUB()

  802. 				kloop--;

  803. 			}

  804. 			// L8_19

  805. 			SAVE64x4(alpha)

  806. 			CO1 += 64;

  807. 			AO += 48 * K;

  808. 	

  809. 			i -= 64;

  810. 		}

  811. 		while (i >= 32) {

  812. 			float *BO;

  813. 			float *A1;

  814. 			// L8_11

  815. 			__m512 zmm0, zmm1, zmm2, zmm3, row0, row1, row2, row3, row0b, row1b, row2b, row3b;

  816. 			BO = B;

  817. 			int kloop = K;

  818. 

  819. 			A1 = AO + 16 * K;

  820. 	

  821. 			INIT32x4()

  822. 

  823. 			while (kloop > 0) {

  824. 				// L12_17

  825. 				KERNEL32x4_SUB()

  826. 				kloop--;

  827. 			}

  828. 			// L8_19

  829. 			SAVE32x4(alpha)

  830. 			CO1 += 32;

  831. 			AO += 16 * K;

  832. 	

  833. 			i -= 32;

  834. 		}

  835. 		while (i >= 16) {

  836. 			float *BO;

  837. 			// L8_11

  838. 			__m512 zmm0, zmm2, zmm3, row0, row1, row2, row3;

  839. 			BO = B;

  840. 			int kloop = K;

  841. 	

  842. 			INIT16x4()

  843. 

  844. 			while (kloop > 0) {

  845. 				// L12_17

  846. 				KERNEL16x4_SUB()

  847. 				kloop--;

  848. 			}

  849. 			// L8_19

  850. 			SAVE16x4(alpha)

  851. 			CO1 += 16;

  852. 	

  853. 			i -= 16;

  854. 		}

  855. 		while (i >= 8) {

  856. 			float *BO;

  857. 			// L8_11

  858. 			__m256 ymm0, ymm2, ymm3, ymm4, ymm6,ymm8,ymm10;

  859. 			BO = B;

  860. 			int kloop = K;

  861. 	

  862. 			INIT8x4()

  863. 

  864. 			while (kloop > 0) {

  865. 				// L12_17

  866. 				KERNEL8x4_SUB()

  867. 				kloop--;

  868. 			}

  869. 			// L8_19

  870. 			SAVE8x4(alpha)

  871. 			CO1 += 8;

  872. 	

  873. 			i -= 8;

  874. 		}

  875. 		while (i >= 4) {

  876. 			// L8_11

  877. 			float *BO;

  878. 			__m128 xmm0, xmm2, xmm3, row0, row1, row2, row3;

  879. 			BO = B;

  880. 			int kloop = K;

  881. 

  882. 			INIT4x4()

  883. 			// L8_16

  884. 			while (kloop > 0) {

  885. 				// L12_17

  886. 				KERNEL4x4_SUB()

  887. 				kloop--;

  888. 			}

  889. 			// L8_19

  890. 			SAVE4x4(alpha)

  891. 			CO1 += 4;

  892. 

  893. 			i -= 4;

  894. 		}

  895. 

  896. /**************************************************************************

  897. * Rest of M 

  898. ***************************************************************************/

  899. 

  900. 		while (i >= 2) {

  901. 			float *BO;

  902. 			float xmm0, xmm1, xmm2, xmm3, row0, row0b, row1, row1b, row2, row2b, row3, row3b;

  903. 			BO = B;

  904. 

  905. 			INIT2x4()

  906. 			int kloop = K;

  907. 			

  908. 			while (kloop > 0) {

  909. 				KERNEL2x4_SUB()

  910. 				kloop--;

  911. 			}

  912. 			SAVE2x4(alpha)

  913. 			CO1 += 2;

  914. 			i -= 2;

  915. 		}

  916. 			// L13_40

  917. 		while (i >= 1) {

  918. 			float *BO;

  919. 			float xmm0, xmm2, xmm3, row0, row1, row2, row3;

  920. 			int kloop = K;

  921. 			BO = B;

  922. 			INIT1x4()

  923. 				

  924. 			while (kloop > 0) {

  925. 				KERNEL1x4_SUB()

  926. 				kloop--;

  927. 			}

  928. 			SAVE1x4(alpha)

  929. 			CO1 += 1;

  930. 			i -= 1;

  931. 		}

  932. 			

  933. 		B += K * 4;

  934. 		N -= 4;

  935. 	}

  936. 

  937. /**************************************************************************************************/

  938. 

  939. 		// L8_0

  940. 	while (N >= 2) {

  941. 		float *CO1;

  942. 		float *AO;

  943. 		int i;

  944. 		// L8_10

  945. 		CO1 = C;

  946. 		C += 2 * ldc;

  947. 

  948. 		AO = A;

  949. 

  950. 		i = m;

  951. 		while (i >= 16) {

  952. 			float *BO;

  953. 

  954. 			// L8_11

  955. 			__m512 zmm0, zmm2, zmm3, row0, row1;

  956. 			BO = B;

  957. 			int kloop = K;

  958. 	

  959. 			INIT16x2()

  960. 

  961. 			while (kloop > 0) {

  962. 				// L12_17

  963. 				KERNEL16x2_SUB()

  964. 				kloop--;

  965. 			}

  966. 			// L8_19

  967. 			SAVE16x2(alpha)

  968. 			CO1 += 16;

  969. 	

  970. 			i -= 16;

  971. 		}

  972. 		while (i >= 8) {

  973. 			float *BO;

  974. 			__m256 ymm0, ymm2, ymm3, ymm4, ymm6;

  975. 			// L8_11

  976. 			BO = B;

  977. 			int kloop = K;

  978. 

  979. 			INIT8x2()

  980. 

  981. 			// L8_16

  982. 			while (kloop > 0) {

  983. 				// L12_17

  984. 				KERNEL8x2_SUB()

  985. 				kloop--;

  986. 			}

  987. 			// L8_19

  988. 			SAVE8x2(alpha)

  989. 			CO1 += 8;

  990. 

  991. 			i-=8;

  992. 		}

  993. 

  994. 		while (i >= 4) {

  995. 			float *BO;

  996. 			__m128 xmm0, xmm2, xmm3, row0, row1;

  997. 			// L8_11

  998. 			BO = B;

  999. 			int kloop = K;

  1000. 	

  1001. 			INIT4x2()

  1002. 

  1003. 			// L8_16

  1004. 			while (kloop > 0) {

  1005. 				// L12_17

  1006. 				KERNEL4x2_SUB()

  1007. 				kloop--;

  1008. 			}

  1009. 			// L8_19

  1010. 			SAVE4x2(alpha)

  1011. 			CO1 += 4;

  1012. 	

  1013. 			i-=4;

  1014. 		}

  1015. 

  1016. /**************************************************************************

  1017. * Rest of M 

  1018. ***************************************************************************/

  1019. 

  1020. 		while (i >= 2) {

  1021. 			float *BO;

  1022. 			float xmm0, xmm1, xmm2, xmm3, row0, row0b, row1, row1b;

  1023. 			int kloop = K;

  1024. 			BO = B;

  1025. 

  1026. 			INIT2x2()

  1027. 				

  1028. 			while (kloop > 0) {

  1029. 				KERNEL2x2_SUB()

  1030. 				kloop--;

  1031. 			}

  1032. 			SAVE2x2(alpha)

  1033. 			CO1 += 2;

  1034. 			i -= 2;

  1035. 		}

  1036. 			// L13_40

  1037. 		while (i >= 1) {

  1038. 			float *BO;

  1039. 			float xmm0, xmm2, xmm3, row0, row1;

  1040. 			int kloop = K;

  1041. 			BO = B;

  1042. 

  1043. 			INIT1x2()

  1044. 					

  1045. 			while (kloop > 0) {

  1046. 				KERNEL1x2_SUB()

  1047. 				kloop--;

  1048. 			}

  1049. 			SAVE1x2(alpha)

  1050. 			CO1 += 1;

  1051. 			i -= 1;

  1052. 		}

  1053. 			

  1054. 		B += K * 2;

  1055. 		N -= 2;

  1056. 	}

  1057. 

  1058. 		// L8_0

  1059. 	while (N >= 1) {

  1060. 		// L8_10

  1061. 		float *CO1;

  1062. 		float *AO;

  1063. 		int i;

  1064. 

  1065. 		CO1 = C;

  1066. 		C += ldc;

  1067. 

  1068. 		AO = A;

  1069. 

  1070. 		i = m;

  1071. 		while (i >= 16) {

  1072. 			float *BO;

  1073. 			__m512 zmm0, zmm2, row0;

  1074. 			// L8_11

  1075. 			BO = B;

  1076. 			int kloop = K;

  1077. 

  1078. 			INIT16x1()

  1079. 			// L8_16

  1080. 			while (kloop > 0) {

  1081. 				// L12_17

  1082. 				KERNEL16x1_SUB()

  1083. 				kloop--;

  1084. 			}

  1085. 			// L8_19

  1086. 			SAVE16x1(alpha)

  1087. 			CO1 += 16;

  1088. 

  1089. 			i-= 16;

  1090. 		}

  1091. 		while (i >= 8) {

  1092. 			float *BO;

  1093. 			__m256 ymm0, ymm2, ymm4;

  1094. 			// L8_11

  1095. 			BO = B;

  1096. 			int kloop = K;

  1097. 

  1098. 			INIT8x1()

  1099. 			// L8_16

  1100. 			while (kloop > 0) {

  1101. 				// L12_17

  1102. 				KERNEL8x1_SUB()

  1103. 				kloop--;

  1104. 			}

  1105. 			// L8_19

  1106. 			SAVE8x1(alpha)

  1107. 			CO1 += 8;

  1108. 

  1109. 			i-= 8;

  1110. 		}

  1111. 		while (i >= 4) {

  1112. 			float *BO;

  1113. 			__m128 xmm0, xmm2, row0;

  1114. 			// L8_11

  1115. 			BO = B;

  1116. 			int kloop = K;

  1117. 

  1118. 			INIT4x1()

  1119. 			// L8_16

  1120. 			while (kloop > 0) {

  1121. 				// L12_17

  1122. 				KERNEL4x1_SUB()

  1123. 				kloop--;

  1124. 			}

  1125. 			// L8_19

  1126. 			SAVE4x1(alpha)

  1127. 			CO1 += 4;

  1128. 

  1129. 			i-= 4;

  1130. 		}

  1131. 

  1132. /**************************************************************************

  1133. * Rest of M 

  1134. ***************************************************************************/

  1135. 

  1136. 		while (i >= 2) {

  1137. 			float *BO;

  1138. 			float xmm0, xmm1, xmm2, row0, row0b;

  1139. 			int kloop = K;

  1140. 			BO = B;

  1141. 

  1142. 			INIT2x1()

  1143. 				

  1144. 			while (kloop > 0) {

  1145. 				KERNEL2x1_SUB()

  1146. 				kloop--;

  1147. 			}

  1148. 			SAVE2x1(alpha)

  1149. 			CO1 += 2;

  1150. 			i -= 2;

  1151. 		}

  1152. 				// L13_40

  1153. 		while (i >= 1) {

  1154. 			float *BO;

  1155. 			float xmm0, xmm2, row0;

  1156. 			int kloop = K;

  1157. 

  1158. 			BO = B;

  1159. 			INIT1x1()

  1160. 				

  1161. 

  1162. 			while (kloop > 0) {

  1163. 				KERNEL1x1_SUB()

  1164. 				kloop--;

  1165. 			}

  1166. 			SAVE1x1(alpha)

  1167. 			CO1 += 1;

  1168. 			i -= 1;

  1169. 		}

  1170. 			

  1171. 		B += K * 1;

  1172. 		N -= 1;

  1173. 	}

  1174. 

  1175. 

  1176. 	return 0;

  1177. }

  1178. 

  1179. 

  1180. /*

  1181.  * "Direct sgemm" code. This code operates directly on the inputs and outputs

  1182.  * of the sgemm call, avoiding the copies, memory realignments and threading,

  1183.  * and only supports alpha = 1 and beta = 0.

  1184.  * This is a common case and provides value for relatively small matrixes.

  1185.  * For larger matrixes the "regular" sgemm code is superior, there the cost of

  1186.  * copying/shuffling the B matrix really pays off.

  1187.  */

  1188. 

  1189. 

  1190. 

  1191. #define DECLARE_RESULT_512(N,M) __m512 result##N##M = _mm512_setzero_ps()

  1192. #define BROADCAST_LOAD_A_512(N,M) __m512 Aval##M = _mm512_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  1193. #define LOAD_B_512(N,M)  __m512 Bval##N = _mm512_loadu_ps(&B[strideB * k + j + (N*16)])

  1194. #define MATMUL_512(N,M)  result##N##M = _mm512_fmadd_ps(Aval##M, Bval##N , result##N##M)

  1195. #define STORE_512(N,M) _mm512_storeu_ps(&R[(i+M) * strideR + j+(N*16)], result##N##M)

  1196. 

  1197. 

  1198. #define DECLARE_RESULT_256(N,M) __m256 result##N##M = _mm256_setzero_ps()

  1199. #define BROADCAST_LOAD_A_256(N,M) __m256 Aval##M = _mm256_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  1200. #define LOAD_B_256(N,M)  __m256 Bval##N = _mm256_loadu_ps(&B[strideB * k + j + (N*8)])

  1201. #define MATMUL_256(N,M)  result##N##M = _mm256_fmadd_ps(Aval##M, Bval##N , result##N##M)

  1202. #define STORE_256(N,M) _mm256_storeu_ps(&R[(i+M) * strideR + j+(N*8)], result##N##M)

  1203. 

  1204. #define DECLARE_RESULT_128(N,M) __m128 result##N##M = _mm_setzero_ps()

  1205. #define BROADCAST_LOAD_A_128(N,M) __m128 Aval##M = _mm_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  1206. #define LOAD_B_128(N,M)  __m128 Bval##N = _mm_loadu_ps(&B[strideB * k + j + (N*4)])

  1207. #define MATMUL_128(N,M)  result##N##M = _mm_fmadd_ps(Aval##M, Bval##N , result##N##M)

  1208. #define STORE_128(N,M) _mm_storeu_ps(&R[(i+M) * strideR + j+(N*4)], result##N##M)

  1209. 

  1210. #define DECLARE_RESULT_SCALAR(N,M) float result##N##M = 0;

  1211. #define BROADCAST_LOAD_A_SCALAR(N,M) float Aval##M = A[k + strideA * (i + M)];

  1212. #define LOAD_B_SCALAR(N,M)  float Bval##N  = B[k * strideB + j + N];

  1213. #define MATMUL_SCALAR(N,M) result##N##M +=  Aval##M * Bval##N;

  1214. #define STORE_SCALAR(N,M)  R[(i+M) * strideR + j + N] = result##N##M;

  1215. 

  1216. int sgemm_kernel_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K)

  1217. {

  1218. 	int mnk = M * N * K;

  1219. 	/* large matrixes -> not performant */

  1220. 	if (mnk >= 28 * 512 * 512)

  1221. 		return 0;

  1222. 

  1223. 	/*

  1224. 	 * if the B matrix is not a nice multiple if 4 we get many unaligned accesses,

  1225. 	 * and the regular sgemm copy/realignment of data pays off much quicker

  1226. 	 */

  1227. 	if ((N & 3) != 0 && (mnk >= 8 * 512 * 512))

  1228. 		return 0;

  1229. 

  1230. #ifdef SMP

  1231. 	/* if we can run multithreaded, the threading changes the based threshold */

  1232. 	if (mnk > 2 * 350 * 512 && num_cpu_avail(3)> 1)

  1233. 		return 0;

  1234. #endif

  1235. 

  1236. 	return 1;

  1237. }

  1238. 

  1239. 

  1240. 

  1241. void sgemm_kernel_direct (BLASLONG M, BLASLONG N, BLASLONG K, float * __restrict A, BLASLONG strideA, float * __restrict B, BLASLONG strideB , float * __restrict R, BLASLONG strideR)

  1242. {

  1243. 	int i, j, k;

  1244. 

  1245.         int m4 = M & ~3;

  1246. 	int m2 = M & ~1;

  1247. 

  1248. 	int n64 = N & ~63;

  1249. 	int n32 = N & ~31;

  1250. 	int n16 = N & ~15;

  1251. 	int n8 = N & ~7;

  1252. 	int n4 = N & ~3;

  1253. 	int n2 = N & ~1;

  1254. 

  1255. 	i = 0;

  1256. 

  1257. 	for (i = 0; i < m4; i+=4) {

  1258. 

  1259. 		for (j = 0; j < n64; j+= 64) {

  1260. 			k = 0;

  1261. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  1262. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);    			DECLARE_RESULT_512(2, 1);    DECLARE_RESULT_512(3, 1);

  1263. 			DECLARE_RESULT_512(0, 2);    DECLARE_RESULT_512(1, 2);    			DECLARE_RESULT_512(2, 2);    DECLARE_RESULT_512(3, 2);

  1264. 			DECLARE_RESULT_512(0, 3);    DECLARE_RESULT_512(1, 3);    			DECLARE_RESULT_512(2, 3);    DECLARE_RESULT_512(3, 3);

  1265. 

  1266. 

  1267. 			for (k = 0; k < K; k++) {

  1268. 				BROADCAST_LOAD_A_512(x, 0);

  1269. 				BROADCAST_LOAD_A_512(x, 1);

  1270. 				BROADCAST_LOAD_A_512(x, 2);

  1271. 				BROADCAST_LOAD_A_512(x, 3);

  1272. 

  1273. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  1274. 

  1275. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  1276. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);			MATMUL_512(2, 1);		MATMUL_512(3, 1);

  1277. 				MATMUL_512(0, 2);		MATMUL_512(1, 2);			MATMUL_512(2, 2);		MATMUL_512(3, 2);

  1278. 				MATMUL_512(0, 3);		MATMUL_512(1, 3);			MATMUL_512(2, 3);		MATMUL_512(3, 3);

  1279. 			}

  1280. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  1281. 			STORE_512(0, 1);		STORE_512(1, 1);			STORE_512(2, 1);		STORE_512(3, 1);

  1282. 			STORE_512(0, 2);		STORE_512(1, 2);			STORE_512(2, 2);		STORE_512(3, 2);

  1283. 			STORE_512(0, 3);		STORE_512(1, 3);			STORE_512(2, 3);		STORE_512(3, 3);

  1284. 		}

  1285. 

  1286. 		for (; j < n32; j+= 32) {

  1287. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  1288. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);

  1289. 			DECLARE_RESULT_512(0, 2);    DECLARE_RESULT_512(1, 2);

  1290. 			DECLARE_RESULT_512(0, 3);    DECLARE_RESULT_512(1, 3);

  1291. 

  1292. 			for (k = 0; k < K; k++) {

  1293. 				BROADCAST_LOAD_A_512(x, 0);

  1294. 				BROADCAST_LOAD_A_512(x, 1);

  1295. 				BROADCAST_LOAD_A_512(x, 2);

  1296. 				BROADCAST_LOAD_A_512(x, 3);

  1297. 

  1298. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  1299. 

  1300. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  1301. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);

  1302. 				MATMUL_512(0, 2);		MATMUL_512(1, 2);

  1303. 				MATMUL_512(0, 3);		MATMUL_512(1, 3);

  1304. 			}

  1305. 			STORE_512(0, 0);		STORE_512(1, 0);

  1306. 			STORE_512(0, 1);		STORE_512(1, 1);

  1307. 			STORE_512(0, 2);		STORE_512(1, 2);

  1308. 			STORE_512(0, 3);		STORE_512(1, 3);

  1309. 		}

  1310. 

  1311. 		for (; j < n16; j+= 16) {

  1312. 			DECLARE_RESULT_512(0, 0);

  1313. 			DECLARE_RESULT_512(0, 1);

  1314. 			DECLARE_RESULT_512(0, 2);

  1315. 			DECLARE_RESULT_512(0, 3);

  1316. 

  1317. 		 	for (k = 0; k < K; k++) {

  1318. 				BROADCAST_LOAD_A_512(x, 0);

  1319. 				BROADCAST_LOAD_A_512(x, 1);

  1320. 				BROADCAST_LOAD_A_512(x, 2);

  1321. 				BROADCAST_LOAD_A_512(x, 3);

  1322. 

  1323. 				LOAD_B_512(0, x);

  1324. 

  1325. 				MATMUL_512(0, 0);

  1326. 				MATMUL_512(0, 1);

  1327. 				MATMUL_512(0, 2);

  1328. 				MATMUL_512(0, 3);

  1329. 			}

  1330. 			STORE_512(0, 0);

  1331. 			STORE_512(0, 1);

  1332. 			STORE_512(0, 2);

  1333. 			STORE_512(0, 3);

  1334. 		}

  1335. 

  1336. 		for (; j < n8; j+= 8) {

  1337. 			DECLARE_RESULT_256(0, 0);

  1338. 			DECLARE_RESULT_256(0, 1);

  1339. 			DECLARE_RESULT_256(0, 2);

  1340. 			DECLARE_RESULT_256(0, 3);

  1341. 

  1342. 			for (k = 0; k < K; k++) {

  1343. 				BROADCAST_LOAD_A_256(x, 0);

  1344. 				BROADCAST_LOAD_A_256(x, 1);

  1345. 				BROADCAST_LOAD_A_256(x, 2);

  1346. 				BROADCAST_LOAD_A_256(x, 3);

  1347. 

  1348. 				LOAD_B_256(0, x);

  1349. 

  1350. 				MATMUL_256(0, 0);

  1351. 				MATMUL_256(0, 1);

  1352. 				MATMUL_256(0, 2);

  1353. 				MATMUL_256(0, 3);

  1354. 			}

  1355. 			STORE_256(0, 0);

  1356. 			STORE_256(0, 1);

  1357. 			STORE_256(0, 2);

  1358. 			STORE_256(0, 3);

  1359. 		}

  1360. 

  1361. 		for (; j < n4; j+= 4) {

  1362. 			DECLARE_RESULT_128(0, 0);

  1363. 			DECLARE_RESULT_128(0, 1);

  1364. 			DECLARE_RESULT_128(0, 2);

  1365. 			DECLARE_RESULT_128(0, 3);

  1366. 

  1367. 			for (k = 0; k < K; k++) {

  1368. 				BROADCAST_LOAD_A_128(x, 0);

  1369. 				BROADCAST_LOAD_A_128(x, 1);

  1370. 				BROADCAST_LOAD_A_128(x, 2);

  1371. 				BROADCAST_LOAD_A_128(x, 3);

  1372. 

  1373. 				LOAD_B_128(0, x);

  1374. 

  1375. 				MATMUL_128(0, 0);

  1376. 				MATMUL_128(0, 1);

  1377. 				MATMUL_128(0, 2);

  1378. 				MATMUL_128(0, 3);

  1379. 			}

  1380. 			STORE_128(0, 0);

  1381. 			STORE_128(0, 1);

  1382. 			STORE_128(0, 2);

  1383. 			STORE_128(0, 3);

  1384. 		}

  1385. 

  1386. 		for (; j < n2; j+= 2) {

  1387. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  1388. 			DECLARE_RESULT_SCALAR(0, 1);	DECLARE_RESULT_SCALAR(1, 1);

  1389. 			DECLARE_RESULT_SCALAR(0, 2);	DECLARE_RESULT_SCALAR(1, 2);

  1390. 			DECLARE_RESULT_SCALAR(0, 3);	DECLARE_RESULT_SCALAR(1, 3);

  1391. 

  1392. 			for (k = 0; k < K; k++) {

  1393. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  1394. 				BROADCAST_LOAD_A_SCALAR(x, 1);

  1395. 				BROADCAST_LOAD_A_SCALAR(x, 2);

  1396. 				BROADCAST_LOAD_A_SCALAR(x, 3);

  1397. 

  1398. 				LOAD_B_SCALAR(0, x);	LOAD_B_SCALAR(1, x);

  1399. 

  1400. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  1401. 				MATMUL_SCALAR(0, 1);	MATMUL_SCALAR(1, 1);

  1402. 				MATMUL_SCALAR(0, 2);	MATMUL_SCALAR(1, 2);

  1403. 				MATMUL_SCALAR(0, 3);	MATMUL_SCALAR(1, 3);

  1404. 			}

  1405. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  1406. 			STORE_SCALAR(0, 1);	STORE_SCALAR(1, 1);

  1407. 			STORE_SCALAR(0, 2);	STORE_SCALAR(1, 2);

  1408. 			STORE_SCALAR(0, 3);	STORE_SCALAR(1, 3);

  1409. 		}

  1410. 

  1411. 		for (; j < N; j++) {

  1412. 			DECLARE_RESULT_SCALAR(0, 0)

  1413. 			DECLARE_RESULT_SCALAR(0, 1)

  1414. 			DECLARE_RESULT_SCALAR(0, 2)

  1415. 			DECLARE_RESULT_SCALAR(0, 3)

  1416. 

  1417. 			for (k = 0; k < K; k++) {

  1418. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  1419. 				BROADCAST_LOAD_A_SCALAR(0, 1);

  1420. 				BROADCAST_LOAD_A_SCALAR(0, 2);

  1421. 				BROADCAST_LOAD_A_SCALAR(0, 3);

  1422. 

  1423. 				LOAD_B_SCALAR(0, 0);

  1424. 

  1425. 				MATMUL_SCALAR(0, 0);

  1426. 				MATMUL_SCALAR(0, 1);

  1427. 				MATMUL_SCALAR(0, 2);

  1428. 				MATMUL_SCALAR(0, 3);

  1429. 			}

  1430. 			STORE_SCALAR(0, 0);

  1431. 			STORE_SCALAR(0, 1);

  1432. 			STORE_SCALAR(0, 2);

  1433. 			STORE_SCALAR(0, 3);

  1434. 		}

  1435. 	}

  1436. 

  1437. 	for (; i < m2; i+=2) {

  1438. 		j = 0;

  1439. 

  1440. 		for (; j < n64; j+= 64) {

  1441. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  1442. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);    			DECLARE_RESULT_512(2, 1);    DECLARE_RESULT_512(3, 1);

  1443. 

  1444. 

  1445. 			for (k = 0; k < K; k++) {

  1446. 				BROADCAST_LOAD_A_512(x, 0);

  1447. 				BROADCAST_LOAD_A_512(x, 1);

  1448. 

  1449. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  1450. 

  1451. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  1452. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);			MATMUL_512(2, 1);		MATMUL_512(3, 1);

  1453. 			}

  1454. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  1455. 			STORE_512(0, 1);		STORE_512(1, 1);			STORE_512(2, 1);		STORE_512(3, 1);

  1456. 		}

  1457. 

  1458. 		for (; j < n32; j+= 32) {

  1459. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  1460. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);

  1461. 

  1462. 			for (k = 0; k < K; k++) {

  1463. 				BROADCAST_LOAD_A_512(x, 0);

  1464. 				BROADCAST_LOAD_A_512(x, 1);

  1465. 

  1466. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  1467. 

  1468. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  1469. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);

  1470. 			}

  1471. 			STORE_512(0, 0);		STORE_512(1, 0);

  1472. 			STORE_512(0, 1);		STORE_512(1, 1);

  1473. 		}

  1474. 

  1475. 

  1476. 		for (; j < n16; j+= 16) {

  1477. 			DECLARE_RESULT_512(0, 0);

  1478. 			DECLARE_RESULT_512(0, 1);

  1479. 

  1480. 			for (k = 0; k < K; k++) {

  1481. 				BROADCAST_LOAD_A_512(x, 0);

  1482. 				BROADCAST_LOAD_A_512(x, 1);

  1483. 

  1484. 				LOAD_B_512(0, x);

  1485. 

  1486. 				MATMUL_512(0, 0);

  1487. 				MATMUL_512(0, 1);

  1488. 			}

  1489. 			STORE_512(0, 0);

  1490. 			STORE_512(0, 1);

  1491. 		}

  1492. 

  1493. 		for (; j < n8; j+= 8) {

  1494. 			DECLARE_RESULT_256(0, 0);

  1495. 			DECLARE_RESULT_256(0, 1);

  1496. 

  1497. 			for (k = 0; k < K; k++) {

  1498. 				BROADCAST_LOAD_A_256(x, 0);

  1499. 				BROADCAST_LOAD_A_256(x, 1);

  1500. 

  1501. 				LOAD_B_256(0, x);

  1502. 

  1503. 				MATMUL_256(0, 0);

  1504. 				MATMUL_256(0, 1);

  1505. 			}

  1506. 			STORE_256(0, 0);

  1507. 			STORE_256(0, 1);

  1508. 		}

  1509. 

  1510. 		for (; j < n4; j+= 4) {

  1511. 			DECLARE_RESULT_128(0, 0);

  1512. 			DECLARE_RESULT_128(0, 1);

  1513. 

  1514. 			for (k = 0; k < K; k++) {

  1515. 				BROADCAST_LOAD_A_128(x, 0);

  1516. 				BROADCAST_LOAD_A_128(x, 1);

  1517. 

  1518. 				LOAD_B_128(0, x);

  1519. 

  1520. 				MATMUL_128(0, 0);

  1521. 				MATMUL_128(0, 1);

  1522. 			}

  1523. 			STORE_128(0, 0);

  1524. 			STORE_128(0, 1);

  1525. 		}

  1526. 		for (; j < n2; j+= 2) {

  1527. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  1528. 			DECLARE_RESULT_SCALAR(0, 1);	DECLARE_RESULT_SCALAR(1, 1);

  1529. 

  1530. 			for (k = 0; k < K; k++) {

  1531. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  1532. 				BROADCAST_LOAD_A_SCALAR(x, 1);

  1533. 

  1534. 				LOAD_B_SCALAR(0, x);	LOAD_B_SCALAR(1, x);

  1535. 

  1536. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  1537. 				MATMUL_SCALAR(0, 1);	MATMUL_SCALAR(1, 1);

  1538. 			}

  1539. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  1540. 			STORE_SCALAR(0, 1);	STORE_SCALAR(1, 1);

  1541. 		}

  1542. 

  1543. 		for (; j < N; j++) {

  1544. 			DECLARE_RESULT_SCALAR(0, 0);

  1545. 			DECLARE_RESULT_SCALAR(0, 1);

  1546. 

  1547. 			for (k = 0; k < K; k++) {

  1548. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  1549. 				BROADCAST_LOAD_A_SCALAR(0, 1);

  1550. 

  1551. 				LOAD_B_SCALAR(0, 0);

  1552. 

  1553. 				MATMUL_SCALAR(0, 0);

  1554. 				MATMUL_SCALAR(0, 1);

  1555. 			}

  1556. 			STORE_SCALAR(0, 0);

  1557. 			STORE_SCALAR(0, 1);

  1558. 		}

  1559. 	}

  1560. 

  1561. 	for (; i < M; i+=1) {

  1562. 		j = 0;

  1563. 		for (; j < n64; j+= 64) {

  1564. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  1565. 

  1566. 			for (k = 0; k < K; k++) {

  1567. 				BROADCAST_LOAD_A_512(x, 0);

  1568. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  1569. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  1570. 			}

  1571. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  1572. 		}

  1573. 		for (; j < n32; j+= 32) {

  1574. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  1575. 

  1576. 			for (k = 0; k < K; k++) {

  1577. 				BROADCAST_LOAD_A_512(x, 0);

  1578. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  1579. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  1580. 			}

  1581. 			STORE_512(0, 0);		STORE_512(1, 0);

  1582. 		}

  1583. 

  1584. 

  1585. 		for (; j < n16; j+= 16) {

  1586. 			DECLARE_RESULT_512(0, 0);

  1587. 

  1588. 			for (k = 0; k < K; k++) {

  1589. 				BROADCAST_LOAD_A_512(x, 0);

  1590. 

  1591. 				LOAD_B_512(0, x);

  1592. 

  1593. 				MATMUL_512(0, 0);

  1594. 			}

  1595. 			STORE_512(0, 0);

  1596. 		}

  1597. 

  1598. 		for (; j < n8; j+= 8) {

  1599. 			DECLARE_RESULT_256(0, 0);

  1600. 

  1601. 			for (k = 0; k < K; k++) {

  1602. 				BROADCAST_LOAD_A_256(x, 0);

  1603. 				LOAD_B_256(0, x);

  1604. 				MATMUL_256(0, 0);

  1605. 			}

  1606. 			STORE_256(0, 0);

  1607. 		}

  1608. 

  1609. 		for (; j < n4; j+= 4) {

  1610. 			DECLARE_RESULT_128(0, 0);

  1611. 

  1612. 			for (k = 0; k < K; k++) {

  1613. 				BROADCAST_LOAD_A_128(x, 0);

  1614. 				LOAD_B_128(0, x);

  1615. 				MATMUL_128(0, 0);

  1616. 			}

  1617. 			STORE_128(0, 0);

  1618. 		}

  1619. 

  1620. 		for (; j < n2; j+= 2) {

  1621. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  1622. 

  1623. 			for (k = 0; k < K; k++) {

  1624. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  1625. 				LOAD_B_SCALAR(0, 0);	LOAD_B_SCALAR(1, 0);

  1626. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  1627. 			}

  1628. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  1629. 		}

  1630. 

  1631. 		for (; j < N; j++) {

  1632. 			DECLARE_RESULT_SCALAR(0, 0);

  1633. 

  1634. 			for (k = 0; k < K; k++) {

  1635. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  1636. 				LOAD_B_SCALAR(0, 0);

  1637. 				MATMUL_SCALAR(0, 0);

  1638. 			}

  1639. 			STORE_SCALAR(0, 0);

  1640. 		}

  1641. 	}

  1642. }