OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 94de62e8be
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '94de62e8be'
${ noResults }
OpenBLAS/kernel/x86_64/zgemv_t_microk_haswell-4.c

429 lines
20 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary froms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary from must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define HAVE_KERNEL_4x4 1

  29. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  30. 

  31. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  32. {

  33. 

  34. 	BLASLONG register i = 0;

  35. 

  36. 	__asm__  __volatile__

  37. 	(

  38. 	"vzeroupper			 \n\t"

  39. 

  40. 	"vxorpd		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  41. 	"vxorpd		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  42. 	"vxorpd		%%ymm10, %%ymm10, %%ymm10	\n\t" // temp

  43. 	"vxorpd		%%ymm11, %%ymm11, %%ymm11	\n\t" // temp

  44. 	"vxorpd		%%ymm12, %%ymm12, %%ymm12	\n\t" // temp

  45. 	"vxorpd		%%ymm13, %%ymm13, %%ymm13	\n\t"

  46. 	"vxorpd		%%ymm14, %%ymm14, %%ymm14	\n\t"

  47. 	"vxorpd		%%ymm15, %%ymm15, %%ymm15	\n\t"

  48. 

  49. 	//	".align 16				        \n\t"

  50. 	"1:				        \n\t"

  51. 

  52.         "prefetcht0      192(%2,%0,8)                   \n\t"

  53. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  54.         "prefetcht0      192(%4,%0,8)                   \n\t"

  55. 	"vmovups	(%5,%0,8), %%ymm5               \n\t" // 2 complex values from a1

  56. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  57. 	"vmovups	(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  58.         "prefetcht0      192(%5,%0,8)                   \n\t"

  59. 	"vmovddup	 16(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  60.         "prefetcht0      192(%6,%0,8)                   \n\t"

  61. 	"vmovups	(%6,%0,8), %%ymm6	        \n\t" // 2 complex values from a2

  62. 	"vmovddup	 24(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  63.         "prefetcht0      192(%7,%0,8)                   \n\t"

  64. 	"vmovups	(%7,%0,8), %%ymm7               \n\t" // 2 complex values from a3

  65. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  66. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  67. 

  68. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  69. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  70. 	"vfmadd231pd      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  71. 	"vfmadd231pd      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  72. 	"vfmadd231pd      %%ymm6 , %%ymm0, %%ymm12      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  73. 	"vfmadd231pd      %%ymm6 , %%ymm1, %%ymm13      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  74. 	"vfmadd231pd      %%ymm7 , %%ymm0, %%ymm14      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  75. 	"vfmadd231pd      %%ymm7 , %%ymm1, %%ymm15      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  76. 

  77. 	"vmovups       32(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  78. 	"vmovups       32(%5,%0,8), %%ymm5              \n\t" // 2 complex values from a1

  79. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  80. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  81. 	"vmovddup	 48(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  82. 	"vmovddup	 56(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  83. 	"vmovups       32(%6,%0,8), %%ymm6	        \n\t" // 2 complex values from a2

  84. 	"vmovups       32(%7,%0,8), %%ymm7               \n\t" // 2 complex values from a3

  85. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  86. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  87. 

  88. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  89. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  90. 	"vfmadd231pd      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  91. 	"vfmadd231pd      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  92. 	"vfmadd231pd      %%ymm6 , %%ymm0, %%ymm12      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  93. 	"vfmadd231pd      %%ymm6 , %%ymm1, %%ymm13      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  94. 	"vfmadd231pd      %%ymm7 , %%ymm0, %%ymm14      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  95. 	"vfmadd231pd      %%ymm7 , %%ymm1, %%ymm15      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  96. 

  97.         "addq		$8 , %0	  	 	        \n\t"

  98. 	"subq	        $4 , %1			        \n\t"		

  99. 	"jnz		1b		        \n\t"

  100. 

  101.         "vmovddup               (%8)  , %%xmm0                \n\t"  // value from alpha

  102.         "vmovddup              8(%8)  , %%xmm1                \n\t"  // value from alpha

  103. 

  104. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  105.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  106.         "vpermilpd      $0x5 , %%ymm11, %%ymm11               \n\t"

  107.         "vpermilpd      $0x5 , %%ymm13, %%ymm13               \n\t"

  108.         "vpermilpd      $0x5 , %%ymm15, %%ymm15               \n\t"

  109.         "vaddsubpd      %%ymm9 , %%ymm8, %%ymm8               \n\t" 

  110.         "vaddsubpd      %%ymm11, %%ymm10, %%ymm10             \n\t"

  111.         "vaddsubpd      %%ymm13, %%ymm12, %%ymm12             \n\t"

  112.         "vaddsubpd      %%ymm15, %%ymm14, %%ymm14             \n\t"

  113. #else

  114.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  115.         "vpermilpd      $0x5 , %%ymm10, %%ymm10               \n\t"

  116.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  117.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  118.         "vaddsubpd      %%ymm8 , %%ymm9 , %%ymm8              \n\t"

  119.         "vaddsubpd      %%ymm10, %%ymm11, %%ymm10             \n\t"

  120.         "vaddsubpd      %%ymm12, %%ymm13, %%ymm12             \n\t"

  121.         "vaddsubpd      %%ymm14, %%ymm15, %%ymm14             \n\t"

  122.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  123.         "vpermilpd      $0x5 , %%ymm10, %%ymm10               \n\t"

  124.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  125.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  126. #endif

  127. 

  128. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  129. 	"vextractf128   $1, %%ymm10, %%xmm11	      	      \n\t"

  130. 	"vextractf128   $1, %%ymm12, %%xmm13		      \n\t"

  131. 	"vextractf128   $1, %%ymm14, %%xmm15		      \n\t"

  132. 

  133. 	"vaddpd		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  134. 	"vaddpd		%%xmm10, %%xmm11, %%xmm10      \n\t"

  135. 	"vaddpd		%%xmm12, %%xmm13, %%xmm12      \n\t"

  136. 	"vaddpd		%%xmm14, %%xmm15, %%xmm14      \n\t"

  137. 

  138.         "vmulpd         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  139.         "vmulpd         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  140.         "vmulpd         %%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  141.         "vmulpd         %%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  142.         "vmulpd         %%xmm12, %%xmm1 , %%xmm13             \n\t"  // t_r * alpha_i , t_i * alpha_i

  143.         "vmulpd         %%xmm12, %%xmm0 , %%xmm12             \n\t"  // t_r * alpha_r , t_i * alpha_r

  144.         "vmulpd         %%xmm14, %%xmm1 , %%xmm15             \n\t"  // t_r * alpha_i , t_i * alpha_i

  145.         "vmulpd         %%xmm14, %%xmm0 , %%xmm14             \n\t"  // t_r * alpha_r , t_i * alpha_r

  146. 

  147. #if !defined(XCONJ) 

  148.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  149.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  150.         "vpermilpd      $0x1 , %%xmm13, %%xmm13               \n\t"

  151.         "vpermilpd      $0x1 , %%xmm15, %%xmm15               \n\t"

  152.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  153.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  154.         "vaddsubpd      %%xmm13, %%xmm12, %%xmm12             \n\t"

  155.         "vaddsubpd      %%xmm15, %%xmm14, %%xmm14             \n\t"

  156. #else

  157.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  158.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  159.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  160.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  161.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  162.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  163.         "vaddsubpd      %%xmm12, %%xmm13, %%xmm12             \n\t"

  164.         "vaddsubpd      %%xmm14, %%xmm15, %%xmm14             \n\t"

  165.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  166.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  167.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  168.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  169. #endif

  170. 

  171.         "vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  172.         "vaddpd         16(%3) , %%xmm10, %%xmm10             \n\t"

  173.         "vaddpd         32(%3) , %%xmm12, %%xmm12             \n\t"

  174.         "vaddpd         48(%3) , %%xmm14, %%xmm14             \n\t"

  175. 

  176. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  177. 	"vmovups	%%xmm10, 16(%3)			\n\t"

  178. 	"vmovups	%%xmm12, 32(%3)			\n\t"

  179. 	"vmovups	%%xmm14, 48(%3)			\n\t"

  180. 

  181. 	"vzeroupper			 \n\t"

  182. 

  183. 	:

  184.           "+r" (i),	// 0	

  185. 	  "+r" (n)  	// 1

  186. 	:

  187.           "r" (x),      // 2

  188.           "r" (y),      // 3

  189.           "r" (ap[0]),  // 4

  190.           "r" (ap[1]),  // 5

  191.           "r" (ap[2]),  // 6

  192.           "r" (ap[3]),  // 7

  193.           "r" (alpha)   // 8

  194. 	: "cc", 

  195. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  196. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  197. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  198. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  199. 	  "memory"

  200. 	);

  201. 

  202. } 

  203. 

  204. 

  205. #define HAVE_KERNEL_4x2 1

  206. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  207. 

  208. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  209. {

  210. 

  211. 	BLASLONG register i = 0;

  212. 

  213. 	__asm__  __volatile__

  214. 	(

  215. 	"vzeroupper			 \n\t"

  216. 

  217. 	"vxorpd		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  218. 	"vxorpd		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  219. 	"vxorpd		%%ymm10, %%ymm10, %%ymm10	\n\t" // temp

  220. 	"vxorpd		%%ymm11, %%ymm11, %%ymm11	\n\t" // temp

  221. 

  222. 	//	".align 16				        \n\t"

  223. 	"1:				        \n\t"

  224. 

  225.         "prefetcht0      192(%2,%0,8)                   \n\t"

  226. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  227.         "prefetcht0      192(%4,%0,8)                   \n\t"

  228. 	"vmovups	(%5,%0,8), %%ymm5               \n\t" // 2 complex values from a1

  229. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  230. 	"vmovups	(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  231.         "prefetcht0      192(%5,%0,8)                   \n\t"

  232. 	"vmovddup	 16(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  233. 	"vmovddup	 24(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  234. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  235. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  236. 

  237. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  238. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  239. 	"vfmadd231pd      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  240. 	"vfmadd231pd      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  241. 

  242. 	"vmovups       32(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  243. 	"vmovups       32(%5,%0,8), %%ymm5              \n\t" // 2 complex values from a1

  244. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  245. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  246. 	"vmovddup	 48(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  247. 	"vmovddup	 56(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  248. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  249. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  250. 

  251. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  252. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  253. 	"vfmadd231pd      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  254. 	"vfmadd231pd      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  255. 

  256.         "addq		$8 , %0	  	 	        \n\t"

  257. 	"subq	        $4 , %1			        \n\t"		

  258. 	"jnz		1b		        \n\t"

  259. 

  260.         "vmovddup               (%6)  , %%xmm0                \n\t"  // value from alpha

  261.         "vmovddup              8(%6)  , %%xmm1                \n\t"  // value from alpha

  262. 

  263. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  264.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  265.         "vpermilpd      $0x5 , %%ymm11, %%ymm11               \n\t"

  266.         "vaddsubpd      %%ymm9 , %%ymm8, %%ymm8               \n\t" 

  267.         "vaddsubpd      %%ymm11, %%ymm10, %%ymm10             \n\t"

  268. #else

  269.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  270.         "vpermilpd      $0x5 , %%ymm10, %%ymm10               \n\t"

  271.         "vaddsubpd      %%ymm8 , %%ymm9 , %%ymm8              \n\t"

  272.         "vaddsubpd      %%ymm10, %%ymm11, %%ymm10             \n\t"

  273.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  274.         "vpermilpd      $0x5 , %%ymm10, %%ymm10               \n\t"

  275. #endif

  276. 

  277. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  278. 	"vextractf128   $1, %%ymm10, %%xmm11	      	      \n\t"

  279. 

  280. 	"vaddpd		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  281. 	"vaddpd		%%xmm10, %%xmm11, %%xmm10      \n\t"

  282. 

  283.         "vmulpd         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  284.         "vmulpd         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  285.         "vmulpd         %%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  286.         "vmulpd         %%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  287. 

  288. #if !defined(XCONJ) 

  289.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  290.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  291.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  292.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  293. #else

  294.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  295.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  296.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  297.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  298.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  299.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  300. #endif

  301. 

  302.         "vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  303.         "vaddpd         16(%3) , %%xmm10, %%xmm10             \n\t"

  304. 

  305. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  306. 	"vmovups	%%xmm10, 16(%3)			\n\t"

  307. 

  308. 	"vzeroupper			 \n\t"

  309. 

  310. 	:

  311.           "+r" (i),	// 0	

  312. 	  "+r" (n)  	// 1

  313. 	:

  314.           "r" (x),      // 2

  315.           "r" (y),      // 3

  316.           "r" (ap[0]),  // 4

  317.           "r" (ap[1]),  // 5

  318.           "r" (alpha)   // 6

  319. 	: "cc", 

  320. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  321. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  322. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  323. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  324. 	  "memory"

  325. 	);

  326. 

  327. } 

  328. 

  329. #define HAVE_KERNEL_4x1 1

  330. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  331. 

  332. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  333. {

  334. 

  335. 	BLASLONG register i = 0;

  336. 

  337. 	__asm__  __volatile__

  338. 	(

  339. 	"vzeroupper			 \n\t"

  340. 

  341. 	"vxorpd		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  342. 	"vxorpd		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  343. 

  344. 	//	".align 16				        \n\t"

  345. 	"1:				        \n\t"

  346. 

  347.         "prefetcht0      192(%2,%0,8)                   \n\t"

  348. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  349.         "prefetcht0      192(%4,%0,8)                   \n\t"

  350. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  351. 	"vmovups	(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  352. 	"vmovddup	 16(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  353. 	"vmovddup	 24(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  354. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  355. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  356. 

  357. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  358. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  359. 

  360. 	"vmovups       32(%4,%0,8), %%ymm4	        \n\t" // 2 complex values from a0

  361. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  362. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  363. 	"vmovddup	 48(%2,%0,8), %%xmm2            \n\t"  // real value from x1

  364. 	"vmovddup	 56(%2,%0,8), %%xmm3            \n\t"  // imag value from x1

  365. 	"vinsertf128	 $1, %%xmm2, %%ymm0 , %%ymm0	\n\t"  // real values from x0 and x1

  366. 	"vinsertf128	 $1, %%xmm3, %%ymm1 , %%ymm1	\n\t"  // imag values from x0 and x1

  367. 

  368. 	"vfmadd231pd      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  369. 	"vfmadd231pd      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  370. 

  371.         "addq		$8 , %0	  	 	        \n\t"

  372. 	"subq	        $4 , %1			        \n\t"		

  373. 	"jnz		1b		        \n\t"

  374. 

  375.         "vmovddup               (%5)  , %%xmm0                \n\t"  // value from alpha

  376.         "vmovddup              8(%5)  , %%xmm1                \n\t"  // value from alpha

  377. 

  378. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  379.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  380.         "vaddsubpd      %%ymm9 , %%ymm8, %%ymm8               \n\t" 

  381. #else

  382.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  383.         "vaddsubpd      %%ymm8 , %%ymm9 , %%ymm8              \n\t"

  384.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  385. #endif

  386. 

  387. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  388. 

  389. 	"vaddpd		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  390. 

  391.         "vmulpd         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  392.         "vmulpd         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  393. 

  394. #if !defined(XCONJ) 

  395.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  396.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  397. #else

  398.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  399.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  400.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  401. #endif

  402. 

  403.         "vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  404. 

  405. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  406. 

  407. 	"vzeroupper			 \n\t"

  408. 

  409. 	:

  410.           "+r" (i),	// 0	

  411. 	  "+r" (n)  	// 1

  412. 	:

  413.           "r" (x),      // 2

  414.           "r" (y),      // 3

  415.           "r" (ap),     // 4

  416.           "r" (alpha)   // 5

  417. 	: "cc", 

  418. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  419. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  420. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  421. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  422. 	  "memory"

  423. 	);

  424. 

  425. } 

  426. 

  427.