OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d7ba7679b6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd7ba7679b6'
${ noResults }
OpenBLAS/kernel/x86_64/cgemv_t_microk_haswell-4.c

540 lines
24 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary froms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary from must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define HAVE_KERNEL_4x4 1

  29. static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  30. 

  31. static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  32. {

  33. 

  34. 	BLASLONG register i = 0;

  35. 

  36. 	__asm__  __volatile__

  37. 	(

  38. 	"vzeroupper			 \n\t"

  39. 

  40. 	"vxorps		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  41. 	"vxorps		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  42. 	"vxorps		%%ymm10, %%ymm10, %%ymm10	\n\t" // temp

  43. 	"vxorps		%%ymm11, %%ymm11, %%ymm11	\n\t" // temp

  44. 	"vxorps		%%ymm12, %%ymm12, %%ymm12	\n\t" // temp

  45. 	"vxorps		%%ymm13, %%ymm13, %%ymm13	\n\t"

  46. 	"vxorps		%%ymm14, %%ymm14, %%ymm14	\n\t"

  47. 	"vxorps		%%ymm15, %%ymm15, %%ymm15	\n\t"

  48. 

  49.         "testq          $0x04, %1                      \n\t"

  50.         "jz             2f                      \n\t"

  51. 

  52. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  53. 	"vmovups	(%5,%0,4), %%ymm5               \n\t" // 4 complex values from a1

  54. 

  55. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  56. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  57. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  58. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  59. 	

  60. 	"vmovups	(%6,%0,4), %%ymm6	        \n\t" // 4 complex values from a2

  61. 	"vmovups	(%7,%0,4), %%ymm7               \n\t" // 4 complex values from a3

  62. 

  63. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  64. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  65. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  66. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  67. 	"vfmadd231ps      %%ymm6 , %%ymm0, %%ymm12      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  68. 	"vfmadd231ps      %%ymm6 , %%ymm1, %%ymm13      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  69. 	"vfmadd231ps      %%ymm7 , %%ymm0, %%ymm14      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  70. 	"vfmadd231ps      %%ymm7 , %%ymm1, %%ymm15      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  71. 	

  72.         "addq		$8  , %0	  	 	        \n\t"

  73. 	"subq	        $4  , %1			        \n\t"		

  74. 

  75.         "2:                                  \n\t"

  76. 	"cmpq           $0, %1                         \n\t"

  77.         "je             3f                      \n\t"

  78. 

  79. 	//	".align 16				        \n\t"

  80. 	"1:				        \n\t"

  81.         "prefetcht0      192(%4,%0,4)                   \n\t"

  82. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  83.         "prefetcht0      192(%5,%0,4)                   \n\t"

  84. 	"vmovups	(%5,%0,4), %%ymm5               \n\t" // 4 complex values from a1

  85. 

  86.         "prefetcht0      192(%2,%0,4)                   \n\t"

  87. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  88. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  89. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  90. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  91. 	

  92.         "prefetcht0      192(%6,%0,4)                   \n\t"

  93. 	"vmovups	(%6,%0,4), %%ymm6	        \n\t" // 4 complex values from a2

  94.         "prefetcht0      192(%7,%0,4)                   \n\t"

  95. 	"vmovups	(%7,%0,4), %%ymm7               \n\t" // 4 complex values from a3

  96. 

  97. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  98. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  99. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  100. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  101. 	"vfmadd231ps      %%ymm6 , %%ymm0, %%ymm12      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  102. 	"vfmadd231ps      %%ymm6 , %%ymm1, %%ymm13      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  103. 	"vfmadd231ps      %%ymm7 , %%ymm0, %%ymm14      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  104. 	"vfmadd231ps      %%ymm7 , %%ymm1, %%ymm15      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  105. 

  106. 	"vmovups       32(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  107. 	"vmovups       32(%5,%0,4), %%ymm5              \n\t" // 4 complex values from a1

  108. 

  109. 	"vmovups	  32(%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  110. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  111. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  112. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  113. 

  114. 	"vmovups       32(%6,%0,4), %%ymm6	        \n\t" // 4 complex values from a2

  115. 	"vmovups       32(%7,%0,4), %%ymm7              \n\t" // 4 complex values from a3

  116. 

  117. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  118. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  119. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  120. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  121. 	"vfmadd231ps      %%ymm6 , %%ymm0, %%ymm12      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  122. 	"vfmadd231ps      %%ymm6 , %%ymm1, %%ymm13      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  123. 	"vfmadd231ps      %%ymm7 , %%ymm0, %%ymm14      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  124. 	"vfmadd231ps      %%ymm7 , %%ymm1, %%ymm15      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  125. 

  126.         "addq		$16 , %0	  	 	        \n\t"

  127. 	"subq	        $8  , %1			        \n\t"		

  128. 	"jnz		1b		        \n\t"

  129. 

  130.         "3:                                   \n\t"

  131. 

  132.         "vbroadcastss    (%8)  , %%xmm0                \n\t"  // value from alpha

  133.         "vbroadcastss   4(%8)  , %%xmm1                \n\t"  // value from alpha

  134. 

  135. 

  136. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  137.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  138.         "vpermilps      $0xb1 , %%ymm11, %%ymm11               \n\t"

  139.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  140.         "vpermilps      $0xb1 , %%ymm15, %%ymm15               \n\t"

  141.         "vaddsubps      %%ymm9 , %%ymm8, %%ymm8                \n\t" 

  142.         "vaddsubps      %%ymm11, %%ymm10, %%ymm10              \n\t"

  143.         "vaddsubps      %%ymm13, %%ymm12, %%ymm12              \n\t"

  144.         "vaddsubps      %%ymm15, %%ymm14, %%ymm14              \n\t"

  145. #else

  146.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  147.         "vpermilps      $0xb1 , %%ymm10, %%ymm10               \n\t"

  148.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  149.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  150.         "vaddsubps      %%ymm8 , %%ymm9 , %%ymm8               \n\t"

  151.         "vaddsubps      %%ymm10, %%ymm11, %%ymm10              \n\t"

  152.         "vaddsubps      %%ymm12, %%ymm13, %%ymm12              \n\t"

  153.         "vaddsubps      %%ymm14, %%ymm15, %%ymm14              \n\t"

  154.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  155.         "vpermilps      $0xb1 , %%ymm10, %%ymm10               \n\t"

  156.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  157.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  158. #endif

  159. 

  160. 	"vmovsd         (%3), %%xmm4			\n\t" // read y

  161. 	"vmovsd        8(%3), %%xmm5			\n\t"

  162. 	"vmovsd       16(%3), %%xmm6			\n\t"

  163. 	"vmovsd       24(%3), %%xmm7			\n\t"

  164. 

  165. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  166. 	"vextractf128   $1, %%ymm10, %%xmm11	      	      \n\t"

  167. 	"vextractf128   $1, %%ymm12, %%xmm13		      \n\t"

  168. 	"vextractf128   $1, %%ymm14, %%xmm15		      \n\t"

  169. 

  170. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  171. 	"vaddps		%%xmm10, %%xmm11, %%xmm10      \n\t"

  172. 	"vaddps		%%xmm12, %%xmm13, %%xmm12      \n\t"

  173. 	"vaddps		%%xmm14, %%xmm15, %%xmm14      \n\t"

  174. 

  175. 	"vshufpd        $0x1, %%xmm8 , %%xmm8 , %%xmm9   \n\t"

  176. 	"vshufpd        $0x1, %%xmm10, %%xmm10, %%xmm11  \n\t"

  177. 	"vshufpd        $0x1, %%xmm12, %%xmm12, %%xmm13  \n\t"

  178. 	"vshufpd        $0x1, %%xmm14, %%xmm14, %%xmm15  \n\t"

  179. 

  180. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  181. 	"vaddps		%%xmm10, %%xmm11, %%xmm10      \n\t"

  182. 	"vaddps		%%xmm12, %%xmm13, %%xmm12      \n\t"

  183. 	"vaddps		%%xmm14, %%xmm15, %%xmm14      \n\t"

  184. 

  185. 

  186.         "vmulps         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  187.         "vmulps         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  188.         "vmulps         %%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  189.         "vmulps         %%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  190.         "vmulps         %%xmm12, %%xmm1 , %%xmm13             \n\t"  // t_r * alpha_i , t_i * alpha_i

  191.         "vmulps         %%xmm12, %%xmm0 , %%xmm12             \n\t"  // t_r * alpha_r , t_i * alpha_r

  192.         "vmulps         %%xmm14, %%xmm1 , %%xmm15             \n\t"  // t_r * alpha_i , t_i * alpha_i

  193.         "vmulps         %%xmm14, %%xmm0 , %%xmm14             \n\t"  // t_r * alpha_r , t_i * alpha_r

  194. 

  195. #if !defined(XCONJ)

  196.         "vpermilps      $0xb1 , %%xmm9 , %%xmm9                \n\t"

  197.         "vpermilps      $0xb1 , %%xmm11, %%xmm11               \n\t"

  198.         "vpermilps      $0xb1 , %%xmm13, %%xmm13               \n\t"

  199.         "vpermilps      $0xb1 , %%xmm15, %%xmm15               \n\t"

  200.         "vaddsubps      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  201.         "vaddsubps      %%xmm11, %%xmm10, %%xmm10             \n\t"

  202.         "vaddsubps      %%xmm13, %%xmm12, %%xmm12             \n\t"

  203.         "vaddsubps      %%xmm15, %%xmm14, %%xmm14             \n\t"

  204. #else

  205.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  206.         "vpermilps      $0xb1 , %%xmm10, %%xmm10               \n\t"

  207.         "vpermilps      $0xb1 , %%xmm12, %%xmm12               \n\t"

  208.         "vpermilps      $0xb1 , %%xmm14, %%xmm14               \n\t"

  209.         "vaddsubps      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  210.         "vaddsubps      %%xmm10, %%xmm11, %%xmm10             \n\t"

  211.         "vaddsubps      %%xmm12, %%xmm13, %%xmm12             \n\t"

  212.         "vaddsubps      %%xmm14, %%xmm15, %%xmm14             \n\t"

  213.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  214.         "vpermilps      $0xb1 , %%xmm10, %%xmm10               \n\t"

  215.         "vpermilps      $0xb1 , %%xmm12, %%xmm12               \n\t"

  216.         "vpermilps      $0xb1 , %%xmm14, %%xmm14               \n\t"

  217. #endif

  218. 

  219. 

  220. 	"vaddps		%%xmm8 , %%xmm4 , %%xmm8       \n\t"

  221. 	"vaddps		%%xmm10, %%xmm5 , %%xmm10      \n\t"

  222. 	"vaddps		%%xmm12, %%xmm6 , %%xmm12      \n\t"

  223. 	"vaddps		%%xmm14, %%xmm7 , %%xmm14      \n\t"

  224. 

  225. 	"vmovsd	%%xmm8 ,   (%3)			\n\t"

  226. 	"vmovsd	%%xmm10,  8(%3)			\n\t"

  227. 	"vmovsd	%%xmm12, 16(%3)			\n\t"

  228. 	"vmovsd	%%xmm14, 24(%3)			\n\t"

  229. 

  230. 	"vzeroupper			 \n\t"

  231. 

  232. 	:

  233.           "+r" (i),	// 0	

  234. 	  "+r" (n)  	// 1

  235. 	:

  236.           "r" (x),      // 2

  237.           "r" (y),      // 3

  238.           "r" (ap[0]),  // 4

  239.           "r" (ap[1]),  // 5

  240.           "r" (ap[2]),  // 6

  241.           "r" (ap[3]),  // 7

  242.           "r" (alpha)   // 8

  243. 	: "cc", 

  244. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  245. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  246. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  247. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  248. 	  "memory"

  249. 	);

  250. 

  251. } 

  252. 

  253. 

  254. #define HAVE_KERNEL_4x2 1

  255. static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  256. 

  257. static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  258. {

  259. 

  260. 	BLASLONG register i = 0;

  261. 

  262. 	__asm__  __volatile__

  263. 	(

  264. 	"vzeroupper			 \n\t"

  265. 

  266. 	"vxorps		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  267. 	"vxorps		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  268. 	"vxorps		%%ymm10, %%ymm10, %%ymm10	\n\t" // temp

  269. 	"vxorps		%%ymm11, %%ymm11, %%ymm11	\n\t" // temp

  270. 

  271.         "testq          $0x04, %1                      \n\t"

  272.         "jz             2f                    \n\t"

  273. 

  274. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  275. 	"vmovups	(%5,%0,4), %%ymm5               \n\t" // 4 complex values from a1

  276. 

  277. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  278. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  279. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  280. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  281. 	

  282. 

  283. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  284. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  285. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  286. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  287. 	

  288.         "addq		$8  , %0	  	 	        \n\t"

  289. 	"subq	        $4  , %1			        \n\t"		

  290. 

  291.         "2:                                  \n\t"

  292. 	"cmpq           $0, %1                         \n\t"

  293.         "je             3f                      \n\t"

  294. 

  295. 	//	".align 16				        \n\t"

  296. 	"1:				        \n\t"

  297.         "prefetcht0      192(%4,%0,4)                   \n\t"

  298. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  299.         "prefetcht0      192(%5,%0,4)                   \n\t"

  300. 	"vmovups	(%5,%0,4), %%ymm5               \n\t" // 4 complex values from a1

  301. 

  302.         "prefetcht0      192(%2,%0,4)                   \n\t"

  303. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  304. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  305. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  306. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  307. 	

  308. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  309. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  310. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  311. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  312. 

  313. 	"vmovups       32(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  314. 	"vmovups       32(%5,%0,4), %%ymm5              \n\t" // 4 complex values from a1

  315. 

  316. 	"vmovups	  32(%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  317. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  318. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  319. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  320. 

  321. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  322. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  323. 	"vfmadd231ps      %%ymm5 , %%ymm0, %%ymm10      \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  324. 	"vfmadd231ps      %%ymm5 , %%ymm1, %%ymm11      \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  325. 

  326.         "addq		$16 , %0	  	 	        \n\t"

  327. 	"subq	        $8  , %1			        \n\t"		

  328. 	"jnz		1b		        \n\t"

  329. 

  330.         "3:                                   \n\t"

  331. 

  332.         "vbroadcastss    (%6)  , %%xmm0                \n\t"  // value from alpha

  333.         "vbroadcastss   4(%6)  , %%xmm1                \n\t"  // value from alpha

  334. 

  335. 

  336. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  337.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  338.         "vpermilps      $0xb1 , %%ymm11, %%ymm11               \n\t"

  339.         "vaddsubps      %%ymm9 , %%ymm8, %%ymm8                \n\t" 

  340.         "vaddsubps      %%ymm11, %%ymm10, %%ymm10              \n\t"

  341. #else

  342.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  343.         "vpermilps      $0xb1 , %%ymm10, %%ymm10               \n\t"

  344.         "vaddsubps      %%ymm8 , %%ymm9 , %%ymm8               \n\t"

  345.         "vaddsubps      %%ymm10, %%ymm11, %%ymm10              \n\t"

  346.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  347.         "vpermilps      $0xb1 , %%ymm10, %%ymm10               \n\t"

  348. #endif

  349. 

  350. 	"vmovsd         (%3), %%xmm4			\n\t" // read y

  351. 	"vmovsd        8(%3), %%xmm5			\n\t"

  352. 

  353. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  354. 	"vextractf128   $1, %%ymm10, %%xmm11	      	      \n\t"

  355. 

  356. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  357. 	"vaddps		%%xmm10, %%xmm11, %%xmm10      \n\t"

  358. 

  359. 	"vshufpd        $0x1, %%xmm8 , %%xmm8 , %%xmm9   \n\t"

  360. 	"vshufpd        $0x1, %%xmm10, %%xmm10, %%xmm11  \n\t"

  361. 

  362. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  363. 	"vaddps		%%xmm10, %%xmm11, %%xmm10      \n\t"

  364. 

  365.         "vmulps         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  366.         "vmulps         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  367.         "vmulps         %%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  368.         "vmulps         %%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  369. 

  370. #if !defined(XCONJ)

  371.         "vpermilps      $0xb1 , %%xmm9 , %%xmm9                \n\t"

  372.         "vpermilps      $0xb1 , %%xmm11, %%xmm11               \n\t"

  373.         "vaddsubps      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  374.         "vaddsubps      %%xmm11, %%xmm10, %%xmm10             \n\t"

  375. #else

  376.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  377.         "vpermilps      $0xb1 , %%xmm10, %%xmm10               \n\t"

  378.         "vaddsubps      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  379.         "vaddsubps      %%xmm10, %%xmm11, %%xmm10             \n\t"

  380.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  381.         "vpermilps      $0xb1 , %%xmm10, %%xmm10               \n\t"

  382. #endif

  383. 

  384. 

  385. 	"vaddps		%%xmm8 , %%xmm4 , %%xmm8       \n\t"

  386. 	"vaddps		%%xmm10, %%xmm5 , %%xmm10      \n\t"

  387. 

  388. 	"vmovsd	%%xmm8 ,   (%3)			\n\t"

  389. 	"vmovsd	%%xmm10,  8(%3)			\n\t"

  390. 

  391. 	"vzeroupper			 \n\t"

  392. 

  393. 	:

  394.           "+r" (i),	// 0	

  395. 	  "+r" (n)  	// 1

  396. 	:

  397.           "r" (x),      // 2

  398.           "r" (y),      // 3

  399.           "r" (ap[0]),  // 4

  400.           "r" (ap[1]),  // 5

  401.           "r" (alpha)   // 6

  402. 	: "cc", 

  403. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  404. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  405. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  406. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  407. 	  "memory"

  408. 	);

  409. 

  410. } 

  411. 

  412. 

  413. #define HAVE_KERNEL_4x1 1

  414. static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  415. 

  416. static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  417. {

  418. 

  419. 	BLASLONG register i = 0;

  420. 

  421. 	__asm__  __volatile__

  422. 	(

  423. 	"vzeroupper			 \n\t"

  424. 

  425. 	"vxorps		%%ymm8 , %%ymm8 , %%ymm8 	\n\t" // temp

  426. 	"vxorps		%%ymm9 , %%ymm9 , %%ymm9 	\n\t" // temp

  427. 

  428.         "testq          $0x04, %1                      \n\t"

  429.         "jz             2f                    \n\t"

  430. 

  431. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  432. 

  433. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  434. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  435. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  436. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  437. 	

  438. 

  439. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  440. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  441. 	

  442.         "addq		$8  , %0	  	 	        \n\t"

  443. 	"subq	        $4  , %1			        \n\t"		

  444. 

  445.         "2:                                  \n\t"

  446. 	"cmpq           $0, %1                         \n\t"

  447.         "je             3f                      \n\t"

  448. 

  449. 	//	".align 16				        \n\t"

  450. 	"1:				        \n\t"

  451.         "prefetcht0      192(%4,%0,4)                   \n\t"

  452. 	"vmovups	(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  453. 

  454.         "prefetcht0      192(%2,%0,4)                   \n\t"

  455. 	"vmovups	    (%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  456. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  457. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  458. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  459. 	

  460. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  461. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  462. 

  463. 	"vmovups       32(%4,%0,4), %%ymm4	        \n\t" // 4 complex values from a0

  464. 

  465. 	"vmovups	  32(%2,%0,4)  , %%ymm6		\n\t" // 4 complex values from x

  466. 	"vpermilps        $0xb1, %%ymm6, %%ymm7		\n\t" // exchange real and imap parts

  467. 	"vblendps $0x55, %%ymm6, %%ymm7, %%ymm0         \n\t" // only the real parts

  468. 	"vblendps $0x55, %%ymm7, %%ymm6, %%ymm1         \n\t" // only the imag parts

  469. 

  470. 	"vfmadd231ps      %%ymm4 , %%ymm0, %%ymm8       \n\t" // ar0*xr0,al0*xr0,ar1*xr1,al1*xr1 

  471. 	"vfmadd231ps      %%ymm4 , %%ymm1, %%ymm9       \n\t" // ar0*xl0,al0*xl0,ar1*xl1,al1*xl1 

  472. 

  473.         "addq		$16 , %0	  	 	        \n\t"

  474. 	"subq	        $8  , %1			        \n\t"		

  475. 	"jnz		1b		        \n\t"

  476. 

  477.         "3:                                   \n\t"

  478. 

  479.         "vbroadcastss    (%5)  , %%xmm0                \n\t"  // value from alpha

  480.         "vbroadcastss   4(%5)  , %%xmm1                \n\t"  // value from alpha

  481. 

  482. 

  483. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  484.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  485.         "vaddsubps      %%ymm9 , %%ymm8, %%ymm8                \n\t" 

  486. #else

  487.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  488.         "vaddsubps      %%ymm8 , %%ymm9 , %%ymm8               \n\t"

  489.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  490. #endif

  491. 

  492. 	"vmovsd         (%3), %%xmm4			\n\t" // read y

  493. 

  494. 	"vextractf128   $1, %%ymm8 , %%xmm9		      \n\t"

  495. 

  496. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  497. 

  498. 	"vshufpd        $0x1, %%xmm8 , %%xmm8 , %%xmm9   \n\t"

  499. 

  500. 	"vaddps		%%xmm8 , %%xmm9 , %%xmm8       \n\t"

  501. 

  502.         "vmulps         %%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  503.         "vmulps         %%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  504. 

  505. #if !defined(XCONJ)

  506.         "vpermilps      $0xb1 , %%xmm9 , %%xmm9                \n\t"

  507.         "vaddsubps      %%xmm9 , %%xmm8, %%xmm8               \n\t"

  508. #else

  509.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  510.         "vaddsubps      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  511.         "vpermilps      $0xb1 , %%xmm8 , %%xmm8                \n\t"

  512. #endif

  513. 

  514. 

  515. 	"vaddps		%%xmm8 , %%xmm4 , %%xmm8       \n\t"

  516. 

  517. 	"vmovsd	%%xmm8 ,   (%3)			\n\t"

  518. 

  519. 	"vzeroupper			 \n\t"

  520. 

  521. 	:

  522.           "+r" (i),	// 0	

  523. 	  "+r" (n)  	// 1

  524. 	:

  525.           "r" (x),      // 2

  526.           "r" (y),      // 3

  527.           "r" (ap),     // 4

  528.           "r" (alpha)   // 5

  529. 	: "cc", 

  530. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  531. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  532. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  533. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  534. 	  "memory"

  535. 	);

  536. 

  537. } 

  538.