OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d7ba7679b6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd7ba7679b6'
${ noResults }
OpenBLAS/kernel/x86_64/zgemv_n_microk_sandy-4.c

418 lines
15 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define HAVE_KERNEL_4x4 1

  29. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  30. 

  31. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  32. {

  33. 

  34. 	BLASLONG register i = 0;

  35. 

  36. 	__asm__  __volatile__

  37. 	(

  38. 	"vzeroupper			 \n\t"

  39. 

  40. 	"vbroadcastsd	  (%2), %%ymm0                  \n\t"  // real part x0

  41. 	"vbroadcastsd	 8(%2), %%ymm1                  \n\t"  // imag part x0

  42. 	"vbroadcastsd	16(%2), %%ymm2                  \n\t"  // real part x1

  43. 	"vbroadcastsd	24(%2), %%ymm3                  \n\t"  // imag part x1

  44. 	"vbroadcastsd	32(%2), %%ymm4                  \n\t"  // real part x2

  45. 	"vbroadcastsd	40(%2), %%ymm5                  \n\t"  // imag part x2

  46. 	"vbroadcastsd	48(%2), %%ymm6                  \n\t"  // real part x3

  47. 	"vbroadcastsd	56(%2), %%ymm7                  \n\t"  // imag part x3

  48. 

  49. 

  50. 	".p2align 4				        \n\t"

  51. 	"1:				        \n\t"

  52. 

  53.         //"prefetcht0      256(%4,%0,8)                   \n\t"

  54. 	"vmovups	(%4,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  55. 	"vmovups      32(%4,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  56. 

  57. 	"vmulpd		  %%ymm8 , %%ymm0 , %%ymm12	\n\t"

  58. 	"vmulpd		  %%ymm8 , %%ymm1 , %%ymm13	\n\t"

  59.         //"prefetcht0      256(%5,%0,8)                   \n\t"

  60. 	"vmulpd		  %%ymm9 , %%ymm0 , %%ymm14	\n\t"

  61. 	"vmovups	(%5,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  62. 	"vmulpd		  %%ymm9 , %%ymm1 , %%ymm15	\n\t"

  63. 	"vmovups      32(%5,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  64. 

  65. 	"vmulpd		  %%ymm8 , %%ymm2 , %%ymm10	\n\t"

  66. 	"vmulpd		  %%ymm8 , %%ymm3 , %%ymm11	\n\t"

  67. 	"vaddpd		  %%ymm12, %%ymm10, %%ymm12	\n\t"

  68. 	"vaddpd		  %%ymm13, %%ymm11, %%ymm13	\n\t"

  69.         //"prefetcht0      256(%6,%0,8)                   \n\t"

  70. 	"vmulpd		  %%ymm9 , %%ymm2 , %%ymm10	\n\t"

  71. 	"vmovups	(%6,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  72. 	"vaddpd		  %%ymm14, %%ymm10, %%ymm14	\n\t"

  73. 	"vmulpd		  %%ymm9 , %%ymm3 , %%ymm11	\n\t"

  74. 

  75. 	"vmovups      32(%6,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  76. 	"vaddpd		  %%ymm15, %%ymm11, %%ymm15	\n\t"

  77. 

  78. 	"vmulpd		  %%ymm8 , %%ymm4 , %%ymm10	\n\t"

  79. 	"vmulpd		  %%ymm8 , %%ymm5 , %%ymm11	\n\t"

  80. 	"vaddpd		  %%ymm12, %%ymm10, %%ymm12	\n\t"

  81. 	"vaddpd		  %%ymm13, %%ymm11, %%ymm13	\n\t"

  82.         // "prefetcht0      256(%7,%0,8)                   \n\t"

  83. 	"vmulpd		  %%ymm9 , %%ymm4 , %%ymm10	\n\t"

  84. 	"vmovups	(%7,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  85. 	"vaddpd		  %%ymm14, %%ymm10, %%ymm14	\n\t"

  86. 	"vmulpd		  %%ymm9 , %%ymm5 , %%ymm11	\n\t"

  87. 

  88. 	"vmovups      32(%7,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  89. 	"vaddpd		  %%ymm15, %%ymm11, %%ymm15	\n\t"

  90. 

  91. 	"vmulpd		  %%ymm8 , %%ymm6 , %%ymm10	\n\t"

  92. 	"vmulpd		  %%ymm8 , %%ymm7 , %%ymm11	\n\t"

  93. 	"vaddpd		  %%ymm12, %%ymm10, %%ymm12	\n\t"

  94. 	"vaddpd		  %%ymm13, %%ymm11, %%ymm13	\n\t"

  95. 	"vmulpd		  %%ymm9 , %%ymm6 , %%ymm10	\n\t"

  96. 	"vmulpd		  %%ymm9 , %%ymm7 , %%ymm11	\n\t"

  97. 	"vaddpd		  %%ymm14, %%ymm10, %%ymm14	\n\t"

  98. 	"vaddpd		  %%ymm15, %%ymm11, %%ymm15	\n\t"

  99. 

  100. 	// "prefetcht0      256(%3,%0,8)			\n\t"

  101. 	"vmovups	  (%3,%0,8),  %%ymm10           \n\t"

  102. 	"vmovups	32(%3,%0,8),  %%ymm11           \n\t"

  103. 

  104. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  105.         "vpermilpd      $0x5 , %%ymm13, %%ymm13               \n\t"

  106.         "vpermilpd      $0x5 , %%ymm15, %%ymm15               \n\t"

  107.         "vaddsubpd      %%ymm13, %%ymm12, %%ymm8              \n\t"

  108.         "vaddsubpd      %%ymm15, %%ymm14, %%ymm9              \n\t"

  109. #else

  110.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  111.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  112.         "vaddsubpd      %%ymm12, %%ymm13, %%ymm8              \n\t"

  113.         "vaddsubpd      %%ymm14, %%ymm15, %%ymm9              \n\t"

  114.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  115.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  116. #endif

  117. 

  118.         "vaddpd         %%ymm8, %%ymm10, %%ymm12              \n\t"

  119.         "vaddpd         %%ymm9, %%ymm11, %%ymm13              \n\t"

  120. 

  121. 	"vmovups  %%ymm12,   (%3,%0,8)		        \n\t" // 2 complex values to y	

  122. 	"vmovups  %%ymm13, 32(%3,%0,8)		        \n\t"	

  123. 

  124.         "addq		$8 , %0	  	 	        \n\t"

  125. 	"subq	        $4 , %1			        \n\t"		

  126. 	"jnz		1b		        \n\t"

  127. 	"vzeroupper			 \n\t"

  128. 

  129. 	:

  130.           "+r" (i),	// 0	

  131. 	  "+r" (n)  	// 1

  132.         : 

  133.           "r" (x),      // 2

  134.           "r" (y),      // 3

  135.           "r" (ap[0]),  // 4

  136.           "r" (ap[1]),  // 5

  137.           "r" (ap[2]),  // 6

  138.           "r" (ap[3])   // 7

  139. 	: "cc", 

  140. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  141. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  142. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  143. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  144. 	  "memory"

  145. 	);

  146. 

  147. } 

  148. 

  149. 

  150. #define HAVE_KERNEL_4x2 1

  151. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  152. 

  153. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  154. {

  155. 

  156. 	BLASLONG register i = 0;

  157. 

  158. 	__asm__  __volatile__

  159. 	(

  160. 	"vzeroupper			 \n\t"

  161. 

  162. 	"vbroadcastsd	  (%2), %%ymm0                  \n\t"  // real part x0

  163. 	"vbroadcastsd	 8(%2), %%ymm1                  \n\t"  // imag part x0

  164. 	"vbroadcastsd	16(%2), %%ymm2                  \n\t"  // real part x1

  165. 	"vbroadcastsd	24(%2), %%ymm3                  \n\t"  // imag part x1

  166. 

  167. 	".p2align 4				        \n\t"

  168. 	"1:				        \n\t"

  169. 

  170.         // "prefetcht0      256(%4,%0,8)                   \n\t"

  171. 	"vmovups	(%4,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  172. 	"vmovups      32(%4,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  173. 	"vmulpd		  %%ymm8 , %%ymm0 , %%ymm12	\n\t"

  174. 	"vmulpd		  %%ymm8 , %%ymm1 , %%ymm13	\n\t"

  175. 

  176.         // "prefetcht0      256(%5,%0,8)                   \n\t"

  177. 	"vmulpd		  %%ymm9 , %%ymm0 , %%ymm14	\n\t"

  178. 	"vmovups	(%5,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  179. 	"vmulpd		  %%ymm9 , %%ymm1 , %%ymm15	\n\t"

  180. 	"vmovups      32(%5,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  181. 

  182. 	"vmulpd		  %%ymm8 , %%ymm2 , %%ymm10	\n\t"

  183. 	"vmulpd		  %%ymm8 , %%ymm3 , %%ymm11	\n\t"

  184. 	"vaddpd		  %%ymm12, %%ymm10, %%ymm12	\n\t"

  185. 	"vaddpd		  %%ymm13, %%ymm11, %%ymm13	\n\t"

  186. 

  187. 	"vmulpd		  %%ymm9 , %%ymm2 , %%ymm10	\n\t"

  188. 	"vmulpd		  %%ymm9 , %%ymm3 , %%ymm11	\n\t"

  189. 	"vaddpd		  %%ymm14, %%ymm10, %%ymm14	\n\t"

  190. 	"vaddpd		  %%ymm15, %%ymm11, %%ymm15	\n\t"

  191. 

  192. 	

  193. 	// "prefetcht0      256(%3,%0,8)			\n\t"

  194. 	"vmovups	  (%3,%0,8),  %%ymm10           \n\t"

  195. 	"vmovups	32(%3,%0,8),  %%ymm11           \n\t"

  196. 

  197. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  198.         "vpermilpd      $0x5 , %%ymm13, %%ymm13               \n\t"

  199.         "vpermilpd      $0x5 , %%ymm15, %%ymm15               \n\t"

  200.         "vaddsubpd      %%ymm13, %%ymm12, %%ymm8              \n\t"

  201.         "vaddsubpd      %%ymm15, %%ymm14, %%ymm9              \n\t"

  202. #else

  203.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  204.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  205.         "vaddsubpd      %%ymm12, %%ymm13, %%ymm8              \n\t"

  206.         "vaddsubpd      %%ymm14, %%ymm15, %%ymm9              \n\t"

  207.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  208.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  209. #endif

  210. 

  211.         "vaddpd         %%ymm8, %%ymm10, %%ymm12              \n\t"

  212.         "vaddpd         %%ymm9, %%ymm11, %%ymm13              \n\t"

  213. 

  214. 	"vmovups  %%ymm12,   (%3,%0,8)		        \n\t" // 2 complex values to y	

  215. 	"vmovups  %%ymm13, 32(%3,%0,8)		        \n\t"	

  216. 

  217.         "addq		$8 , %0	  	 	        \n\t"

  218. 	"subq	        $4 , %1			        \n\t"		

  219. 	"jnz		1b		        \n\t"

  220. 	"vzeroupper			 \n\t"

  221. 

  222. 	:

  223.           "+r" (i),	// 0	

  224. 	  "+r" (n)  	// 1

  225.         : 

  226.           "r" (x),      // 2

  227.           "r" (y),      // 3

  228.           "r" (ap[0]),  // 4

  229.           "r" (ap[1])   // 5

  230. 	: "cc", 

  231. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  232. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  233. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  234. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  235. 	  "memory"

  236. 	);

  237. 

  238. } 

  239. 

  240. 

  241. #define HAVE_KERNEL_4x1 1

  242. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  243. 

  244. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)

  245. {

  246. 

  247. 	BLASLONG register i = 0;

  248. 

  249. 	__asm__  __volatile__

  250. 	(

  251. 	"vzeroupper			 \n\t"

  252. 

  253. 	"vbroadcastsd	  (%2), %%ymm0                  \n\t"  // real part x0

  254. 	"vbroadcastsd	 8(%2), %%ymm1                  \n\t"  // imag part x0

  255. 

  256. 	".p2align 4				        \n\t"

  257. 	"1:				        \n\t"

  258. 

  259.         // "prefetcht0      256(%4,%0,8)                   \n\t"

  260. 	"vmovups	(%4,%0,8), %%ymm8	        \n\t" // 2 complex values form a0

  261. 	"vmovups      32(%4,%0,8), %%ymm9	        \n\t" // 2 complex values form a0

  262. 	"vmulpd		  %%ymm8 , %%ymm0 , %%ymm12	\n\t"

  263. 	"vmulpd		  %%ymm8 , %%ymm1 , %%ymm13	\n\t"

  264. 

  265. 	"vmulpd		  %%ymm9 , %%ymm0 , %%ymm14	\n\t"

  266. 	"vmulpd		  %%ymm9 , %%ymm1 , %%ymm15	\n\t"

  267. 

  268. 	// "prefetcht0      256(%3,%0,8)			\n\t"

  269. 	"vmovups	  (%3,%0,8),  %%ymm10           \n\t"

  270. 	"vmovups	32(%3,%0,8),  %%ymm11           \n\t"

  271. 

  272. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  273.         "vpermilpd      $0x5 , %%ymm13, %%ymm13               \n\t"

  274.         "vpermilpd      $0x5 , %%ymm15, %%ymm15               \n\t"

  275.         "vaddsubpd      %%ymm13, %%ymm12, %%ymm8              \n\t"

  276.         "vaddsubpd      %%ymm15, %%ymm14, %%ymm9              \n\t"

  277. #else

  278.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  279.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  280.         "vaddsubpd      %%ymm12, %%ymm13, %%ymm8              \n\t"

  281.         "vaddsubpd      %%ymm14, %%ymm15, %%ymm9              \n\t"

  282.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  283.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  284. #endif

  285. 

  286.         "vaddpd         %%ymm8, %%ymm10, %%ymm12              \n\t"

  287.         "vaddpd         %%ymm9, %%ymm11, %%ymm13              \n\t"

  288. 

  289. 	"vmovups  %%ymm12,   (%3,%0,8)		        \n\t" // 2 complex values to y	

  290. 	"vmovups  %%ymm13, 32(%3,%0,8)		        \n\t"	

  291. 

  292.         "addq		$8 , %0	  	 	        \n\t"

  293. 	"subq	        $4 , %1			        \n\t"		

  294. 	"jnz		1b		        \n\t"

  295. 	"vzeroupper			 \n\t"

  296. 

  297. 	:

  298.           "+r" (i),	// 0	

  299. 	  "+r" (n)  	// 1

  300.         : 

  301.           "r" (x),      // 2

  302.           "r" (y),      // 3

  303.           "r" (ap)      // 4

  304. 	: "cc", 

  305. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  306. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  307. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  308. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  309. 	  "memory"

  310. 	);

  311. 

  312. } 

  313. 

  314. 

  315. 

  316. #define HAVE_KERNEL_ADDY 1

  317. 

  318. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)  __attribute__ ((noinline));

  319. 

  320. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)

  321. {

  322. 	BLASLONG i;

  323. 

  324. 	if ( inc_dest != 2 )

  325. 	{

  326. 

  327. 		FLOAT temp_r;

  328. 		FLOAT temp_i;

  329. 		for ( i=0; i<n; i++ )

  330. 		{

  331. #if !defined(XCONJ) 

  332. 			temp_r = alpha_r * src[0] - alpha_i * src[1];

  333. 			temp_i = alpha_r * src[1] + alpha_i * src[0];

  334. #else

  335. 			temp_r =  alpha_r * src[0] + alpha_i * src[1];

  336. 			temp_i = -alpha_r * src[1] + alpha_i * src[0];

  337. #endif

  338. 

  339. 			*dest += temp_r;

  340. 			*(dest+1) += temp_i;

  341. 

  342. 			src+=2;

  343. 			dest += inc_dest;

  344. 		}

  345. 		return;

  346. 	}

  347. 

  348. 	i=0;

  349. 

  350. 	__asm__  __volatile__

  351. 	(

  352. 

  353. 	"vzeroupper			 \n\t"

  354. 

  355. 	"vbroadcastsd	  (%4), %%ymm0                  \n\t"  // alpha_r

  356. 	"vbroadcastsd	  (%5), %%ymm1                  \n\t"  // alpha_i

  357. 

  358. 	".p2align 4				        \n\t"

  359. 	"1:				        \n\t"

  360. 	// "prefetcht0      192(%2,%0,8)			\n\t"

  361. 	"vmovups	(%2,%0,8), %%ymm8	        \n\t" // 2 complex values from src

  362. 	"vmovups      32(%2,%0,8), %%ymm9	        \n\t" 

  363. 

  364. 	"vmulpd      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  365. 	"vmulpd      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  366. 	"vmulpd      %%ymm9 , %%ymm0, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  367. 	"vmulpd      %%ymm9 , %%ymm1, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  368. 

  369. 	// "prefetcht0      192(%3,%0,8)			\n\t"

  370. 	"vmovups	  (%3,%0,8),  %%ymm10           \n\t" // 2 complex values from dest

  371. 	"vmovups	32(%3,%0,8),  %%ymm11           \n\t"

  372. 

  373. #if !defined(XCONJ)

  374.         "vpermilpd      $0x5 , %%ymm13, %%ymm13               \n\t"

  375.         "vpermilpd      $0x5 , %%ymm15, %%ymm15               \n\t"

  376.         "vaddsubpd      %%ymm13, %%ymm12, %%ymm8              \n\t"

  377.         "vaddsubpd      %%ymm15, %%ymm14, %%ymm9              \n\t"

  378. #else

  379.         "vpermilpd      $0x5 , %%ymm12, %%ymm12               \n\t"

  380.         "vpermilpd      $0x5 , %%ymm14, %%ymm14               \n\t"

  381.         "vaddsubpd      %%ymm12, %%ymm13, %%ymm8              \n\t"

  382.         "vaddsubpd      %%ymm14, %%ymm15, %%ymm9              \n\t"

  383.         "vpermilpd      $0x5 , %%ymm8 , %%ymm8                \n\t"

  384.         "vpermilpd      $0x5 , %%ymm9 , %%ymm9                \n\t"

  385. #endif

  386. 

  387.         "vaddpd         %%ymm8, %%ymm10, %%ymm12              \n\t"

  388.         "vaddpd         %%ymm9, %%ymm11, %%ymm13              \n\t"

  389. 

  390. 	"vmovups  %%ymm12,   (%3,%0,8)		        \n\t" // 2 complex values to y	

  391. 	"vmovups  %%ymm13, 32(%3,%0,8)		        \n\t"	

  392. 

  393.         "addq		$8 , %0	  	 	        \n\t"

  394. 	"subq	        $4 , %1			        \n\t"		

  395. 	"jnz		1b		        \n\t"

  396. 	"vzeroupper			 \n\t"

  397. 

  398. 	:

  399.           "+r" (i),	      // 0	

  400. 	  "+r" (n)  	      // 1

  401.         : 

  402.           "r" (src),          // 2

  403.           "r" (dest),         // 3

  404.           "r" (&alpha_r),     // 4

  405.           "r" (&alpha_i)      // 5

  406. 	: "cc", 

  407. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  408. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  409. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  410. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  411. 	  "memory"

  412. 	);

  413. 

  414. 	return;

  415. 

  416. }