From 4ba4ab623f045c54d68bd55519b55a3ac5596b74 Mon Sep 17 00:00:00 2001
From: wernsaar <wernsaar@googlemail.com>
Date: Tue, 19 Aug 2014 17:09:45 +0200
Subject: [PATCH] added optimized ssymv_U kernel for nehalem

---
 kernel/x86_64/KERNEL.NEHALEM             |   3 +
 kernel/x86_64/ssymv_U.c                  | 132 +++++++++++++++++------
 kernel/x86_64/ssymv_U_microk_nehalem-2.c | 130 ++++++++++++++++++++++
 3 files changed, 231 insertions(+), 34 deletions(-)
 create mode 100644 kernel/x86_64/ssymv_U_microk_nehalem-2.c

diff --git a/kernel/x86_64/KERNEL.NEHALEM b/kernel/x86_64/KERNEL.NEHALEM
index ca9ff252d..353514449 100644
--- a/kernel/x86_64/KERNEL.NEHALEM
+++ b/kernel/x86_64/KERNEL.NEHALEM
@@ -1,3 +1,6 @@
+#DSYMV_U_KERNEL = dsymv_U.c
+SSYMV_U_KERNEL = ssymv_U.c
+
 SGEMVNKERNEL = sgemv_n.c
 SGEMVTKERNEL = sgemv_t.c
 
diff --git a/kernel/x86_64/ssymv_U.c b/kernel/x86_64/ssymv_U.c
index 75b8e2c3e..61127aa3d 100644
--- a/kernel/x86_64/ssymv_U.c
+++ b/kernel/x86_64/ssymv_U.c
@@ -31,41 +31,94 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(BULLDOZER)
 #include "ssymv_U_microk_bulldozer-2.c"
+#elif defined(NEHALEM)
+#include "ssymv_U_microk_nehalem-2.c"
 #endif
 
+#ifndef HAVE_KERNEL_4x4
 
-#ifndef HAVE_KERNEL_16x2
+static void ssymv_kernel_4x4(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT *a3, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
+{
+	FLOAT at0,at1,at2,at3;
+	FLOAT x;
+	FLOAT tmp2[4] = { 0.0, 0.0, 0.0, 0.0 };
+	FLOAT tp0;
+	FLOAT tp1;
+	FLOAT tp2;
+	FLOAT tp3;
+	BLASLONG i;
+
+	tp0 = temp1[0];
+	tp1 = temp1[1];
+	tp2 = temp1[2];
+	tp3 = temp1[3];
+	
+	for (i=0; i<n; i++)
+	{
+		at0     = a0[i];
+		at1     = a1[i];
+		at2     = a2[i];
+		at3     = a3[i];
+		x       = xp[i];
+		yp[i]   += tp0 * at0 + tp1 *at1 + tp2 * at2 + tp3 * at3;
+		tmp2[0] += at0 * x;
+		tmp2[1] += at1 * x;
+		tmp2[2] += at2 * x;
+		tmp2[3] += at3 * x;
+
+	}
 
-static void ssymv_kernel_16x2(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
+	temp2[0] += tmp2[0];
+	temp2[1] += tmp2[1];
+	temp2[2] += tmp2[2];
+	temp2[3] += tmp2[3];
+}
+
+#endif
+
+
+#ifndef HAVE_KERNEL_1x4
+
+static void ssymv_kernel_1x4(BLASLONG from, BLASLONG to, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT *a3, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
 {
 	FLOAT at0,at1,at2,at3;
-	FLOAT tmp2[2] = { 0.0, 0.0 };
+	FLOAT x;
+	FLOAT tmp2[4] = { 0.0, 0.0, 0.0, 0.0 };
 	FLOAT tp0;
 	FLOAT tp1;
+	FLOAT tp2;
+	FLOAT tp3;
 	BLASLONG i;
 
 	tp0 = temp1[0];
 	tp1 = temp1[1];
+	tp2 = temp1[2];
+	tp3 = temp1[3];
 	
-	for (i=0; i<n; i+=2)
+	for (i=from; i<to; i++)
 	{
 		at0     = a0[i];
 		at1     = a1[i];
-		at2     = a0[i+1];
-		at3     = a1[i+1];
-		yp[i]   += tp0 * at0 + tp1 *at1;
-		yp[i+1] += tp0 * at2 + tp1 *at3;
-		tmp2[0] += at0 * xp[i] + at2 * xp[i+1];
-		tmp2[1] += at1 * xp[i] + at3 * xp[i+1];
+		at2     = a2[i];
+		at3     = a3[i];
+		x       = xp[i];
+		yp[i]   += tp0 * at0 + tp1 *at1 + tp2 * at2 + tp3 * at3;
+		tmp2[0] += at0 * x;
+		tmp2[1] += at1 * x;
+		tmp2[2] += at2 * x;
+		tmp2[3] += at3 * x;
 
 	}
-	temp2[0] = tmp2[0];
-	temp2[1] = tmp2[1];
 
+	temp2[0] += tmp2[0];
+	temp2[1] += tmp2[1];
+	temp2[2] += tmp2[2];
+	temp2[3] += tmp2[3];
 }
 
 #endif
 
+
 static void ssymv_kernel_8x1(BLASLONG n, FLOAT *a0, FLOAT *xp, FLOAT *yp, FLOAT *temp1, FLOAT *temp2)
 {
 	FLOAT at0,at1,at2,at3;
@@ -100,13 +153,16 @@ int CNAME(BLASLONG m, BLASLONG offset, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOA
 	BLASLONG ix,iy;
 	BLASLONG jx,jy;
 	BLASLONG j;
+	BLASLONG j1;
+	BLASLONG j2;
 	BLASLONG m2;
 	FLOAT temp1;
 	FLOAT temp2;
 	FLOAT *xp, *yp;
-	FLOAT *a0,*a1;
-	FLOAT tmp1[2];
-	FLOAT tmp2[2];
+	FLOAT *a0,*a1,*a2,*a3;
+	FLOAT at0,at1,at2,at3;
+	FLOAT tmp1[4];
+	FLOAT tmp2[4];
 
 #if 0
 	if( m != offset )
@@ -146,37 +202,45 @@ int CNAME(BLASLONG m, BLASLONG offset, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOA
 	xp = x;
 	yp = y;
 
-	m2 = m - ( mrange % 2 );
+	m2 = m - ( mrange % 4 );
 
-	for (j=m1; j<m2; j+=2)
+	for (j=m1; j<m2; j+=4)
 	{
 		tmp1[0] = alpha * xp[j];
 		tmp1[1] = alpha * xp[j+1];
+		tmp1[2] = alpha * xp[j+2];
+		tmp1[3] = alpha * xp[j+3];
 		tmp2[0] = 0.0;
 		tmp2[1] = 0.0;
+		tmp2[2] = 0.0;
+		tmp2[3] = 0.0;
 		a0    = &a[j*lda];
 		a1    = a0+lda;
-		FLOAT at0,at1;
-		BLASLONG j1 = (j/16)*16;		
+		a2    = a1+lda;
+		a3    = a2+lda;
+		j1 = (j/8)*8;		
 		if ( j1 )
-			ssymv_kernel_16x2(j1, a0, a1, xp, yp, tmp1, tmp2);
+			ssymv_kernel_4x4(j1, a0, a1, a2, a3, xp, yp, tmp1, tmp2);
+		if ( j1 < j )
+			ssymv_kernel_1x4(j1, j,  a0, a1, a2, a3, xp, yp, tmp1, tmp2);
 
-		for (i=j1; i<j; i++)
+		j2 = 0;
+		for ( j1 = j ; j1 < j+4 ; j1++ )
 		{
-			at0     = a0[i];
-			at1     = a1[i];
-			yp[i]   += tmp1[0] * at0 + tmp1[1] *at1;
-			tmp2[0] += at0 * xp[i];
-			tmp2[1] += at1 * xp[i];
-			
-		}
+			temp1 = tmp1[j2];
+			temp2 = tmp2[j2];
+			a0    = &a[j1*lda];
+			for ( i=j ; i<j1; i++ )
+			{
+				yp[i] += temp1 * a0[i];	
+				temp2 += a0[i] * xp[i];
+				
+			}
+			y[j1] += temp1 * a0[j1] + alpha * temp2;
+			j2++;
 
-		at1     = a1[j];
-		yp[j]   += tmp1[1] * at1;
-		tmp2[1] += at1 * xp[j];
+		}
 
-		yp[j]   += tmp1[0] * a0[j]   + alpha * tmp2[0];
-		yp[j+1] += tmp1[1] * a1[j+1] + alpha * tmp2[1];
 	}
 
 	for ( ; j<m; j++)
@@ -185,7 +249,7 @@ int CNAME(BLASLONG m, BLASLONG offset, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOA
 		temp2 = 0.0;
 		a0    = &a[j*lda];
 		FLOAT at0;
-		BLASLONG j1 = (j/8)*8;		
+		j1 = (j/8)*8;		
 
 		if ( j1 )
 			ssymv_kernel_8x1(j1, a0, xp, yp, &temp1, &temp2);
diff --git a/kernel/x86_64/ssymv_U_microk_nehalem-2.c b/kernel/x86_64/ssymv_U_microk_nehalem-2.c
new file mode 100644
index 000000000..9505a395a
--- /dev/null
+++ b/kernel/x86_64/ssymv_U_microk_nehalem-2.c
@@ -0,0 +1,130 @@
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_4x4 1
+static void ssymv_kernel_4x4( BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT *a3,  FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2) __attribute__ ((noinline));
+
+static void ssymv_kernel_4x4(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *a2, FLOAT *a3, FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2)
+{
+
+	BLASLONG register i = 0;
+
+	__asm__  __volatile__
+	(
+	"xorps          %%xmm0 , %%xmm0              \n\t"	// temp2[0]
+	"xorps          %%xmm1 , %%xmm1              \n\t"	// temp2[1]
+	"xorps          %%xmm2 , %%xmm2              \n\t"	// temp2[2]
+	"xorps          %%xmm3 , %%xmm3              \n\t"	// temp2[3]
+	"movss	       (%8),    %%xmm4	             \n\t"	// temp1[0]
+	"movss        4(%8),    %%xmm5	             \n\t"	// temp1[1]
+	"movss        8(%8),    %%xmm6	             \n\t"	// temp1[2]
+	"movss       12(%8),    %%xmm7	             \n\t"	// temp1[3]
+	"shufps $0,  %%xmm4, %%xmm4                  \n\t"
+	"shufps $0,  %%xmm5, %%xmm5                  \n\t"
+	"shufps $0,  %%xmm6, %%xmm6                  \n\t"
+	"shufps $0,  %%xmm7, %%xmm7                  \n\t"
+
+	"xorq		%0,%0			     \n\t"
+
+	".align 16		  		       \n\t"
+	".L01LOOP%=:				       \n\t"
+	"movups	            (%2,%0,4), %%xmm8	       \n\t"	// 4 * x
+	"movups	            (%3,%0,4), %%xmm9         \n\t"	// 4 * y
+
+	"movups	            (%4,%0,4), %%xmm12	       \n\t"	// 4 * a
+	"movups	            (%5,%0,4), %%xmm13	       \n\t"	// 4 * a
+
+	"movups		    %%xmm12  , %%xmm11	       \n\t"
+	"mulps		    %%xmm4   , %%xmm11	       \n\t"    // temp1 * a
+	"addps		    %%xmm11  , %%xmm9	       \n\t"    // y += temp1 * a
+	"mulps		    %%xmm8   , %%xmm12	       \n\t"    // a * x
+	"addps		    %%xmm12  , %%xmm0 	       \n\t"    // temp2 += x * a
+
+	"movups	            (%6,%0,4), %%xmm14	       \n\t"	// 4 * a
+	"movups	            (%7,%0,4), %%xmm15	       \n\t"	// 4 * a
+
+	"movups		    %%xmm13  , %%xmm11	       \n\t"
+	"mulps		    %%xmm5   , %%xmm11	       \n\t"    // temp1 * a
+	"addps		    %%xmm11  , %%xmm9	       \n\t"    // y += temp1 * a
+	"mulps		    %%xmm8   , %%xmm13	       \n\t"    // a * x
+	"addps		    %%xmm13  , %%xmm1 	       \n\t"    // temp2 += x * a
+
+	"movups		    %%xmm14  , %%xmm11	       \n\t"
+	"mulps		    %%xmm6   , %%xmm11	       \n\t"    // temp1 * a
+	"addps		    %%xmm11  , %%xmm9	       \n\t"    // y += temp1 * a
+	"mulps		    %%xmm8   , %%xmm14	       \n\t"    // a * x
+	"addps		    %%xmm14  , %%xmm2 	       \n\t"    // temp2 += x * a
+
+	"movups		    %%xmm15  , %%xmm11	       \n\t"
+	"mulps		    %%xmm7   , %%xmm11	       \n\t"    // temp1 * a
+	"addps		    %%xmm11  , %%xmm9	       \n\t"    // y += temp1 * a
+	"mulps		    %%xmm8   , %%xmm15	       \n\t"    // a * x
+	"addps		    %%xmm15  , %%xmm3 	       \n\t"    // temp2 += x * a
+
+	"movups             %%xmm9,   (%3,%0,4)       \n\t"    // 4 * y
+
+        "addq		$4 , %0	  	 	      \n\t"
+	"subq	        $4 , %1			      \n\t"		
+	"jnz		.L01LOOP%=		      \n\t"
+
+	"haddps        %%xmm0, %%xmm0  \n\t"
+	"haddps        %%xmm1, %%xmm1  \n\t"
+	"haddps        %%xmm2, %%xmm2  \n\t"
+	"haddps        %%xmm3, %%xmm3  \n\t"
+	"haddps        %%xmm0, %%xmm0  \n\t"
+	"haddps        %%xmm1, %%xmm1  \n\t"
+	"haddps        %%xmm2, %%xmm2  \n\t"
+	"haddps        %%xmm3, %%xmm3  \n\t"
+
+	"movss         %%xmm0 ,   (%9)		\n\t"	// save temp2
+	"movss         %%xmm1 ,  4(%9)		\n\t"	// save temp2
+	"movss         %%xmm2 ,  8(%9)		\n\t"	// save temp2
+	"movss         %%xmm3 , 12(%9)		\n\t"	// save temp2
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x),      // 2
+          "r" (y),      // 3
+          "r" (a0),     // 4
+          "r" (a1),     // 5
+          "r" (a2),     // 6
+          "r" (a3),     // 7
+          "r" (temp1),  // 8
+          "r" (temp2)   // 9
+	: "cc", 
+	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 
+	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 
+	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 
+	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+	  "memory"
+	);
+
+} 
+
+