Merge pull request #1076 from ashwinyes/develop_20170130_thunderx2t99

More optimized implementations for ThunderX2T99

.gitignore

@@ -85,4 +85,5 @@ build
 build.*
 *.swp
 benchmark/*.goto
+benchmark/smallscaling
 

benchmark/Makefile

@@ -37,6 +37,18 @@ ESSL=/opt/ibm/lib
 #LIBESSL = -lesslsmp $(ESSL)/libxlomp_ser.so.1 $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a
 LIBESSL = -lesslsmp  $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a
 
+ifneq ($(NO_LAPACK), 1)
+GOTO_LAPACK_TARGETS=slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
+		    scholesky.goto dcholesky.goto ccholesky.goto zcholesky.goto \
+		    sgesv.goto dgesv.goto cgesv.goto zgesv.goto \
+		    sgeev.goto dgeev.goto cgeev.goto zgeev.goto \
+		    csymv.goto zsymv.goto \
+		    sgetri.goto dgetri.goto cgetri.goto zgetri.goto \
+		    spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto
+else
+GOTO_LAPACK_TARGETS=
+endif
+
 ifeq ($(OSNAME), WINNT)
 
 goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
@@ -147,9 +159,7 @@ mkl :: slinpack.mkl dlinpack.mkl clinpack.mkl zlinpack.mkl \
 
 else
 
-goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
-       scholesky.goto dcholesky.goto ccholesky.goto zcholesky.goto \
-       sgemm.goto dgemm.goto cgemm.goto zgemm.goto \
+goto :: sgemm.goto dgemm.goto cgemm.goto zgemm.goto \
        strmm.goto dtrmm.goto ctrmm.goto ztrmm.goto \
        strsm.goto dtrsm.goto ctrsm.goto ztrsm.goto \
        ssyrk.goto dsyrk.goto csyrk.goto zsyrk.goto \
@@ -162,20 +172,16 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
        sswap.goto dswap.goto cswap.goto zswap.goto \
        sscal.goto dscal.goto cscal.goto zscal.goto \
        sasum.goto dasum.goto casum.goto zasum.goto \
-       ssymv.goto dsymv.goto csymv.goto zsymv.goto \
+       ssymv.goto dsymv.goto \
        chemv.goto zhemv.goto \
        chemm.goto zhemm.goto \
        cherk.goto zherk.goto \
        cher2k.goto zher2k.goto \
        sgemv.goto dgemv.goto cgemv.goto zgemv.goto \
-       sgesv.goto dgesv.goto cgesv.goto zgesv.goto \
-       sgeev.goto dgeev.goto cgeev.goto zgeev.goto \
-       sgetri.goto dgetri.goto cgetri.goto zgetri.goto \
-       spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto \
        ssymm.goto dsymm.goto csymm.goto zsymm.goto \
        smallscaling \
        isamax.goto idamax.goto icamax.goto izamax.goto \
-       snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto
+       snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto $(GOTO_LAPACK_TARGETS)
 
 acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \
        scholesky.acml dcholesky.acml ccholesky.acml zcholesky.acml \

benchmark/iamax.c

@@ -149,7 +149,7 @@ int main(int argc, char *argv[]){
   srandom(getpid());
 #endif
 
-  fprintf(stderr, "   SIZE       Time\n");
+  fprintf(stderr, "   SIZE       Flops\n");
 
   for(m = from; m <= to; m += step)
   {
@@ -180,7 +180,9 @@ int main(int argc, char *argv[]){
 
     timeg /= loops;
 
-    fprintf(stderr, " %10.6f secs\n", timeg);
+    fprintf(stderr,
+	    " %10.2f MFlops %10.6f sec\n",
+	    COMPSIZE * sizeof(FLOAT) * 1. * (double)m / timeg * 1.e-6, timeg);
 
   }
 

driver/others/parameter.c

@@ -747,6 +747,10 @@ void blas_set_parameter(void)
   sgemm_q = 352;
   sgemm_r = 4096;
 
+  cgemm_p = 128;
+  cgemm_q = 224;
+  cgemm_r = 4096;
+
   dgemm_prefetch_size_a = 3584;
   dgemm_prefetch_size_b = 512;
   dgemm_prefetch_size_c = 128;

kernel/arm64/KERNEL.THUNDERX

@@ -1,6 +1,6 @@
 include $(KERNELDIR)/KERNEL.ARMV8
 
-SDOTKERNEL=dot-thunderx.c
-DDOTKERNEL=ddot-thunderx.c
-DAXPYKERNEL=daxpy-thunderx.c
+SDOTKERNEL=dot_thunderx.c
+DDOTKERNEL=ddot_thunderx.c
+DAXPYKERNEL=daxpy_thunderx.c
 

kernel/arm64/KERNEL.THUNDERX2T99

@@ -1,25 +1,31 @@
 include $(KERNELDIR)/KERNEL.CORTEXA57
 
-SNRM2KERNEL    = snrm2_thunderx2t99.S
+SASUMKERNEL    = sasum_thunderx2t99.c
+DASUMKERNEL    = dasum_thunderx2t99.c
+CASUMKERNEL    = casum_thunderx2t99.c
+ZASUMKERNEL    = zasum_thunderx2t99.c
+
+SCOPYKERNEL    = copy_thunderx2t99.c
+DCOPYKERNEL    = copy_thunderx2t99.c
+CCOPYKERNEL    = copy_thunderx2t99.c
+ZCOPYKERNEL    = copy_thunderx2t99.c
+
+SNRM2KERNEL    = snrm2_thunderx2t99.c
 CNRM2KERNEL    = cnrm2_thunderx2t99.S
 
 DAXPYKERNEL    = daxpy_thunderx2t99.S
 
-ifndef SMP
-DDOTKERNEL     = ddot_thunderx2t99.S
-else
 DDOTKERNEL     = ddot_thunderx2t99.c
-endif
+
 
 ifeq ($(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N), 8x4)
 DGEMMKERNEL    = dgemm_kernel_8x4_thunderx2t99.S
-else
-DGEMMKERNEL    = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
 endif
 
 ifeq ($(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N), 16x4)
 SGEMMKERNEL    =  sgemm_kernel_16x4_thunderx2t99.S
-else
-SGEMMKERNEL    =  sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S
 endif
 
+ifeq ($(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N), 8x4)
+CGEMMKERNEL    =  cgemm_kernel_8x4_thunderx2t99.S
+endif

kernel/arm64/casum_thunderx2t99.c

@@ -0,0 +1,268 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* "X" vector address */
+#define	INC_X	"x2"	/* "X" stride */
+#define J	"x5"	/* loop variable */
+
+#define REG0	"wzr"
+#define SUMF	"s0"
+#define SUMFD	"d0"
+
+/******************************************************************************/
+
+#define KERNEL_F1					\
+	"ldr	d1, ["X"]			\n"	\
+	"add	"X", "X", #8			\n"	\
+	"fabs	v1.2s, v1.2s			\n"	\
+	"ext	v2.8b, v1.8b, v1.8b, #4		\n"	\
+	"fadd	s1, s1, s2			\n"	\
+	"fadd	"SUMF", "SUMF", s1		\n"
+
+#define KERNEL_F32					\
+	"ldr	q16, ["X"]			\n"	\
+	"ldr	q17, ["X", #16]			\n"	\
+	"ldr	q18, ["X", #32]			\n"	\
+	"ldr	q19, ["X", #48]			\n"	\
+	"ldp	q20, q21, ["X", #64]		\n"	\
+	"ldp	q22, q23, ["X", #96]		\n"	\
+	"fabs	v16.4s, v16.4s			\n"	\
+	"fabs	v17.4s, v17.4s			\n"	\
+	"fabs	v18.4s, v18.4s			\n"	\
+	"fabs	v19.4s, v19.4s			\n"	\
+	"ldp	q24, q25, ["X", #128]		\n"	\
+	"ldp	q26, q27, ["X", #160]		\n"	\
+	"fabs	v20.4s, v20.4s			\n"	\
+	"fabs	v21.4s, v21.4s			\n"	\
+	"fabs	v22.4s, v22.4s			\n"	\
+	"fabs	v23.4s, v23.4s			\n"	\
+	"fadd	v16.4s, v16.4s, v17.4s		\n"	\
+	"fadd	v18.4s, v18.4s, v19.4s		\n"	\
+	"ldp	q28, q29, ["X", #192]		\n"	\
+	"ldp	q30, q31, ["X", #224]		\n"	\
+	"fabs	v24.4s, v24.4s			\n"	\
+	"fabs	v25.4s, v25.4s			\n"	\
+	"fabs	v26.4s, v26.4s			\n"	\
+	"fabs	v27.4s, v27.4s			\n"	\
+	"add	"X", "X", #256			\n"	\
+	"fadd	v20.4s, v20.4s, v21.4s		\n"	\
+	"fadd	v22.4s, v22.4s, v23.4s		\n"	\
+	"fabs	v28.4s, v28.4s			\n"	\
+	"fabs	v29.4s, v29.4s			\n"	\
+	"fabs	v30.4s, v30.4s			\n"	\
+	"fabs	v31.4s, v31.4s			\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024]		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+64]	\n"	\
+	"fadd	v24.4s, v24.4s, v25.4s		\n"	\
+	"fadd	v26.4s, v26.4s, v27.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v16.4s		\n"	\
+	"fadd	v1.4s, v1.4s, v18.4s		\n"	\
+	"fadd	v2.4s, v2.4s, v20.4s		\n"	\
+	"fadd	v3.4s, v3.4s, v22.4s		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+128]	\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+192]	\n"	\
+	"fadd	v28.4s, v28.4s, v29.4s		\n"	\
+	"fadd	v30.4s, v30.4s, v31.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v24.4s		\n"	\
+	"fadd	v5.4s, v5.4s, v26.4s		\n"	\
+	"fadd	v6.4s, v6.4s, v28.4s		\n"	\
+	"fadd	v7.4s, v7.4s, v30.4s		\n"
+
+#define KERNEL_F32_FINALIZE				\
+	"fadd	v0.4s, v0.4s, v1.4s		\n"	\
+	"fadd	v2.4s, v2.4s, v3.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v5.4s		\n"	\
+	"fadd	v6.4s, v6.4s, v7.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v2.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v6.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v4.4s		\n"	\
+	"ext	v1.16b, v0.16b, v0.16b, #8	\n"	\
+	"fadd	v0.2s, v0.2s, v1.2s		\n"	\
+	"faddp	"SUMF", v0.2s			\n"
+
+#define INIT_S						\
+	"lsl	"INC_X", "INC_X", #3		\n"
+
+#define KERNEL_S1					\
+	"ldr	d1, ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"fabs	v1.2s, v1.2s			\n"	\
+	"ext	v2.8b, v1.8b, v1.8b, #4		\n"	\
+	"fadd	s1, s1, s2			\n"	\
+	"fadd	"SUMF", "SUMF", s1		\n"
+
+
+#if defined(SMP)
+extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
+	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
+	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
+
+
+static FLOAT casum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	FLOAT  asum = 0.0 ;
+
+	if ( n < 0 )  return(asum);
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	fmov	"SUMF", "REG0"			\n"
+	"	fmov	s1, "REG0"			\n"
+	"	fmov	s2, "REG0"			\n"
+	"	fmov	s3, "REG0"			\n"
+	"	fmov	s4, "REG0"			\n"
+	"	fmov	s5, "REG0"			\n"
+	"	fmov	s6, "REG0"			\n"
+	"	fmov	s7, "REG0"			\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lasum_kernel_S_BEGIN		\n"
+
+	".Lasum_kernel_F_BEGIN:				\n"
+	"	asr	"J", "N", #5			\n"
+	"	cmp	"J", xzr			\n"
+	"	beq	.Lasum_kernel_F1		\n"
+
+	".Lasum_kernel_F32:				\n"
+	"	"KERNEL_F32"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F32		\n"
+	"	"KERNEL_F32_FINALIZE"			\n"
+
+	".Lasum_kernel_F1:				\n"
+	"	ands	"J", "N", #31			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs    "J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F10		\n"
+	"	b	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S_BEGIN:				\n"
+	"	"INIT_S"				\n"
+	"	asr	"J", "N", #2			\n"
+	"	cmp	"J", xzr			\n"
+	"	ble	.Lasum_kernel_S1		\n"
+
+	".Lasum_kernel_S4:				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S4		\n"
+
+	".Lasum_kernel_S1:				\n"
+	"	ands	"J", "N", #3			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S10:				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S10		\n"
+
+	".Lasum_kernel_L999:				\n"
+	"	fmov	%[ASUM_], "SUMFD"		\n"
+
+	: [ASUM_] "=r" (asum)		//%0
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x)		//%3
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"
+	);
+
+	return asum;
+}
+
+#if defined(SMP)
+static int casum_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
+	BLASLONG inc_y, FLOAT *result, BLASLONG dummy3)
+{
+	*result = casum_compute(n, x, inc_x);
+
+	return 0;
+}
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+	FLOAT asum = 0.0;
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (inc_x == 0)
+		nthreads = 1;
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		asum = casum_compute(n, x, inc_x);
+	} else {
+		int mode, i;
+		char result[MAX_CPU_NUMBER * sizeof(double) * 2];
+		FLOAT *ptr;
+
+		mode = BLAS_SINGLE  | BLAS_COMPLEX;
+
+		blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, NULL, 0, result, 0,
+				   ( void *)casum_thread_function, nthreads);
+
+		ptr = (FLOAT *)result;
+		for (i = 0; i < nthreads; i++) {
+			asum = asum + (*ptr);
+			ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2);
+		}
+	}
+#else
+	asum = casum_compute(n, x, inc_x);
+#endif
+
+	return asum;
+}

kernel/arm64/copy_thunderx2t99.c

@@ -0,0 +1,219 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* X vector address */
+#define	INC_X	"x2"	/* X stride */
+#define	Y	"x3"	/* Y vector address */
+#define	INC_Y	"x4"	/* Y stride */
+#define J	"x5"	/* loop variable */
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+#if !defined(COMPLEX)
+#if !defined(DOUBLE)
+#define TMPF		"s0"
+#define INC_SHIFT	"2"
+#define N_DIV_SHIFT	"2"
+#define N_REM_MASK	"3"
+#else
+#define TMPF		"d0"
+#define INC_SHIFT	"3"
+#define N_DIV_SHIFT	"1"
+#define N_REM_MASK	"1"
+#endif
+#else
+#if !defined(DOUBLE)
+#define TMPF		"d0"
+#define INC_SHIFT	"3"
+#define N_DIV_SHIFT	"1"
+#define N_REM_MASK	"1"
+#else
+#define TMPF		"q0"
+#define INC_SHIFT	"4"
+#define N_DIV_SHIFT	"0"
+#define N_REM_MASK	"0"
+#endif
+#endif
+
+#define KERNEL_F1					\
+	"ldr	"TMPF", ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"str	"TMPF", ["Y"]			\n"	\
+	"add	"Y", "Y", "INC_Y"		\n"
+
+#define KERNEL_F					\
+	"ldr	q0, ["X"], #16			\n"	\
+	"str	q0, ["Y"], #16			\n"
+
+#define INIT						\
+	"lsl	"INC_X", "INC_X", #"INC_SHIFT"	\n"	\
+	"lsl	"INC_Y", "INC_Y", #"INC_SHIFT"	\n"
+
+
+static int do_copy(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	if ( n < 0 )  return 0;
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	mov	"Y", %[Y_]			\n"
+	"	mov	"INC_Y", %[INCY_]		\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lcopy_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lcopy_kernel_S_BEGIN		\n"
+	"	cmp	"INC_Y", #1			\n"
+	"	bne	.Lcopy_kernel_S_BEGIN		\n"
+
+	".Lcopy_kernel_F_BEGIN:				\n"
+	"	"INIT"					\n"
+	"	asr	"J", "N", #"N_DIV_SHIFT"	\n"
+	"	cmp	"J", xzr			\n"
+	"	beq	.Lcopy_kernel_F1		\n"
+	"	.align 5				\n"
+
+	".Lcopy_kernel_F:				\n"
+	"	"KERNEL_F"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lcopy_kernel_F			\n"
+
+	".Lcopy_kernel_F1:				\n"
+#if defined(COMPLEX) && defined(DOUBLE)
+	"	b	.Lcopy_kernel_L999		\n"
+#else
+	"	ands	"J", "N", #"N_REM_MASK"		\n"
+	"	ble	.Lcopy_kernel_L999		\n"
+#endif
+
+	".Lcopy_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lcopy_kernel_F10		\n"
+	"	b	.Lcopy_kernel_L999		\n"
+
+	".Lcopy_kernel_S_BEGIN:				\n"
+	"	"INIT"					\n"
+	"	asr	"J", "N", #2			\n"
+	"	cmp	"J", xzr			\n"
+	"	ble	.Lcopy_kernel_S1		\n"
+
+	".Lcopy_kernel_S4:				\n"
+	"	"KERNEL_F1"				\n"
+	"	"KERNEL_F1"				\n"
+	"	"KERNEL_F1"				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lcopy_kernel_S4		\n"
+
+	".Lcopy_kernel_S1:				\n"
+	"	ands	"J", "N", #3			\n"
+	"	ble	.Lcopy_kernel_L999		\n"
+
+	".Lcopy_kernel_S10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lcopy_kernel_S10		\n"
+
+	".Lcopy_kernel_L999:				\n"
+
+	:
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x),		//%3
+	  [Y_]    "r"  (y),		//%4
+	  [INCY_] "r"  (inc_y)		//%5
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0"
+	);
+
+	return 0;
+}
+
+#if defined(SMP)
+static int copy_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
+	BLASLONG inc_y, FLOAT *dummy3, BLASLONG dummy4)
+{
+	do_copy(n, x, inc_x, y, inc_y);
+
+	return 0;
+}
+#endif
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+
+	if (n <= 0) return 0;
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (inc_x == 0)
+		nthreads = 1;
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		do_copy(n, x, inc_x, y, inc_y);
+	} else {
+		int mode = 0;
+
+#if !defined(COMPLEX)
+		mode = BLAS_REAL;
+#else
+		mode = BLAS_COMPLEX;
+#endif
+#if !defined(DOUBLE)
+		mode |= BLAS_SINGLE;
+#else
+		mode |= BLAS_DOUBLE;
+#endif
+
+		blas_level1_thread(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, y, inc_y, NULL, 0,
+				   ( void *)copy_thread_function, nthreads);
+	}
+#else
+	do_copy(n, x, inc_x, y, inc_y);
+#endif
+
+	return 0;
+}

kernel/arm64/dasum_thunderx2t99.c

@@ -0,0 +1,263 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* "X" vector address */
+#define	INC_X	"x2"	/* "X" stride */
+#define J	"x5"	/* loop variable */
+
+#define REG0	"xzr"
+#define SUMF	"d0"
+#define TMPF	"d1"
+
+/******************************************************************************/
+
+#define KERNEL_F1					\
+	"ldr	"TMPF", ["X"]			\n"	\
+	"add	"X", "X", #8			\n"	\
+	"fabs	"TMPF", "TMPF"			\n"	\
+	"fadd	"SUMF", "SUMF", "TMPF"		\n"
+
+#define KERNEL_F32					\
+	"ldr	q16, ["X"]			\n"	\
+	"ldr	q17, ["X", #16]			\n"	\
+	"ldr	q18, ["X", #32]			\n"	\
+	"ldr	q19, ["X", #48]			\n"	\
+	"ldp	q20, q21, ["X", #64]		\n"	\
+	"ldp	q22, q23, ["X", #96]		\n"	\
+	"fabs	v16.2d, v16.2d			\n"	\
+	"fabs	v17.2d, v17.2d			\n"	\
+	"fabs	v18.2d, v18.2d			\n"	\
+	"fabs	v19.2d, v19.2d			\n"	\
+	"ldp	q24, q25, ["X", #128]		\n"	\
+	"ldp	q26, q27, ["X", #160]		\n"	\
+	"fabs	v20.2d, v20.2d			\n"	\
+	"fabs	v21.2d, v21.2d			\n"	\
+	"fabs	v22.2d, v22.2d			\n"	\
+	"fabs	v23.2d, v23.2d			\n"	\
+	"fadd	v16.2d, v16.2d, v17.2d		\n"	\
+	"fadd	v18.2d, v18.2d, v19.2d		\n"	\
+	"ldp	q28, q29, ["X", #192]		\n"	\
+	"ldp	q30, q31, ["X", #224]		\n"	\
+	"fabs	v24.2d, v24.2d			\n"	\
+	"fabs	v25.2d, v25.2d			\n"	\
+	"fabs	v26.2d, v26.2d			\n"	\
+	"fabs	v27.2d, v27.2d			\n"	\
+	"add	"X", "X", #256			\n"	\
+	"fadd	v20.2d, v20.2d, v21.2d		\n"	\
+	"fadd	v22.2d, v22.2d, v23.2d		\n"	\
+	"fabs	v28.2d, v28.2d			\n"	\
+	"fabs	v29.2d, v29.2d			\n"	\
+	"fabs	v30.2d, v30.2d			\n"	\
+	"fabs	v31.2d, v31.2d			\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024]		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+64]	\n"	\
+	"fadd	v24.2d, v24.2d, v25.2d		\n"	\
+	"fadd	v26.2d, v26.2d, v27.2d		\n"	\
+	"fadd	v28.2d, v28.2d, v29.2d		\n"	\
+	"fadd	v30.2d, v30.2d, v31.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v16.2d		\n"	\
+	"fadd	v1.2d, v1.2d, v18.2d		\n"	\
+	"fadd	v2.2d, v2.2d, v20.2d		\n"	\
+	"fadd	v3.2d, v3.2d, v22.2d		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+128]	\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+192]	\n"	\
+	"fadd	v4.2d, v4.2d, v24.2d		\n"	\
+	"fadd	v5.2d, v5.2d, v26.2d		\n"	\
+	"fadd	v6.2d, v6.2d, v28.2d		\n"	\
+	"fadd	v7.2d, v7.2d, v30.2d		\n"
+
+#define KERNEL_F32_FINALIZE				\
+	"fadd	v0.2d, v0.2d, v1.2d		\n"	\
+	"fadd	v2.2d, v2.2d, v3.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v5.2d		\n"	\
+	"fadd	v6.2d, v6.2d, v7.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v2.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v6.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v4.2d		\n"	\
+	"faddp	"SUMF", v0.2d			\n"
+
+#define INIT_S						\
+	"lsl	"INC_X", "INC_X", #3		\n"
+
+#define KERNEL_S1					\
+	"ldr	"TMPF", ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"fabs	"TMPF", "TMPF"			\n"	\
+	"fadd	"SUMF", "SUMF", "TMPF"		\n"
+
+
+#if defined(SMP)
+extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
+	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
+	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
+
+
+static FLOAT dasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	FLOAT  asum = 0.0 ;
+
+	if ( n < 0 )  return(asum);
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	fmov	"SUMF", "REG0"			\n"
+	"	fmov	d1, "REG0"			\n"
+	"	fmov	d2, "REG0"			\n"
+	"	fmov	d3, "REG0"			\n"
+	"	fmov	d4, "REG0"			\n"
+	"	fmov	d5, "REG0"			\n"
+	"	fmov	d6, "REG0"			\n"
+	"	fmov	d7, "REG0"			\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lasum_kernel_S_BEGIN		\n"
+
+	".Lasum_kernel_F_BEGIN:				\n"
+	"	asr	"J", "N", #5			\n"
+	"	cmp	"J", xzr			\n"
+	"	beq	.Lasum_kernel_F1		\n"
+
+	".align 5					\n"
+	".Lasum_kernel_F32:				\n"
+	"	"KERNEL_F32"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F32		\n"
+	"	"KERNEL_F32_FINALIZE"			\n"
+
+	".Lasum_kernel_F1:				\n"
+	"	ands	"J", "N", #31			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs    "J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F10		\n"
+	"	b	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S_BEGIN:				\n"
+	"	"INIT_S"				\n"
+	"	asr	"J", "N", #2			\n"
+	"	cmp	"J", xzr			\n"
+	"	ble	.Lasum_kernel_S1		\n"
+
+	".Lasum_kernel_S4:				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S4		\n"
+
+	".Lasum_kernel_S1:				\n"
+	"	ands	"J", "N", #3			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S10:				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S10		\n"
+
+	".Lasum_kernel_L999:				\n"
+	"	fmov	%[ASUM_], "SUMF"		\n"
+
+	: [ASUM_] "=r" (asum)		//%0
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x)		//%3
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"
+	);
+
+	return asum;
+}
+
+#if defined(SMP)
+static int dasum_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
+	BLASLONG inc_y, FLOAT *result, BLASLONG dummy3)
+{
+	*result = dasum_compute(n, x, inc_x);
+
+	return 0;
+}
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+	FLOAT asum = 0.0;
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (inc_x == 0)
+		nthreads = 1;
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		asum = dasum_compute(n, x, inc_x);
+	} else {
+		int mode, i;
+		char result[MAX_CPU_NUMBER * sizeof(double) * 2];
+		FLOAT *ptr;
+
+		mode = BLAS_DOUBLE;
+
+		blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, NULL, 0, result, 0,
+				   ( void *)dasum_thread_function, nthreads);
+
+		ptr = (FLOAT *)result;
+		for (i = 0; i < nthreads; i++) {
+			asum = asum + (*ptr);
+			ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2);
+		}
+	}
+#else
+	asum = dasum_compute(n, x, inc_x);
+#endif
+
+	return asum;
+}

kernel/arm64/ddot_thunderx2t99.S

@@ -1,207 +0,0 @@
-/*******************************************************************************
-Copyright (c) 2017, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*******************************************************************************/
-
-#define ASSEMBLER
-#include "common.h"
-
-#define	N	x0	/* vector length */
-#define	X	x1	/* X vector address */
-#define	INC_X	x2	/* X stride */
-#define	Y	x3	/* Y vector address */
-#define	INC_Y	x4	/* Y stride */
-#define I	x5	/* loop variable */
-
-/*******************************************************************************
-* Macro definitions
-*******************************************************************************/
-
-#define REG0	xzr
-#define DOTF	d0
-#define TMPX	d16
-#define LD1VX	{v16.d}[0]
-#define TMPY	d24
-#define LD1VY	{v24.d}[0]
-#define SZ	8
-
-/******************************************************************************/
-
-.macro KERNEL_F1
-	ldr	TMPX, [X]
-	ldr	TMPY, [Y]
-	add	X, X, #SZ
-	add	Y, Y, #SZ
-	fmadd	DOTF, TMPX, TMPY, DOTF
-.endm
-
-.macro KERNEL_F16
-	ldp	q16, q17, [X]
-	ldp	q24, q25, [Y]
-
-	ldp	q18, q19, [X, #32]
-	ldp	q26, q27, [Y, #32]
-
-	fmla	v0.2d, v16.2d, v24.2d
-	fmla	v1.2d, v17.2d, v25.2d
-
-	ldp	q20, q21, [X, #64]
-	ldp	q28, q29, [Y, #64]
-
-	fmla	v2.2d, v18.2d, v26.2d
-	fmla	v3.2d, v19.2d, v27.2d
-
-	ldp	q22, q23, [X, #96]
-	ldp	q30, q31, [Y, #96]
-
-	add	Y, Y, #128
-	add	X, X, #128
-
-	fmla	v4.2d, v20.2d, v28.2d
-	fmla	v5.2d, v21.2d, v29.2d
-
-	PRFM	PLDL1KEEP, [X, #896]
-	PRFM	PLDL1KEEP, [Y, #896]
-	PRFM	PLDL1KEEP, [X, #896+64]
-	PRFM	PLDL1KEEP, [Y, #896+64]
-
-	fmla	v6.2d, v22.2d, v30.2d
-	fmla	v7.2d, v23.2d, v31.2d
-.endm
-
-.macro KERNEL_F32
-	KERNEL_F16
-	KERNEL_F16
-.endm
-
-.macro KERNEL_F32_FINALIZE
-	fadd	v0.2d, v0.2d, v1.2d
-	fadd	v2.2d, v2.2d, v3.2d
-	fadd	v4.2d, v4.2d, v5.2d
-	fadd	v6.2d, v6.2d, v7.2d
-	fadd	v0.2d, v0.2d, v2.2d
-	fadd	v4.2d, v4.2d, v6.2d
-	fadd	v0.2d, v0.2d, v4.2d
-	faddp	DOTF, v0.2d
-.endm
-
-.macro INIT_S
-	lsl	INC_X, INC_X, #3
-	lsl	INC_Y, INC_Y, #3
-.endm
-
-.macro KERNEL_S1
-	ld1	LD1VX, [X], INC_X
-	ld1	LD1VY, [Y], INC_Y
-	fmadd	DOTF, TMPX, TMPY, DOTF
-.endm
-
-/*******************************************************************************
-* End of macro definitions
-*******************************************************************************/
-
-	PROLOGUE
-
-	fmov	DOTF, REG0
-	fmov	d1, REG0
-	fmov	d2, REG0
-	fmov	d3, REG0
-	fmov	d4, REG0
-	fmov	d5, REG0
-	fmov	d6, REG0
-	fmov	d7, REG0
-
-	cmp	N, xzr
-	ble	dot_kernel_L999
-
-	cmp	INC_X, #1
-	bne	dot_kernel_S_BEGIN
-	cmp	INC_Y, #1
-	bne	dot_kernel_S_BEGIN
-
-dot_kernel_F_BEGIN:
-
-	asr	I, N, #5
-	cmp	I, xzr
-	beq	dot_kernel_F1
-
-dot_kernel_F32:
-
-	KERNEL_F32
-
-	subs	I, I, #1
-	bne	dot_kernel_F32
-
-	KERNEL_F32_FINALIZE
-
-dot_kernel_F1:
-
-	ands	I, N, #31
-	ble	dot_kernel_L999
-
-dot_kernel_F10:
-
-	KERNEL_F1
-
-	subs    I, I, #1
-        bne     dot_kernel_F10
-
-	ret
-
-dot_kernel_S_BEGIN:
-
-	INIT_S
-
-	asr	I, N, #2
-	cmp	I, xzr
-	ble	dot_kernel_S1
-
-dot_kernel_S4:
-
-	KERNEL_S1
-	KERNEL_S1
-	KERNEL_S1
-	KERNEL_S1
-
-	subs	I, I, #1
-	bne	dot_kernel_S4
-
-dot_kernel_S1:
-
-	ands	I, N, #3
-	ble	dot_kernel_L999
-
-dot_kernel_S10:
-
-	KERNEL_S1
-
-	subs    I, I, #1
-        bne     dot_kernel_S10
-
-dot_kernel_L999:
-
-	ret
-
-	EPILOGUE

kernel/arm64/ddot_thunderx2t99.c

@@ -45,9 +45,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LD1VY	"{v24.d}[0]"
 #define SZ	"8"
 
+#if defined(SMP)
 extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
 	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
 	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
 
 
 static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
@@ -62,7 +64,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	mov	"INC_X", %[INCX_]		\n"
 	"	mov	"Y", %[Y_]			\n"
 	"	mov	"INC_Y", %[INCY_]		\n"
-	"	fmov	"DOTF", "REG0"		\n"
+	"	fmov	"DOTF", "REG0"			\n"
 	"	fmov	d1, "REG0"			\n"
 	"	fmov	d2, "REG0"			\n"
 	"	fmov	d3, "REG0"			\n"
@@ -72,20 +74,20 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	fmov	d7, "REG0"			\n"
 
 	"	cmp	"N", xzr			\n"
-	"	ble	9f	//dot_kernel_L999	\n"
+	"	ble	.Ldot_kernel_L999		\n"
 
 	"	cmp	"INC_X", #1			\n"
-	"	bne	5f	//dot_kernel_S_BEGIN	\n"
+	"	bne	.Ldot_kernel_S_BEGIN		\n"
 	"	cmp	"INC_Y", #1			\n"
-	"	bne	5f	//dot_kernel_S_BEGIN	\n"
+	"	bne	.Ldot_kernel_S_BEGIN		\n"
 
-	"1:	//dot_kernel_F_BEGIN			\n"
+	".Ldot_kernel_F_BEGIN:				\n"
 	"	asr	"J", "N", #5			\n"
 	"	cmp	"J", xzr			\n"
-	"	beq	3f	//dot_kernel_F1		\n"
+	"	beq	.Ldot_kernel_F1			\n"
 
 	"	.align 5				\n"
-	"2:	//dot_kernel_F32			\n"
+	".Ldot_kernel_F32:				\n"
 	"	ldp	q16, q17, ["X"]			\n"
 	"	ldp	q24, q25, ["Y"]			\n"
 	"	ldp	q18, q19, ["X", #32]		\n"
@@ -133,7 +135,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	fmla	v7.2d, v23.2d, v31.2d		\n"
 
 	"	subs	"J", "J", #1			\n"
-	"	bne	2b	//dot_kernel_F32	\n"
+	"	bne	.Ldot_kernel_F32		\n"
 
 	"	fadd	v0.2d, v0.2d, v1.2d		\n"
 	"	fadd	v2.2d, v2.2d, v3.2d		\n"
@@ -144,11 +146,11 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	fadd	v0.2d, v0.2d, v4.2d		\n"
 	"	faddp	"DOTF", v0.2d			\n"
 
-	"3:	//dot_kernel_F1				\n"
+	".Ldot_kernel_F1:				\n"
 	"	ands	"J", "N", #31			\n"
-	"	ble	9f	//dot_kernel_L999	\n"
+	"	ble	.Ldot_kernel_L999		\n"
 
-	"4:	//dot_kernel_F10			\n"
+	".Ldot_kernel_F10:				\n"
 	"	ldr	"TMPX", ["X"]			\n"
 	"	ldr	"TMPY", ["Y"]			\n"
 	"	add	"X", "X", #"SZ"			\n"
@@ -156,18 +158,18 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	fmadd	"DOTF", "TMPX", "TMPY", "DOTF"	\n"
 
 	"	subs	"J", "J", #1			\n"
-	"	bne	4b	//dot_kernel_F10	\n"
+	"	bne	.Ldot_kernel_F10		\n"
 
-	"	b	9f	//dot_kernel_L999	\n"
+	"	b	.Ldot_kernel_L999		\n"
 
-	"5:	//dot_kernel_S_BEGIN			\n"
+	".Ldot_kernel_S_BEGIN:				\n"
 	"	lsl	"INC_X", "INC_X", #3		\n"
 	"	lsl	"INC_Y", "INC_Y", #3		\n"
 	"	asr	"J", "N", #2			\n"
 	"	cmp	"J", xzr			\n"
-	"	ble	7f	//dot_kernel_S1		\n"
+	"	ble	.Ldot_kernel_S1			\n"
 
-	"6:	//dot_kernel_S4:			\n"
+	".Ldot_kernel_S4:				\n"
 	"	ld1	"LD1VX", ["X"], "INC_X"		\n"
 	"	ld1	"LD1VY", ["Y"], "INC_Y"		\n"
 	"	fmadd	"DOTF", "TMPX", "TMPY", "DOTF"	\n"
@@ -181,21 +183,22 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	"	ld1	"LD1VY", ["Y"], "INC_Y"		\n"
 	"	fmadd	"DOTF", "TMPX", "TMPY", "DOTF"	\n"
 	"	subs	"J", "J", #1			\n"
-	"	bne	6b	//dot_kernel_S4		\n"
+	"	bne	.Ldot_kernel_S4			\n"
 
-	"7:	//dot_kernel_S1:			\n"
+	".Ldot_kernel_S1:				\n"
 	"	ands	"J", "N", #3			\n"
-	"	ble	9f	//dot_kernel_L999	\n"
+	"	ble	.Ldot_kernel_L999		\n"
 
-	"8:	//dot_kernel_S10			\n"
+	".Ldot_kernel_S10:				\n"
 	"	ld1	"LD1VX", ["X"], "INC_X"		\n"
 	"	ld1	"LD1VY", ["Y"], "INC_Y"		\n"
 	"	fmadd	"DOTF", "TMPX", "TMPY", "DOTF"	\n"
 	"	subs	"J", "J", #1			\n"
-	"	bne	8b	//dot_kernel_S10	\n"
+	"	bne	.Ldot_kernel_S10		\n"
 
-	"9:	//dot_kernel_L999			\n"
+	".Ldot_kernel_L999:				\n"
 	"	fmov	%[DOT_], "DOTF"			\n"
+
 	: [DOT_]  "=r" (dot)		//%0
 	: [N_]    "r"  (n),		//%1
 	  [X_]    "r"  (x),		//%2
@@ -211,6 +214,7 @@ static FLOAT ddot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLO
 	return(dot);
 }
 
+#if defined(SMP)
 static int ddot_thread_function(BLASLONG n, BLASLONG dummy0,
 	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
 	BLASLONG inc_y, FLOAT *result, BLASLONG dummy3)
@@ -219,13 +223,17 @@ static int ddot_thread_function(BLASLONG n, BLASLONG dummy0,
 
 	return 0;
 }
+#endif
 
 FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 {
+#if defined(SMP)
 	int nthreads;
-	FLOAT dot = 0.0;
 	FLOAT dummy_alpha;
+#endif
+	FLOAT dot = 0.0;
 
+#if defined(SMP)
 	nthreads = num_cpu_avail(1);
 
 	if (inc_x == 0 || inc_y == 0)
@@ -253,6 +261,9 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 			ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2);
 		}
 	}
+#else
+	dot = ddot_compute(n, x, inc_x, y, inc_y);
+#endif
 
 	return dot;
 }

kernel/arm64/sasum_thunderx2t99.c

@@ -0,0 +1,265 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* "X" vector address */
+#define	INC_X	"x2"	/* "X" stride */
+#define J	"x5"	/* loop variable */
+
+#define REG0	"wzr"
+#define SUMF	"s0"
+#define SUMFD	"d0"
+
+/******************************************************************************/
+
+#define KERNEL_F1					\
+	"ldr	s1, ["X"]			\n"	\
+	"add	"X", "X", #4			\n"	\
+	"fabs	s1, s1				\n"	\
+	"fadd	"SUMF", "SUMF", s1		\n"
+
+#define KERNEL_F64					\
+	"ldr	q16, ["X"]			\n"	\
+	"ldr	q17, ["X", #16]			\n"	\
+	"ldr	q18, ["X", #32]			\n"	\
+	"ldr	q19, ["X", #48]			\n"	\
+	"ldp	q20, q21, ["X", #64]		\n"	\
+	"ldp	q22, q23, ["X", #96]		\n"	\
+	"fabs	v16.4s, v16.4s			\n"	\
+	"fabs	v17.4s, v17.4s			\n"	\
+	"fabs	v18.4s, v18.4s			\n"	\
+	"fabs	v19.4s, v19.4s			\n"	\
+	"ldp	q24, q25, ["X", #128]		\n"	\
+	"ldp	q26, q27, ["X", #160]		\n"	\
+	"fabs	v20.4s, v20.4s			\n"	\
+	"fabs	v21.4s, v21.4s			\n"	\
+	"fabs	v22.4s, v22.4s			\n"	\
+	"fabs	v23.4s, v23.4s			\n"	\
+	"fadd	v16.4s, v16.4s, v17.4s		\n"	\
+	"fadd	v18.4s, v18.4s, v19.4s		\n"	\
+	"ldp	q28, q29, ["X", #192]		\n"	\
+	"ldp	q30, q31, ["X", #224]		\n"	\
+	"fabs	v24.4s, v24.4s			\n"	\
+	"fabs	v25.4s, v25.4s			\n"	\
+	"fabs	v26.4s, v26.4s			\n"	\
+	"fabs	v27.4s, v27.4s			\n"	\
+	"add	"X", "X", #256			\n"	\
+	"fadd	v20.4s, v20.4s, v21.4s		\n"	\
+	"fadd	v22.4s, v22.4s, v23.4s		\n"	\
+	"fabs	v28.4s, v28.4s			\n"	\
+	"fabs	v29.4s, v29.4s			\n"	\
+	"fabs	v30.4s, v30.4s			\n"	\
+	"fabs	v31.4s, v31.4s			\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024]		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+64]	\n"	\
+	"fadd	v24.4s, v24.4s, v25.4s		\n"	\
+	"fadd	v26.4s, v26.4s, v27.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v16.4s		\n"	\
+	"fadd	v1.4s, v1.4s, v18.4s		\n"	\
+	"fadd	v2.4s, v2.4s, v20.4s		\n"	\
+	"fadd	v3.4s, v3.4s, v22.4s		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+128]	\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+192]	\n"	\
+	"fadd	v28.4s, v28.4s, v29.4s		\n"	\
+	"fadd	v30.4s, v30.4s, v31.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v24.4s		\n"	\
+	"fadd	v5.4s, v5.4s, v26.4s		\n"	\
+	"fadd	v6.4s, v6.4s, v28.4s		\n"	\
+	"fadd	v7.4s, v7.4s, v30.4s		\n"
+
+#define KERNEL_F64_FINALIZE				\
+	"fadd	v0.4s, v0.4s, v1.4s		\n"	\
+	"fadd	v2.4s, v2.4s, v3.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v5.4s		\n"	\
+	"fadd	v6.4s, v6.4s, v7.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v2.4s		\n"	\
+	"fadd	v4.4s, v4.4s, v6.4s		\n"	\
+	"fadd	v0.4s, v0.4s, v4.4s		\n"	\
+	"ext	v1.16b, v0.16b, v0.16b, #8	\n"	\
+	"fadd	v0.2s, v0.2s, v1.2s		\n"	\
+	"faddp	"SUMF", v0.2s			\n"
+
+#define INIT_S						\
+	"lsl	"INC_X", "INC_X", #2		\n"
+
+#define KERNEL_S1					\
+	"ldr	s1, ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"fabs	s1, s1				\n"	\
+	"fadd	"SUMF", "SUMF", s1		\n"
+
+
+#if defined(SMP)
+extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
+	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
+	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
+
+
+static FLOAT sasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	FLOAT  asum = 0.0 ;
+
+	if ( n < 0 )  return(asum);
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	fmov	"SUMF", "REG0"			\n"
+	"	fmov	s1, "REG0"			\n"
+	"	fmov	s2, "REG0"			\n"
+	"	fmov	s3, "REG0"			\n"
+	"	fmov	s4, "REG0"			\n"
+	"	fmov	s5, "REG0"			\n"
+	"	fmov	s6, "REG0"			\n"
+	"	fmov	s7, "REG0"			\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lasum_kernel_S_BEGIN		\n"
+
+	".Lasum_kernel_F_BEGIN:				\n"
+	"	asr	"J", "N", #6			\n"
+	"	cmp	"J", xzr			\n"
+	"	beq	.Lasum_kernel_F1		\n"
+
+	".align 5					\n"
+	".Lasum_kernel_F64:				\n"
+	"	"KERNEL_F64"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F64		\n"
+	"	"KERNEL_F64_FINALIZE"			\n"
+
+	".Lasum_kernel_F1:				\n"
+	"	ands	"J", "N", #63			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs    "J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F10		\n"
+	"	b	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S_BEGIN:				\n"
+	"	"INIT_S"				\n"
+	"	asr	"J", "N", #2			\n"
+	"	cmp	"J", xzr			\n"
+	"	ble	.Lasum_kernel_S1		\n"
+
+	".Lasum_kernel_S4:				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S4		\n"
+
+	".Lasum_kernel_S1:				\n"
+	"	ands	"J", "N", #3			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S10:				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S10		\n"
+
+	".Lasum_kernel_L999:				\n"
+	"	fmov	%[ASUM_], "SUMFD"		\n"
+
+	: [ASUM_] "=r" (asum)		//%0
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x)		//%3
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"
+	);
+
+	return asum;
+}
+
+#if defined(SMP)
+static int sasum_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
+	BLASLONG inc_y, FLOAT *result, BLASLONG dummy3)
+{
+	*result = sasum_compute(n, x, inc_x);
+
+	return 0;
+}
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+	FLOAT asum = 0.0;
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (inc_x == 0)
+		nthreads = 1;
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		asum = sasum_compute(n, x, inc_x);
+	} else {
+		int mode, i;
+		char result[MAX_CPU_NUMBER * sizeof(double) * 2];
+		FLOAT *ptr;
+
+		mode = BLAS_SINGLE;
+
+		blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, NULL, 0, result, 0,
+				   ( void *)sasum_thread_function, nthreads);
+
+		ptr = (FLOAT *)result;
+		for (i = 0; i < nthreads; i++) {
+			asum = asum + (*ptr);
+			ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2);
+		}
+	}
+#else
+	asum = sasum_compute(n, x, inc_x);
+#endif
+
+	return asum;
+}

kernel/arm64/snrm2_thunderx2t99.S

@@ -1,228 +0,0 @@
-/*******************************************************************************
-Copyright (c) 2017, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*******************************************************************************/
-
-#define ASSEMBLER
-#include "common.h"
-
-#define	N	x0	/* vector length */
-#define	X	x1	/* X vector address */
-#define	INC_X	x2	/* X stride */
-#define I	x5	/* loop variable */
-
-/*******************************************************************************
-* Macro definitions
-*******************************************************************************/
-
-#define TMPF	s16
-#define TMPFD	d17
-#define SSQ	s0
-#define SSQD	d0
-#define TMPVF	{v16.s}[0]
-#define TMPVFD	{v17.s}[0]
-#define SZ	4
-
-/******************************************************************************/
-
-.macro INIT
-	fmov	SSQD, xzr
-	fmov	d1, xzr
-	fmov	d2, xzr
-	fmov	d3, xzr
-	fmov	d4, xzr
-	fmov	d5, xzr
-	fmov	d6, xzr
-	fmov	d7, xzr
-.endm
-
-.macro KERNEL_F1
-	ldr	TMPF, [X], #SZ
-	fcvt	TMPFD, TMPF
-	fmadd	SSQD, TMPFD, TMPFD, SSQD
-.endm
-
-.macro KERNEL_F32
-	ldur	q16, [X]
-	ldur	q18, [X, #16]
-	ldur	q20, [X, #32]
-	ldur	q22, [X, #48]
-	ldur	q24, [X, #64]
-	ldur	q26, [X, #80]
-	ldur	q28, [X, #96]
-	ldur	q30, [X, #112]
-
-	add	X, X, #128
-
-	fcvtl2	v17.2d, v16.4s
-	fcvtl	v16.2d, v16.2s
-	fcvtl2	v19.2d, v18.4s
-	fcvtl	v18.2d, v18.2s
-	fcvtl2	v21.2d, v20.4s
-	fcvtl	v20.2d, v20.2s
-	fcvtl2	v23.2d, v22.4s
-	fcvtl	v22.2d, v22.2s
-	fcvtl2	v25.2d, v24.4s
-	fcvtl	v24.2d, v24.2s
-	fcvtl2	v27.2d, v26.4s
-	fcvtl	v26.2d, v26.2s
-	fcvtl2	v29.2d, v28.4s
-	fcvtl	v28.2d, v28.2s
-	fcvtl2	v31.2d, v30.4s
-	fcvtl	v30.2d, v30.2s
-
-	fmla	v0.2d, v16.2d, v16.2d
-	fmla	v1.2d, v17.2d, v17.2d
-	fmla	v2.2d, v18.2d, v18.2d
-	fmla	v3.2d, v19.2d, v19.2d
-	fmla	v4.2d, v20.2d, v20.2d
-	fmla	v5.2d, v21.2d, v21.2d
-	fmla	v6.2d, v22.2d, v22.2d
-	fmla	v7.2d, v23.2d, v23.2d
-
-	fmla	v0.2d, v24.2d, v24.2d
-	fmla	v1.2d, v25.2d, v25.2d
-	fmla	v2.2d, v26.2d, v26.2d
-	fmla	v3.2d, v27.2d, v27.2d
-	fmla	v4.2d, v28.2d, v28.2d
-	fmla	v5.2d, v29.2d, v29.2d
-	fmla	v6.2d, v30.2d, v30.2d
-	fmla	v7.2d, v31.2d, v31.2d
-
-	prfm	PLDL1KEEP, [X, #1024]
-	prfm	PLDL1KEEP, [X, #1024+64]
-.endm
-
-.macro KERNEL_F32_FINALIZE
-	fadd	v0.2d, v0.2d, v1.2d
-	fadd	v2.2d, v2.2d, v3.2d
-	fadd	v4.2d, v4.2d, v5.2d
-	fadd	v6.2d, v6.2d, v7.2d
-
-	fadd	v0.2d, v0.2d, v2.2d
-	fadd	v4.2d, v4.2d, v6.2d
-
-	fadd	v0.2d, v0.2d, v4.2d
-	faddp	SSQD, v0.2d
-.endm
-
-.macro INIT_S
-	lsl	INC_X, INC_X, #2
-.endm
-
-.macro KERNEL_S1
-	ldr	TMPF, [X]
-	add	X, X, INC_X
-	fcvt	TMPFD, TMPF
-	fmadd	SSQD, TMPFD, TMPFD, SSQD
-.endm
-
-/*******************************************************************************
-* End of macro definitions
-*******************************************************************************/
-
-	PROLOGUE
-
-	INIT
-
-	cmp	N, xzr
-	ble	nrm2_kernel_zero
-	cmp	INC_X, xzr
-	ble	nrm2_kernel_zero
-	cmp	INC_X, #1
-	bne	nrm2_kernel_S_BEGIN
-
-nrm2_kernel_F_BEGIN:
-
-	asr	I, N, #6
-	cmp	I, xzr
-	beq	nrm2_kernel_S_BEGIN
-
-	.align 5
-nrm2_kernel_F64:
-
-	KERNEL_F32
-	KERNEL_F32
-
-	subs	I, I, #1
-	bne	nrm2_kernel_F64
-
-	KERNEL_F32_FINALIZE
-
-nrm2_kernel_F1:
-
-	ands	I, N, #63
-	ble	nrm2_kernel_L999
-
-nrm2_kernel_F10:
-
-	KERNEL_F1
-
-	subs    I, I, #1
-        bne     nrm2_kernel_F10
-
-	b	nrm2_kernel_L999
-
-nrm2_kernel_S_BEGIN:
-
-	INIT_S
-
-	asr	I, N, #2
-	cmp	I, xzr
-	ble	nrm2_kernel_S1
-
-nrm2_kernel_S4:
-
-	KERNEL_S1
-	KERNEL_S1
-	KERNEL_S1
-	KERNEL_S1
-
-	subs	I, I, #1
-	bne	nrm2_kernel_S4
-
-nrm2_kernel_S1:
-
-	ands	I, N, #3
-	ble	nrm2_kernel_L999
-
-nrm2_kernel_S10:
-
-	KERNEL_S1
-
-	subs    I, I, #1
-	bne     nrm2_kernel_S10
-
-nrm2_kernel_L999:
-	fsqrt	SSQD, SSQD
-	fcvt	SSQ, SSQD
-	ret
-
-nrm2_kernel_zero:
-	fmov	SSQ, wzr
-
-	ret
-
-	EPILOGUE

kernel/arm64/snrm2_thunderx2t99.c

@@ -0,0 +1,256 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+#if defined(SMP)
+extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
+	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
+	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
+
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* X vector address */
+#define	INC_X	"x2"	/* X stride */
+#define I	"x5"	/* loop variable */
+
+#define TMPF	"s16"
+#define TMPFD	"d17"
+#define SSQD	"d0"
+
+#define KERNEL_F1					\
+	"ldr	"TMPF", ["X"], #4		\n"	\
+	"fcvt	"TMPFD", "TMPF"			\n"	\
+	"fmadd	"SSQD", "TMPFD", "TMPFD", "SSQD"\n"
+
+#define KERNEL_F32					\
+	"ldur	q16, ["X"]			\n"	\
+	"ldur	q18, ["X", #16]			\n"	\
+	"ldur	q20, ["X", #32]			\n"	\
+	"ldur	q22, ["X", #48]			\n"	\
+	"ldur	q24, ["X", #64]			\n"	\
+	"ldur	q26, ["X", #80]			\n"	\
+	"ldur	q28, ["X", #96]			\n"	\
+	"ldur	q30, ["X", #112]		\n"	\
+	"add	"X", "X", #128			\n"	\
+	"fcvtl2	v17.2d, v16.4s			\n"	\
+	"fcvtl	v16.2d, v16.2s			\n"	\
+	"fcvtl2	v19.2d, v18.4s			\n"	\
+	"fcvtl	v18.2d, v18.2s			\n"	\
+	"fcvtl2	v21.2d, v20.4s			\n"	\
+	"fcvtl	v20.2d, v20.2s			\n"	\
+	"fcvtl2	v23.2d, v22.4s			\n"	\
+	"fcvtl	v22.2d, v22.2s			\n"	\
+	"fcvtl2	v25.2d, v24.4s			\n"	\
+	"fcvtl	v24.2d, v24.2s			\n"	\
+	"fcvtl2	v27.2d, v26.4s			\n"	\
+	"fcvtl	v26.2d, v26.2s			\n"	\
+	"fcvtl2	v29.2d, v28.4s			\n"	\
+	"fcvtl	v28.2d, v28.2s			\n"	\
+	"fcvtl2	v31.2d, v30.4s			\n"	\
+	"fcvtl	v30.2d, v30.2s			\n"	\
+	"fmla	v0.2d, v16.2d, v16.2d		\n"	\
+	"fmla	v1.2d, v17.2d, v17.2d		\n"	\
+	"fmla	v2.2d, v18.2d, v18.2d		\n"	\
+	"fmla	v3.2d, v19.2d, v19.2d		\n"	\
+	"fmla	v4.2d, v20.2d, v20.2d		\n"	\
+	"fmla	v5.2d, v21.2d, v21.2d		\n"	\
+	"fmla	v6.2d, v22.2d, v22.2d		\n"	\
+	"fmla	v7.2d, v23.2d, v23.2d		\n"	\
+	"fmla	v0.2d, v24.2d, v24.2d		\n"	\
+	"fmla	v1.2d, v25.2d, v25.2d		\n"	\
+	"fmla	v2.2d, v26.2d, v26.2d		\n"	\
+	"fmla	v3.2d, v27.2d, v27.2d		\n"	\
+	"fmla	v4.2d, v28.2d, v28.2d		\n"	\
+	"fmla	v5.2d, v29.2d, v29.2d		\n"	\
+	"fmla	v6.2d, v30.2d, v30.2d		\n"	\
+	"fmla	v7.2d, v31.2d, v31.2d		\n"	\
+	"prfm	PLDL1KEEP, ["X", #1024]		\n"	\
+	"prfm	PLDL1KEEP, ["X", #1024+64]	\n"
+
+#define KERNEL_F32_FINALIZE				\
+	"fadd	v0.2d, v0.2d, v1.2d		\n"	\
+	"fadd	v2.2d, v2.2d, v3.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v5.2d		\n"	\
+	"fadd	v6.2d, v6.2d, v7.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v2.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v6.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v4.2d		\n"	\
+	"faddp	"SSQD", v0.2d			\n"
+
+#define KERNEL_S1					\
+	"ldr	"TMPF", ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"fcvt	"TMPFD", "TMPF"			\n"	\
+	"fmadd	"SSQD", "TMPFD", "TMPFD", "SSQD"\n"
+
+
+static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	double  ret = 0.0 ;
+
+	if (n <= 0)  return ret;
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	fmov	"SSQD", xzr			\n"
+	"	fmov	d1, xzr				\n"
+	"	fmov	d2, xzr				\n"
+	"	fmov	d3, xzr				\n"
+	"	fmov	d4, xzr				\n"
+	"	fmov	d5, xzr				\n"
+	"	fmov	d6, xzr				\n"
+	"	fmov	d7, xzr				\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lnrm2_kernel_L999		\n"
+	"	cmp	"INC_X", xzr			\n"
+	"	ble	.Lnrm2_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lnrm2_kernel_S_BEGIN		\n"
+
+	".Lnrm2_kernel_F_BEGIN:				\n"
+	"	asr	"I", "N", #6			\n"
+	"	cmp	"I", xzr			\n"
+	"	beq	.Lnrm2_kernel_S_BEGIN		\n"
+
+	"	.align 5				\n"
+	".Lnrm2_kernel_F64:				\n"
+	"	"KERNEL_F32"				\n"
+	"	"KERNEL_F32"				\n"
+	"	subs	"I", "I", #1			\n"
+	"	bne	.Lnrm2_kernel_F64		\n"
+	"	"KERNEL_F32_FINALIZE"			\n"
+
+	".Lnrm2_kernel_F1:				\n"
+	"	ands	"I", "N", #63			\n"
+	"	ble	.Lnrm2_kernel_L999		\n"
+
+	".Lnrm2_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs	"I", "I", #1			\n"
+	"	bne	.Lnrm2_kernel_F10		\n"
+	"	b	.Lnrm2_kernel_L999		\n"
+
+	".Lnrm2_kernel_S_BEGIN:				\n"
+	"	lsl	"INC_X", "INC_X", #2		\n"
+	"	asr	"I", "N", #2			\n"
+	"	cmp	"I", xzr			\n"
+	"	ble	.Lnrm2_kernel_S1		\n"
+
+	".Lnrm2_kernel_S4:				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"I", "I", #1			\n"
+	"	bne	.Lnrm2_kernel_S4		\n"
+
+	".Lnrm2_kernel_S1:				\n"
+	"	ands	"I", "N", #3			\n"
+	"	ble	.Lnrm2_kernel_L999		\n"
+
+	".Lnrm2_kernel_S10:				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"I", "I", #1			\n"
+	"	bne	.Lnrm2_kernel_S10		\n"
+
+	".Lnrm2_kernel_L999:				\n"
+	"	fmov	%[RET_], "SSQD"			\n"
+
+	: [RET_]  "=r" (ret)		//%0
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x)		//%3
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"
+	);
+
+	return ret;
+}
+
+#if defined(SMP)
+static int nrm2_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *dummy3,
+	BLASLONG dummy4, FLOAT *result, BLASLONG dummy5)
+{
+	*(double *)result = nrm2_compute(n, x, inc_x);
+
+	return 0;
+}
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+	FLOAT nrm2 = 0.0;
+	double nrm2_double = 0.0;
+
+	if (n <= 0 || inc_x <= 0) return 0.0;
+	if (n == 1) return fabs(x[0]);
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		nrm2_double = nrm2_compute(n, x, inc_x);
+	} else {
+		int mode, i;
+		char result[MAX_CPU_NUMBER * sizeof(double) * 2];
+		double *ptr;
+
+		mode = BLAS_SINGLE  | BLAS_REAL;
+
+		blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, NULL, 0, result, 0,
+				   ( void *)nrm2_thread_function, nthreads);
+
+		ptr = (double *)result;
+		for (i = 0; i < nthreads; i++) {
+			nrm2_double = nrm2_double + (*ptr) * (*ptr);
+			ptr = (double *)(((char *)ptr) + sizeof(double) * 2);
+		}
+	}
+#else
+	nrm2_double = nrm2_compute(n, x, inc_x);
+#endif
+	nrm2 = sqrt(nrm2_double);
+
+	return nrm2;
+}

kernel/arm64/zasum_thunderx2t99.c

@@ -0,0 +1,265 @@
+/***************************************************************************
+Copyright (c) 2017, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+#define	N	"x0"	/* vector length */
+#define	X	"x1"	/* "X" vector address */
+#define	INC_X	"x2"	/* "X" stride */
+#define J	"x5"	/* loop variable */
+
+#define REG0	"xzr"
+#define SUMF	"d0"
+#define TMPF	"d1"
+
+/******************************************************************************/
+
+#define KERNEL_F1					\
+	"ldr	q1, ["X"]			\n"	\
+	"add	"X", "X", #16			\n"	\
+	"fabs	v1.2d, v1.2d			\n"	\
+	"faddp	d1, v1.2d			\n"	\
+	"fadd	"SUMF", "SUMF", d1		\n"
+
+#define KERNEL_F16					\
+	"ldr	q16, ["X"]			\n"	\
+	"ldr	q17, ["X", #16]			\n"	\
+	"ldr	q18, ["X", #32]			\n"	\
+	"ldr	q19, ["X", #48]			\n"	\
+	"ldp	q20, q21, ["X", #64]		\n"	\
+	"ldp	q22, q23, ["X", #96]		\n"	\
+	"fabs	v16.2d, v16.2d			\n"	\
+	"fabs	v17.2d, v17.2d			\n"	\
+	"fabs	v18.2d, v18.2d			\n"	\
+	"fabs	v19.2d, v19.2d			\n"	\
+	"ldp	q24, q25, ["X", #128]		\n"	\
+	"ldp	q26, q27, ["X", #160]		\n"	\
+	"fabs	v20.2d, v20.2d			\n"	\
+	"fabs	v21.2d, v21.2d			\n"	\
+	"fabs	v22.2d, v22.2d			\n"	\
+	"fabs	v23.2d, v23.2d			\n"	\
+	"fadd	v16.2d, v16.2d, v17.2d		\n"	\
+	"fadd	v18.2d, v18.2d, v19.2d		\n"	\
+	"ldp	q28, q29, ["X", #192]		\n"	\
+	"ldp	q30, q31, ["X", #224]		\n"	\
+	"fabs	v24.2d, v24.2d			\n"	\
+	"fabs	v25.2d, v25.2d			\n"	\
+	"fabs	v26.2d, v26.2d			\n"	\
+	"fabs	v27.2d, v27.2d			\n"	\
+	"add	"X", "X", #256			\n"	\
+	"fadd	v20.2d, v20.2d, v21.2d		\n"	\
+	"fadd	v22.2d, v22.2d, v23.2d		\n"	\
+	"fabs	v28.2d, v28.2d			\n"	\
+	"fabs	v29.2d, v29.2d			\n"	\
+	"fabs	v30.2d, v30.2d			\n"	\
+	"fabs	v31.2d, v31.2d			\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024]		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+64]	\n"	\
+	"fadd	v24.2d, v24.2d, v25.2d		\n"	\
+	"fadd	v26.2d, v26.2d, v27.2d		\n"	\
+	"fadd	v28.2d, v28.2d, v29.2d		\n"	\
+	"fadd	v30.2d, v30.2d, v31.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v16.2d		\n"	\
+	"fadd	v1.2d, v1.2d, v18.2d		\n"	\
+	"fadd	v2.2d, v2.2d, v20.2d		\n"	\
+	"fadd	v3.2d, v3.2d, v22.2d		\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+128]	\n"	\
+	"PRFM	PLDL1KEEP, ["X", #1024+192]	\n"	\
+	"fadd	v4.2d, v4.2d, v24.2d		\n"	\
+	"fadd	v5.2d, v5.2d, v26.2d		\n"	\
+	"fadd	v6.2d, v6.2d, v28.2d		\n"	\
+	"fadd	v7.2d, v7.2d, v30.2d		\n"
+
+#define KERNEL_F16_FINALIZE				\
+	"fadd	v0.2d, v0.2d, v1.2d		\n"	\
+	"fadd	v2.2d, v2.2d, v3.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v5.2d		\n"	\
+	"fadd	v6.2d, v6.2d, v7.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v2.2d		\n"	\
+	"fadd	v4.2d, v4.2d, v6.2d		\n"	\
+	"fadd	v0.2d, v0.2d, v4.2d		\n"	\
+	"faddp	"SUMF", v0.2d			\n"
+
+#define INIT_S						\
+	"lsl	"INC_X", "INC_X", #4		\n"
+
+#define KERNEL_S1					\
+	"ldr	q1, ["X"]			\n"	\
+	"add	"X", "X", "INC_X"		\n"	\
+	"fabs	v1.2d, v1.2d			\n"	\
+	"faddp	d1, v1.2d			\n"	\
+	"fadd	"SUMF", "SUMF", d1		\n"
+
+
+#if defined(SMP)
+extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n,
+	BLASLONG k, void *alpha, void *a, BLASLONG lda, void *b, BLASLONG ldb,
+	void *c, BLASLONG ldc, int (*function)(), int nthreads);
+#endif
+
+
+static FLOAT zasum_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	FLOAT  asum = 0.0 ;
+
+	if ( n < 0 )  return(asum);
+
+	__asm__ __volatile__ (
+	"	mov	"N", %[N_]			\n"
+	"	mov	"X", %[X_]			\n"
+	"	mov	"INC_X", %[INCX_]		\n"
+	"	fmov	"SUMF", "REG0"			\n"
+	"	fmov	d1, "REG0"			\n"
+	"	fmov	d2, "REG0"			\n"
+	"	fmov	d3, "REG0"			\n"
+	"	fmov	d4, "REG0"			\n"
+	"	fmov	d5, "REG0"			\n"
+	"	fmov	d6, "REG0"			\n"
+	"	fmov	d7, "REG0"			\n"
+	"	cmp	"N", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", xzr			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+	"	cmp	"INC_X", #1			\n"
+	"	bne	.Lasum_kernel_S_BEGIN		\n"
+
+	".Lasum_kernel_F_BEGIN:				\n"
+	"	asr	"J", "N", #4			\n"
+	"	cmp	"J", xzr			\n"
+	"	beq	.Lasum_kernel_F1		\n"
+
+	".align 5					\n"
+	".Lasum_kernel_F16:				\n"
+	"	"KERNEL_F16"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F16		\n"
+	"	"KERNEL_F16_FINALIZE"			\n"
+
+	".Lasum_kernel_F1:				\n"
+	"	ands	"J", "N", #15			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_F10:				\n"
+	"	"KERNEL_F1"				\n"
+	"	subs    "J", "J", #1			\n"
+	"	bne	.Lasum_kernel_F10		\n"
+	"	b	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S_BEGIN:				\n"
+	"	"INIT_S"				\n"
+	"	asr	"J", "N", #2			\n"
+	"	cmp	"J", xzr			\n"
+	"	ble	.Lasum_kernel_S1		\n"
+
+	".Lasum_kernel_S4:				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S4		\n"
+
+	".Lasum_kernel_S1:				\n"
+	"	ands	"J", "N", #3			\n"
+	"	ble	.Lasum_kernel_L999		\n"
+
+	".Lasum_kernel_S10:				\n"
+	"	"KERNEL_S1"				\n"
+	"	subs	"J", "J", #1			\n"
+	"	bne	.Lasum_kernel_S10		\n"
+
+	".Lasum_kernel_L999:				\n"
+	"	fmov	%[ASUM_], "SUMF"		\n"
+
+	: [ASUM_] "=r" (asum)		//%0
+	: [N_]    "r"  (n),		//%1
+	  [X_]    "r"  (x),		//%2
+	  [INCX_] "r"  (inc_x)		//%3
+	: "cc",
+	  "memory",
+	  "x0", "x1", "x2", "x3", "x4", "x5",
+	  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"
+	);
+
+	return asum;
+}
+
+#if defined(SMP)
+static int zasum_thread_function(BLASLONG n, BLASLONG dummy0,
+	BLASLONG dummy1, FLOAT dummy2, FLOAT *x, BLASLONG inc_x, FLOAT *y,
+	BLASLONG inc_y, FLOAT *result, BLASLONG dummy3)
+{
+	*result = zasum_compute(n, x, inc_x);
+
+	return 0;
+}
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+#if defined(SMP)
+	int nthreads;
+	FLOAT dummy_alpha;
+#endif
+	FLOAT asum = 0.0;
+
+#if defined(SMP)
+	nthreads = num_cpu_avail(1);
+
+	if (inc_x == 0)
+		nthreads = 1;
+
+	if (n <= 10000)
+		nthreads = 1;
+
+	if (nthreads == 1) {
+		asum = zasum_compute(n, x, inc_x);
+	} else {
+		int mode, i;
+		char result[MAX_CPU_NUMBER * sizeof(double) * 2];
+		FLOAT *ptr;
+
+		mode = BLAS_DOUBLE | BLAS_COMPLEX;
+
+		blas_level1_thread_with_return_value(mode, n, 0, 0, &dummy_alpha,
+				   x, inc_x, NULL, 0, result, 0,
+				   ( void *)zasum_thread_function, nthreads);
+
+		ptr = (FLOAT *)result;
+		for (i = 0; i < nthreads; i++) {
+			asum = asum + (*ptr);
+			ptr = (FLOAT *)(((char *)ptr) + sizeof(double) * 2);
+		}
+	}
+#else
+	asum = zasum_compute(n, x, inc_x);
+#endif
+
+	return asum;
+}

lapack/getrf/getrf_parallel.c

@@ -284,6 +284,7 @@ static int inner_advanced_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *
       }
     }
 
+    MB;
     for (i = 0; i < args -> nthreads; i++)
       job[mypos].working[i][CACHE_LINE_SIZE * bufferside] = (BLASLONG)buffer[bufferside];
 
@@ -324,6 +325,7 @@ static int inner_advanced_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *
 			   sa, (FLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside],
 			   c, lda, is, xxx);
 
+	  MB;
 	  if (is + min_i >= m) {
 	    job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
 	  }

param.h

@@ -2303,44 +2303,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define ZGEMM_DEFAULT_R 4096
 
 
-#define SYMV_P	16
-#endif
-
-#if defined(VULCAN)
-#define SNUMOPT		2
-#define DNUMOPT		2
-
-#define GEMM_DEFAULT_OFFSET_A 0
-#define GEMM_DEFAULT_OFFSET_B 0
-#define GEMM_DEFAULT_ALIGN 0x03fffUL
-
-#define SGEMM_DEFAULT_UNROLL_M  16
-#define SGEMM_DEFAULT_UNROLL_N  4
-
-#define DGEMM_DEFAULT_UNROLL_M  8
-#define DGEMM_DEFAULT_UNROLL_N  4
-
-#define CGEMM_DEFAULT_UNROLL_M  8
-#define CGEMM_DEFAULT_UNROLL_N  4
-
-#define ZGEMM_DEFAULT_UNROLL_M  4
-#define ZGEMM_DEFAULT_UNROLL_N  4
-
-#define SGEMM_DEFAULT_P	sgemm_p
-#define DGEMM_DEFAULT_P	dgemm_p
-#define CGEMM_DEFAULT_P 256
-#define ZGEMM_DEFAULT_P 128
-
-#define SGEMM_DEFAULT_Q sgemm_q
-#define DGEMM_DEFAULT_Q dgemm_q
-#define CGEMM_DEFAULT_Q 512
-#define ZGEMM_DEFAULT_Q 512
-
-#define SGEMM_DEFAULT_R sgemm_r
-#define DGEMM_DEFAULT_R dgemm_r
-#define CGEMM_DEFAULT_R 4096
-#define ZGEMM_DEFAULT_R 2048
-
 #define SYMV_P	16
 #endif
 
@@ -2462,7 +2424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SYMV_P	16
 #endif
 
-#if defined(THUNDERX2T99)
+#if defined(THUNDERX2T99) || defined(VULCAN)
 #define SNUMOPT		2
 #define DNUMOPT		2
 
@@ -2484,17 +2446,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define SGEMM_DEFAULT_P	sgemm_p
 #define DGEMM_DEFAULT_P	dgemm_p
-#define CGEMM_DEFAULT_P 256
+#define CGEMM_DEFAULT_P cgemm_p
 #define ZGEMM_DEFAULT_P 128
 
 #define SGEMM_DEFAULT_Q sgemm_q
 #define DGEMM_DEFAULT_Q dgemm_q
-#define CGEMM_DEFAULT_Q 512
+#define CGEMM_DEFAULT_Q cgemm_q
 #define ZGEMM_DEFAULT_Q 512
 
 #define SGEMM_DEFAULT_R sgemm_r
 #define DGEMM_DEFAULT_R dgemm_r
-#define CGEMM_DEFAULT_R 4096
+#define CGEMM_DEFAULT_R cgemm_r
 #define ZGEMM_DEFAULT_R 2048
 
 #define SYMV_P	16