optimized sgemm

kernel/power/sgemm_logic_16x8_power8.S

@@ -1,38 +1,3 @@
-/***************************************************************************
-Copyright (c) 2013-2016, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-/**************************************************************************************
-* 2016/04/02 Werner Saar (wernsaar@googlemail.com)
-* 	 BLASTEST 		: OK
-* 	 CTEST			: OK
-* 	 TEST			: OK
-* 	 LAPACK-TEST		: OK
-**************************************************************************************/
-
 	srawi.		J,	N,	3
 	ble		SGEMM_L8_END
 
@@ -40,35 +5,48 @@ SGEMM_L8_BEGIN:
 
 	mr		BO,	B
 	mr		BBO,	BBUFFER
-	slwi		T1,	K,	3
+	srawi.		T1,	K,	2
+	ble		SGEMM_L8_COPYB1
+
+
+SGEMM_L8_COPYB4:
 
-SGEMM_L8_COPYB:
+	dcbt		BO,	PRE
 	dcbtst		BBO,	PRE
+	COPYB_4x8
+	addic.		T1,	T1,	-1
+	ble		SGEMM_L8_COPYB1
 
-	lxvw4x		vs3,	o0,	BO
-	lxvw4x		vs11,	o16,	BO
-	xxspltw		vs4,	vs3,	0
-	xxspltw		vs5,	vs3,	1
-	xxspltw		vs6,	vs3,	2
-	xxspltw		vs7,	vs3,	3
-	xxspltw		vs12,	vs11,	0
-	xxspltw		vs13,	vs11,	1
-	xxspltw		vs14,	vs11,	2
-	xxspltw		vs15,	vs11,	3
-	stxvw4x		vs4,	o0,	BBO
-	stxvw4x		vs5,	o16,	BBO
-	stxvw4x		vs6,	o32,	BBO
-	stxvw4x		vs7,	o48,	BBO
-	addi		BO,	BO,	32
-	addi		BBO,	BBO,	64
-	stxvw4x		vs12,	o0,	BBO
-	stxvw4x		vs13,	o16,	BBO
-	stxvw4x		vs14,	o32,	BBO
-	stxvw4x		vs15,	o48,	BBO
-	addic.		T1,	T1,	-8
-	addi		BBO,	BBO,	64
+	dcbtst		BBO,	PRE
+	COPYB_4x8
+	addic.		T1,	T1,	-1
+	ble		SGEMM_L8_COPYB1
+
+	dcbtst		BBO,	PRE
+	COPYB_4x8
+	addic.		T1,	T1,	-1
+	ble		SGEMM_L8_COPYB1
 
-	bge		SGEMM_L8_COPYB
+	dcbtst		BBO,	PRE
+	COPYB_4x8
+	addic.		T1,	T1,	-1
+
+	bgt		SGEMM_L8_COPYB4
+
+SGEMM_L8_COPYB1:
+
+	andi.		T1,	K,	3
+	ble		SGEMM_L8_COPYB_END	
+
+SGEMM_L8_COPYB1_LOOP:
+
+
+	COPYB_1x8
+	addic.		T1,	T1,	-1
+
+	bgt		SGEMM_L8_COPYB1_LOOP
+
+SGEMM_L8_COPYB_END:
 
 	mr		CO,	C
 	mr		AO,	A
@@ -93,24 +71,24 @@ SGEMM_L8x16_LOOP_START:
 	LOAD8x16_1
 	dcbt		BO,	PRE
 	KERNEL8x16_I1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 
 	addic.		L,	L,	-2
@@ -122,24 +100,24 @@ SGEMM_L8x16_LOOP:
 
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 
 	addic.		L,	L,	-1
@@ -149,18 +127,15 @@ SGEMM_L8x16_LOOP_END:
 
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL8x16_2
 	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
 	KERNEL8x16_2
 
-	dcbt		BO,	PRE
 	KERNEL8x16_1
-	dcbt		BO,	PRE
 	dcbt		AO,	PRE
 	KERNEL8x16_2
 	KERNEL8x16_1

kernel/power/sgemm_macros_16x8_power8.S

@@ -1,38 +1,3 @@
-/***************************************************************************
-Copyright (c) 2013-2016, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-/**************************************************************************************
-* 2016/04/02 Werner Saar (wernsaar@googlemail.com)
-* 	 BLASTEST 		: OK
-* 	 CTEST			: OK
-* 	 TEST			: OK
-* 	 LAPACK-TEST		: OK
-**************************************************************************************/
-
 
 /**********************************************************************************************
 * Macros for N=8 and M=16
@@ -5886,3 +5851,145 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .endm
 
+
+
+
+
+.macro COPYB_4x8
+
+
+	lxvw4x		vs5,	o0,	BO
+	xxspltw		vs6,	vs5,	0
+	xxspltw		vs7,	vs5,	1
+	xxspltw		vs8,	vs5,	2
+	xxspltw		vs9,	vs5,	3
+
+	lxvw4x		vs10,	o16,	BO
+	xxspltw		vs11,	vs10,	0
+	xxspltw		vs12,	vs10,	1
+	xxspltw		vs13,	vs10,	2
+	xxspltw		vs14,	vs10,	3
+
+	lxvw4x		vs15,	o32,	BO
+	xxspltw		vs16,	vs15,	0
+	xxspltw		vs17,	vs15,	1
+	xxspltw		vs18,	vs15,	2
+	xxspltw		vs19,	vs15,	3
+
+	lxvw4x		vs20,	o48,	BO
+	xxspltw		vs21,	vs20,	0
+	xxspltw		vs22,	vs20,	1
+	xxspltw		vs23,	vs20,	2
+	xxspltw		vs24,	vs20,	3
+
+	addi		BO,	BO,	64
+	lxvw4x		vs35,	o0,	BO
+	xxspltw		vs36,	vs35,	0
+	xxspltw		vs37,	vs35,	1
+	xxspltw		vs38,	vs35,	2
+	xxspltw		vs39,	vs35,	3
+
+	lxvw4x		vs40,	o16,	BO
+	xxspltw		vs41,	vs40,	0
+	xxspltw		vs42,	vs40,	1
+	xxspltw		vs43,	vs40,	2
+	xxspltw		vs44,	vs40,	3
+
+	lxvw4x		vs45,	o32,	BO
+	xxspltw		vs46,	vs45,	0
+	xxspltw		vs47,	vs45,	1
+	xxspltw		vs48,	vs45,	2
+	xxspltw		vs49,	vs45,	3
+
+	lxvw4x		vs50,	o48,	BO
+	xxspltw		vs51,	vs50,	0
+	xxspltw		vs52,	vs50,	1
+	xxspltw		vs53,	vs50,	2
+	xxspltw		vs54,	vs50,	3
+
+	addi		BO,	BO,	64
+
+
+	stxvw4x		vs6,	o0,	BBO
+	stxvw4x		vs7,	o16,	BBO
+	stxvw4x		vs8,	o32,	BBO
+	stxvw4x		vs9,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs11,	o0,	BBO
+	stxvw4x		vs12,	o16,	BBO
+	stxvw4x		vs13,	o32,	BBO
+	stxvw4x		vs14,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs16,	o0,	BBO
+	stxvw4x		vs17,	o16,	BBO
+	stxvw4x		vs18,	o32,	BBO
+	stxvw4x		vs19,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs21,	o0,	BBO
+	stxvw4x		vs22,	o16,	BBO
+	stxvw4x		vs23,	o32,	BBO
+	stxvw4x		vs24,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs36,	o0,	BBO
+	stxvw4x		vs37,	o16,	BBO
+	stxvw4x		vs38,	o32,	BBO
+	stxvw4x		vs39,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs41,	o0,	BBO
+	stxvw4x		vs42,	o16,	BBO
+	stxvw4x		vs43,	o32,	BBO
+	stxvw4x		vs44,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs46,	o0,	BBO
+	stxvw4x		vs47,	o16,	BBO
+	stxvw4x		vs48,	o32,	BBO
+	stxvw4x		vs49,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs51,	o0,	BBO
+	stxvw4x		vs52,	o16,	BBO
+	stxvw4x		vs53,	o32,	BBO
+	stxvw4x		vs54,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+.endm
+
+
+.macro COPYB_1x8
+
+
+	lxvw4x		vs5,	o0,	BO
+	xxspltw		vs6,	vs5,	0
+	xxspltw		vs7,	vs5,	1
+	xxspltw		vs8,	vs5,	2
+	xxspltw		vs9,	vs5,	3
+
+	lxvw4x		vs10,	o16,	BO
+	xxspltw		vs11,	vs10,	0
+	xxspltw		vs12,	vs10,	1
+	xxspltw		vs13,	vs10,	2
+	xxspltw		vs14,	vs10,	3
+
+
+	addi		BO,	BO,	32
+
+	stxvw4x		vs6,	o0,	BBO
+	stxvw4x		vs7,	o16,	BBO
+	stxvw4x		vs8,	o32,	BBO
+	stxvw4x		vs9,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+	stxvw4x		vs11,	o0,	BBO
+	stxvw4x		vs12,	o16,	BBO
+	stxvw4x		vs13,	o32,	BBO
+	stxvw4x		vs14,	o48,	BBO
+
+	addi		BBO,	BBO,	64
+.endm
+

kernel/power/sgemm_tcopy_16_power8.S

@@ -136,7 +136,7 @@
 	add	B1, B1, B
 
 	li	PREA,  384
-	li	PREB,  384
+	addi	PREB,  M16, 128
 
 	li	o4,	4
 	li	o16,	16

kernel/power/sgemm_tcopy_logic_16_power8.S

@@ -19,6 +19,26 @@ SCOPYT_L4_BEGIN:
 
 SCOPYT_L4x16_LOOP:
 
+	dcbtst		BO,	M16
+	dcbtst		BO,	PREB
+	dcbt		A0,	PREA
+	dcbt		A1,	PREA
+	dcbt		A2,	PREA
+	dcbt		A3,	PREA
+	COPY_4x16
+
+	addi		A0,	A0,	16*SIZE
+	addi		A1,	A1,	16*SIZE
+	addi		A2,	A2,	16*SIZE
+	addi		A3,	A3,	16*SIZE
+	add		BO,	BO,	M16
+
+	addic.		J,	J,	-1
+	ble		SCOPYT_L4x8_BEGIN
+
+
+	dcbtst		BO,	M16
+	dcbtst		BO,	PREB
 	COPY_4x16
 
 	addi		A0,	A0,	16*SIZE