OpenBLAS/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S

/*******************************************************************************
Copyright (c) 2016, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/

#define ASSEMBLER
#include "common.h"

/*                   X0          X1          X2          s0         X3        x4       x5           x6 */
/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/

#define origM		x0
#define origN		x1
#define origK		x2
#define origPA		x3
#define origPB		x4
#define pC		x5
#define LDC		x6
#define temp		x7
#define counterL	x8
#define counterI	x9
#define counterJ	x10
#define pB		x11
#define pCRow0		x12
#define pCRow1		x13
#define pCRow2		x14
#define pCRow3		x15
#define pA		x16
#define alpha		x17

#define alpha0		d10
#define alphaV0		v10.d[0]

#define A_PRE_SIZE	x20
#define B_PRE_SIZE	x21
#define C_PRE_SIZE	x22

#define A_PRE_SIZE_64	x23
#define B_PRE_SIZE_64	x24


// 00 origM
// 01 origN
// 02 origK
// 03 origPA
// 04 origPB
// 05 pC
// 06 origLDC -> LDC
// 07 temp
// 08 counterL
// 09 counterI
// 10 counterJ
// 11 pB
// 12 pCRow0
// 13 pCRow1
// 14 pCRow2
// 15 pCRow3
// 16 pA
// 17
// 18 must save
// 19 must save
// 20 must save
// 21 must save
// 22 must save
// 23 must save
// 24 must save
// 25 must save
// 26 must save
// 27 must save
// 28 must save
// 29 frame
// 30 link
// 31 sp

//v00 ALPHA -> pA0_0, pA0_1
//v01 pA0_2, pA0_3
//v02 pA0_4, pA0_5
//v03 pA0_6, pA0_7
//v04 pA1_0, pA1_1
//v05 pA1_2, pA1_3
//v06 pA1_4, pA1_5
//v07 pA1_6, pA1_7
//v08 must save pB0_0
//v09 must save pB0_1
//v10 must save pB0_2 --> ALPHA0
//v11 must save pB0_3
//v12 must save pB1_0
//v13 must save pB1_1
//v14 must save pB1_2
//v15 must save pB1_3
//v16 must save C00, C01
//v17 must save C02, C03
//v18 C04, C05
//v19 C06, C07
//v20 C10, C11
//v21 C12, C13
//v22 C14, C15
//v23 C16, C17
//v24 C20, C21
//v25 C22, C23
//v26 C24, C25
//v27 C26, C27
//v28 C30, C31
//v29 C32, C33
//v30 C34, C35
//v31 C36, C37

/*******************************************************************************
* Macro definitions
*******************************************************************************/

.macro INIT8x4
	fmov		d16, xzr
	fmov		d17, xzr
	fmov		d18, d16
	fmov		d19, xzr
	fmov		d20, xzr
	fmov		d21, d16
	fmov		d22, d17
	fmov		d23, d18
	fmov		d24, xzr
	fmov		d25, d16
	fmov		d26, d17
	fmov		d27, d18
	fmov		d28, xzr
	fmov		d29, d16
	fmov		d30, d17
	fmov		d31, d18
.endm

.macro KERNEL8x4_I
	ldp	q0, q1, [pA]
	ldp	q8, q9, [pB]
	ldp	q2, q3, [pA, #32]
	ldp	q4, q5, [pA, #64]
	ldp	q12, q13, [pB, #32]
	ldp	q6, q7, [pA, #96]

	fmul	v16.2d, v0.2d, v8.d[0]
	fmul	v20.2d, v0.2d, v8.d[1]
	fmul	v17.2d, v1.2d, v8.d[0]
	fmul	v21.2d, v1.2d, v8.d[1]

	add	pA, pA, #128
	add	pB, pB, #64

	fmul	v24.2d, v0.2d, v9.d[0]
	fmul	v28.2d, v0.2d, v9.d[1]
	fmul	v25.2d, v1.2d, v9.d[0]
	fmul	v29.2d, v1.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]

	fmul	v18.2d, v2.2d, v8.d[0]
	fmul	v22.2d, v2.2d, v8.d[1]
	fmul	v26.2d, v2.2d, v9.d[0]
	fmul	v30.2d, v2.2d, v9.d[1]

	fmul	v19.2d, v3.2d, v8.d[0]
	fmul	v27.2d, v3.2d, v9.d[0]
	fmul	v31.2d, v3.2d, v9.d[1]
	fmul	v23.2d, v3.2d, v8.d[1]
.endm

.macro KERNEL8x4_M1_M2

	ldp	q12, q13, [pB]
	ldp	q4, q5, [pA]
	ldp	q6, q7, [pA, #32]

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v24.2d, v0.2d, v9.d[0]
	fmla	v28.2d, v0.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	fmla	v17.2d, v1.2d, v8.d[0]
	fmla	v25.2d, v1.2d, v9.d[0]
	fmla	v21.2d, v1.2d, v8.d[1]
	fmla	v29.2d, v1.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]

	fmla	v18.2d, v2.2d, v8.d[0]
	fmla	v22.2d, v2.2d, v8.d[1]
	fmla	v26.2d, v2.2d, v9.d[0]
	fmla	v30.2d, v2.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, #3840]

	fmla	v19.2d, v3.2d, v8.d[0]
	fmla	v23.2d, v3.2d, v8.d[1]
	fmla	v27.2d, v3.2d, v9.d[0]
	fmla	v31.2d, v3.2d, v9.d[1]


	ldp	q8, q9, [pB, #32]
	ldp	q0, q1, [pA, #64]
	ldp	q2, q3, [pA, #96]

	fmla	v16.2d, v4.2d, v12.d[0]
	fmla	v20.2d, v4.2d, v12.d[1]
	fmla	v24.2d, v4.2d, v13.d[0]
	fmla	v28.2d, v4.2d, v13.d[1]

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	fmla	v17.2d, v5.2d, v12.d[0]
	fmla	v25.2d, v5.2d, v13.d[0]
	fmla	v21.2d, v5.2d, v12.d[1]
	fmla	v29.2d, v5.2d, v13.d[1]

	fmla	v18.2d, v6.2d, v12.d[0]
	fmla	v22.2d, v6.2d, v12.d[1]
	fmla	v26.2d, v6.2d, v13.d[0]
	fmla	v30.2d, v6.2d, v13.d[1]

	add	pB, pB, #64
	add	pA, pA, #128

	fmla	v19.2d, v7.2d, v12.d[0]
	fmla	v23.2d, v7.2d, v12.d[1]
	fmla	v27.2d, v7.2d, v13.d[0]
	fmla	v31.2d, v7.2d, v13.d[1]
.endm


.macro KERNEL8x4_M1
	ldp	q12, q13, [pB]
	ldp	q4, q5, [pA]
	ldp	q6, q7, [pA, #32]

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v24.2d, v0.2d, v9.d[0]
	fmla	v28.2d, v0.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	fmla	v17.2d, v1.2d, v8.d[0]
	fmla	v25.2d, v1.2d, v9.d[0]
	fmla	v21.2d, v1.2d, v8.d[1]
	fmla	v29.2d, v1.2d, v9.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]

	fmla	v18.2d, v2.2d, v8.d[0]
	fmla	v22.2d, v2.2d, v8.d[1]
	fmla	v26.2d, v2.2d, v9.d[0]
	fmla	v30.2d, v2.2d, v9.d[1]

	add	pB, pB, #32
	add	pA, pA, #64

	fmla	v19.2d, v3.2d, v8.d[0]
	fmla	v23.2d, v3.2d, v8.d[1]
	fmla	v27.2d, v3.2d, v9.d[0]
	fmla	v31.2d, v3.2d, v9.d[1]
.endm

.macro KERNEL8x4_M2
	ldp	q8, q9, [pB]
	ldp	q0, q1, [pA]
	ldp	q2, q3, [pA, #32]

	fmla	v16.2d, v4.2d, v12.d[0]
	fmla	v20.2d, v4.2d, v12.d[1]
	fmla	v24.2d, v4.2d, v13.d[0]
	fmla	v28.2d, v4.2d, v13.d[1]

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	fmla	v17.2d, v5.2d, v12.d[0]
	fmla	v25.2d, v5.2d, v13.d[0]
	fmla	v21.2d, v5.2d, v12.d[1]
	fmla	v29.2d, v5.2d, v13.d[1]

	fmla	v18.2d, v6.2d, v12.d[0]
	fmla	v22.2d, v6.2d, v12.d[1]
	fmla	v26.2d, v6.2d, v13.d[0]
	fmla	v30.2d, v6.2d, v13.d[1]

	add	pB, pB, #32
	add	pA, pA, #64

	fmla	v19.2d, v7.2d, v12.d[0]
	fmla	v23.2d, v7.2d, v12.d[1]
	fmla	v27.2d, v7.2d, v13.d[0]
	fmla	v31.2d, v7.2d, v13.d[1]
.endm

.macro KERNEL8x4_E
	fmla	v16.2d, v4.2d, v12.d[0]
	fmla	v20.2d, v4.2d, v12.d[1]
	fmla	v24.2d, v4.2d, v13.d[0]
	fmla	v28.2d, v4.2d, v13.d[1]

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	fmla	v17.2d, v5.2d, v12.d[0]
	fmla	v25.2d, v5.2d, v13.d[0]
	fmla	v21.2d, v5.2d, v12.d[1]
	fmla	v29.2d, v5.2d, v13.d[1]

	fmla	v18.2d, v6.2d, v12.d[0]
	fmla	v22.2d, v6.2d, v12.d[1]
	fmla	v26.2d, v6.2d, v13.d[0]
	fmla	v30.2d, v6.2d, v13.d[1]

	fmla	v19.2d, v7.2d, v12.d[0]
	fmla	v23.2d, v7.2d, v12.d[1]
	fmla	v27.2d, v7.2d, v13.d[0]
	fmla	v31.2d, v7.2d, v13.d[1]
.endm

.macro KERNEL8x4_SUB
	ldp	q0, q1, [pA]
	ldp	q8, q9, [pB]
	ldp	q2, q3, [pA, #32]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v17.2d, v1.2d, v8.d[0]
	fmla	v21.2d, v1.2d, v8.d[1]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]

	fmla	v24.2d, v0.2d, v9.d[0]
	fmla	v28.2d, v0.2d, v9.d[1]
	fmla	v25.2d, v1.2d, v9.d[0]
	fmla	v29.2d, v1.2d, v9.d[1]

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	fmla	v18.2d, v2.2d, v8.d[0]
	fmla	v22.2d, v2.2d, v8.d[1]
	fmla	v26.2d, v2.2d, v9.d[0]
	fmla	v30.2d, v2.2d, v9.d[1]

	add	pB, pB, #32
	add	pA, pA, #64

	fmla	v19.2d, v3.2d, v8.d[0]
	fmla	v27.2d, v3.2d, v9.d[0]
	fmla	v31.2d, v3.2d, v9.d[1]
	fmla	v23.2d, v3.2d, v8.d[1]
.endm

.macro SAVE8x4
	fmov	alpha0, alpha
	ldr	q0, [pCRow0]
	ldr	q1, [pCRow0, #16]
	ldr	q2, [pCRow0, #32]
	ldr	q3, [pCRow0, #48]
	ldr	q4, [pCRow1]
	ldr	q5, [pCRow1, #16]
	ldr	q6, [pCRow1, #32]
	ldr	q7, [pCRow1, #48]

	fmla	v0.2d, v16.2d, alphaV0
	fmla	v1.2d, v17.2d, alphaV0
	stp 	q0, q1, [pCRow0]

	fmla	v2.2d, v18.2d, alphaV0
	fmla	v3.2d, v19.2d, alphaV0
	stp 	q2, q3, [pCRow0, #32]
	ldr	q0, [pCRow2]
	ldr	q1, [pCRow2, #16]

	fmla	v4.2d, v20.2d, alphaV0
	fmla	v5.2d, v21.2d, alphaV0
	stp 	q4, q5, [pCRow1]
	ldr	q2, [pCRow2, #32]
	ldr	q3, [pCRow2, #48]

	fmla	v6.2d, v22.2d, alphaV0
	fmla	v7.2d, v23.2d, alphaV0
	stp 	q6, q7, [pCRow1, #32]
	ldr	q4, [pCRow3]
	ldr	q5, [pCRow3, #16]

	fmla	v0.2d, v24.2d, alphaV0
	fmla	v1.2d, v25.2d, alphaV0
	stp 	q0, q1, [pCRow2]
	ldr	q6, [pCRow3, #32]
	ldr	q7, [pCRow3, #48]

	fmla	v2.2d, v26.2d, alphaV0
	fmla	v3.2d, v27.2d, alphaV0
	stp 	q2, q3, [pCRow2, #32]

	fmla	v4.2d, v28.2d, alphaV0
	fmla	v5.2d, v29.2d, alphaV0
	stp 	q4, q5, [pCRow3]

	fmla	v6.2d, v30.2d, alphaV0
	fmla	v7.2d, v31.2d, alphaV0
	stp 	q6, q7, [pCRow3, #32]

	add	pCRow0, pCRow0, #64
	add	pCRow1, pCRow1, #64
	add	pCRow2, pCRow2, #64
	add	pCRow3, pCRow3, #64
.endm

/******************************************************************************/

.macro INIT4x4
	fmov		d16, xzr
	fmov		d17, d16
	fmov		d20, d17
	fmov		d21, d16
	fmov		d24, d17
	fmov		d25, d16
	fmov		d28, d17
	fmov		d29, d16
.endm

.macro KERNEL4x4_SUB
	ld1	{v8.2d, v9.2d}, [pB]
	add	pB, pB, #32
	ld1	{v0.2d, v1.2d}, [pA]
	add	pA, pA, #32

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v29.2d, v1.2d, v9.d[1]

	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v25.2d, v1.2d, v9.d[0]

	fmla	v24.2d, v0.2d, v9.d[0]
	fmla	v21.2d, v1.2d, v8.d[1]

	fmla	v28.2d, v0.2d, v9.d[1]
	fmla	v17.2d, v1.2d, v8.d[0]
.endm

.macro SAVE4x4
	fmov	alpha0, alpha

	ld1	{v8.2d, v9.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	fmla	v9.2d, v17.2d, alphaV0
	st1 	{v8.2d, v9.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #32

	ld1	{v12.2d, v13.2d}, [pCRow1]
	fmla	v12.2d, v20.2d, alphaV0
	fmla	v13.2d, v21.2d, alphaV0
	st1 	{v12.2d, v13.2d}, [pCRow1]

	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #32

	ld1	{v8.2d, v9.2d}, [pCRow2]
	fmla	v8.2d, v24.2d, alphaV0
	fmla	v9.2d, v25.2d, alphaV0
	st1 	{v8.2d, v9.2d}, [pCRow2]

	prfm	PLDL2KEEP, [pCRow2, C_PRE_SIZE]
	add	pCRow2, pCRow2, #32

	ld1	{v12.2d, v13.2d}, [pCRow3]
	fmla	v12.2d, v28.2d, alphaV0
	fmla	v13.2d, v29.2d, alphaV0
	st1 	{v12.2d, v13.2d}, [pCRow3]

	prfm	PLDL2KEEP, [pCRow3, C_PRE_SIZE]
	add	pCRow3, pCRow3, #32
.endm

/******************************************************************************/


.macro INIT2x4
	fmov		d16, xzr
	fmov		d20, d16
	fmov		d24, d20
	fmov		d28, d16
.endm

.macro KERNEL2x4_SUB
	ld1	{v8.2d, v9.2d}, [pB]
	add	pB, pB, #32
	ld1	{v0.2d}, [pA]
	add	pA, pA, #16

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v24.2d, v0.2d, v9.d[0]
	fmla	v28.2d, v0.2d, v9.d[1]
.endm

.macro SAVE2x4
	fmov	alpha0, alpha

	ld1	{v8.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	st1	{v8.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #16

	ld1	{v12.2d}, [pCRow1]
	fmla	v12.2d, v20.2d, alphaV0
	st1	{v12.2d}, [pCRow1]

	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #16

	ld1	{v8.2d}, [pCRow2]
	fmla	v8.2d, v24.2d, alphaV0
	st1	{v8.2d}, [pCRow2]

	prfm	PLDL2KEEP, [pCRow2, C_PRE_SIZE]
	add	pCRow2, pCRow2, #16

	ld1	{v12.2d}, [pCRow3]
	fmla	v12.2d, v28.2d, alphaV0
	st1	{v12.2d}, [pCRow3]

	prfm	PLDL2KEEP, [pCRow3, C_PRE_SIZE]
	add	pCRow3, pCRow3, #16
.endm

/******************************************************************************/

.macro INIT1x4
	fmov		d16, xzr
	fmov		d20, d16
.endm

.macro KERNEL1x4_SUB
	ldr	d0, [pA]
	add	pA, pA, #8

	ld1	{v8.2d, v9.2d}, [pB]
	add	pB, pB, #32

	fmla	v16.2d, v8.2d, v0.d[0]
	fmla	v20.2d, v9.2d, v0.d[0]
.endm

.macro SAVE1x4
	fmov	alpha0, alpha

	ld1	{v8.d}[0], [pCRow0]
	ld1	{v8.d}[1], [pCRow1]
	fmla	v8.2d, v16.2d, alphaV0
	st1	{v8.d}[0], [pCRow0]
	st1	{v8.d}[1], [pCRow1]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #8
	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #8

	ld1	{v12.d}[0], [pCRow2]
	ld1	{v12.d}[1], [pCRow3]
	fmla	v12.2d, v20.2d, alphaV0
	st1	{v12.d}[0], [pCRow2]
	st1	{v12.d}[1], [pCRow3]

	prfm	PLDL2KEEP, [pCRow2, C_PRE_SIZE]
	add	pCRow2, pCRow2, #8
	prfm	PLDL2KEEP, [pCRow3, C_PRE_SIZE]
	add	pCRow3, pCRow3, #8
.endm

/******************************************************************************/

.macro INIT8x2
	fmov	d16, xzr
	fmov	d17, xzr
	fmov	d18, d16
	fmov	d19, d17
	fmov	d20, xzr
	fmov	d21, d16
	fmov	d22, d17
	fmov	d23, d18
.endm

.macro KERNEL8x2_SUB
	ld1	{v0.2d, v1.2d}, [pA]
	add	pA, pA, #32
	ld1	{v8.2d}, [pB]
	add	pB, pB, #16
	ld1	{v2.2d, v3.2d}, [pA]
	add	pA, pA, #32

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v17.2d, v1.2d, v8.d[0]
	fmla	v18.2d, v2.2d, v8.d[0]
	fmla	v19.2d, v3.2d, v8.d[0]

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v21.2d, v1.2d, v8.d[1]
	fmla	v22.2d, v2.2d, v8.d[1]
	fmla	v23.2d, v3.2d, v8.d[1]
.endm

.macro SAVE8x2
	fmov	alpha0, alpha

	ld1	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
	fmla	v0.2d, v16.2d, alphaV0
	fmla	v1.2d, v17.2d, alphaV0
	fmla	v2.2d, v18.2d, alphaV0
	fmla	v3.2d, v19.2d, alphaV0
	st1 	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #64

	ld1	{v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
	fmla	v4.2d, v20.2d, alphaV0
	fmla	v5.2d, v21.2d, alphaV0
	fmla	v6.2d, v22.2d, alphaV0
	fmla	v7.2d, v23.2d, alphaV0
	st1 	{v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]

	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #64
.endm

/******************************************************************************/

.macro INIT4x2
	fmov	d16, xzr
	fmov	d17, d16
	fmov	d20, d17
	fmov	d21, d16
.endm

.macro KERNEL4x2_SUB
	ld1	{v8.2d}, [pB]
	add	pB, pB, #16
	ld1	{v0.2d, v1.2d}, [pA]
	add	pA, pA, #32

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v17.2d, v1.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
	fmla	v21.2d, v1.2d, v8.d[1]
.endm

.macro SAVE4x2
	fmov	alpha0, alpha

	ld1	{v8.2d, v9.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	fmla	v9.2d, v17.2d, alphaV0
	st1	{v8.2d, v9.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #32

	ld1	{v12.2d, v13.2d}, [pCRow1]
	fmla	v12.2d, v20.2d, alphaV0
	fmla	v13.2d, v21.2d, alphaV0
	st1	{v12.2d, v13.2d}, [pCRow1]

	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #32
.endm

/******************************************************************************/

.macro INIT2x2
	fmov		d16, xzr
	fmov		d20, d16
.endm

.macro KERNEL2x2_SUB
	ld1	{v8.2d}, [pB]
	add	pB, pB, #16

	ld1	{v0.2d}, [pA]
	add	pA, pA, #16

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v20.2d, v0.2d, v8.d[1]
.endm

.macro SAVE2x2
	fmov	alpha0, alpha

	ld1	{v8.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	st1	{v8.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #16

	ld1	{v12.2d}, [pCRow1]
	fmla	v12.2d, v20.2d, alphaV0
	st1	{v12.2d}, [pCRow1]

	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #16
.endm

/******************************************************************************/

.macro INIT1x2
	fmov		d16, xzr
.endm

.macro KERNEL1x2_SUB
	ld1	{v8.2d} , [pB]
	add	pB , pB, #16

	ldr	d0 , [pA]
	add	pA, pA, #8

	fmla	v16.2d, v8.2d, v0.d[0]
.endm

.macro SAVE1x2
	fmov	alpha0, alpha

	ld1	{v8.d}[0], [pCRow0]
	ld1	{v8.d}[1], [pCRow1]
	fmla	v8.2d, v16.2d, alphaV0
	st1	{v8.d}[0], [pCRow0]
	st1	{v8.d}[1], [pCRow1]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #8
	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	add	pCRow1, pCRow1, #8
.endm

/******************************************************************************/

.macro INIT8x1
	fmov	d16, xzr
	fmov	d17, xzr
	fmov	d18, d16
	fmov	d19, d17
.endm

.macro KERNEL8x1_SUB
	ld1	{v0.2d, v1.2d}, [pA]
	add	pA , pA, #32

	ldr	d8, [pB]
	add	pB , pB, #8

	ld1	{v2.2d, v3.2d}, [pA]
	add	pA, pA, #32

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v17.2d, v1.2d, v8.d[0]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	fmla	v18.2d, v2.2d, v8.d[0]
	fmla	v19.2d, v3.2d, v8.d[0]
.endm

.macro SAVE8x1
	fmov	alpha0, alpha

	ld1	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
	fmla	v0.2d, v16.2d, alphaV0
	fmla	v1.2d, v17.2d, alphaV0
	fmla	v2.2d, v18.2d, alphaV0
	fmla	v3.2d, v19.2d, alphaV0
	st1 	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #64
.endm


/******************************************************************************/

.macro INIT4x1
	fmov	d16, xzr
	fmov	d17, d16
.endm

.macro KERNEL4x1_SUB
	ldr	d8, [pB]
	add	pB , pB, #8

	ld1	{v0.2d, v1.2d}, [pA]
	add	pA , pA, #32

	fmla	v16.2d, v0.2d, v8.d[0]
	fmla	v17.2d, v1.2d, v8.d[0]
.endm

.macro SAVE4x1
	fmov	alpha0, alpha

	ld1	{v8.2d, v9.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	fmla	v9.2d, v17.2d, alphaV0
	st1	{v8.2d, v9.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #32
.endm


/******************************************************************************/

.macro INIT2x1
	fmov		d16, xzr
.endm

.macro KERNEL2x1_SUB
	ldr	d8, [pB]
	add	pB , pB, #8

	ld1	{v0.2d}, [pA]
	add	pA , pA, #16

	fmla	v16.2d, v0.2d, v8.d[0]
.endm

.macro SAVE2x1
	fmov	alpha0, alpha

	ld1	{v8.2d}, [pCRow0]
	fmla	v8.2d, v16.2d, alphaV0
	st1	{v8.2d}, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #16
.endm

/******************************************************************************/

.macro INIT1x1
	fmov	d16, xzr
.endm

.macro KERNEL1x1_SUB
	ldr	d8, [pB]
	add	pB , pB, #8

	ldr	d0, [pA]
	add	pA , pA, #8

	fmadd 	d16, d0, d8, d16
.endm

.macro SAVE1x1
	fmov	alpha0, alpha
	ldr	d8, [pCRow0]
	fmadd	d8, d16, alpha0, d8
	str 	d8, [pCRow0]

	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	add	pCRow0, pCRow0, #8
.endm

.macro KERNEL8x4_M1_M2_x1
	KERNEL8x4_M1_M2
.endm

.macro KERNEL8x4_M1_M2_x2
	KERNEL8x4_M1_M2_x1
	KERNEL8x4_M1_M2_x1
.endm

.macro KERNEL8x4_M1_M2_x4
	KERNEL8x4_M1_M2_x2
	KERNEL8x4_M1_M2_x2
.endm

.macro KERNEL8x4_M1_M2_x8
	KERNEL8x4_M1_M2_x4
	KERNEL8x4_M1_M2_x4
.endm

.macro KERNEL8x4_M1_M2_x16
	KERNEL8x4_M1_M2_x8
	KERNEL8x4_M1_M2_x8
.endm

.macro KERNEL8x4_M1_M2_x32
	KERNEL8x4_M1_M2_x16
	KERNEL8x4_M1_M2_x16
.endm

.macro KERNEL8x4_M1_M2_x64
	KERNEL8x4_M1_M2_x32
	KERNEL8x4_M1_M2_x32
.endm

/*******************************************************************************
* End of macro definitions
*******************************************************************************/

	PROLOGUE

	.align 5
	add	sp, sp, #-(11 * 16)
	stp	d8, d9, [sp, #(0 * 16)]
	stp	d10, d11, [sp, #(1 * 16)]
	stp	d12, d13, [sp, #(2 * 16)]
	stp	d14, d15, [sp, #(3 * 16)]
	stp	d16, d17, [sp, #(4 * 16)]
	stp	x18, x19, [sp, #(5 * 16)]
	stp	x20, x21, [sp, #(6 * 16)]
	stp	x22, x23, [sp, #(7 * 16)]
	stp	x24, x25, [sp, #(8 * 16)]
	stp	x26, x27, [sp, #(9 * 16)]
	str	x28, [sp, #(10 * 16)]

	prfm	PLDL1KEEP, [origPB]
	prfm	PLDL1KEEP, [origPA]

	mov	A_PRE_SIZE, #3584
	mov	B_PRE_SIZE, #512
	mov	C_PRE_SIZE, #128
	add	A_PRE_SIZE_64, A_PRE_SIZE, #64
	add	B_PRE_SIZE_64, B_PRE_SIZE, #64

	fmov	alpha, d0

	lsl	LDC, LDC, #3			// ldc = ldc * 8

	mov	pB, origPB

	mov	counterJ, origN
	asr 	counterJ, counterJ, #2		// J = J / 4
	cmp 	counterJ, #0
	ble	.Ldgemm_kernel_L2_BEGIN

/******************************************************************************/

	.align 5
.Ldgemm_kernel_L4_BEGIN:
	mov	pCRow0, pC
	add	pCRow1, pCRow0, LDC
	add	pCRow2, pCRow1, LDC
	add	pCRow3, pCRow2, LDC

	add	pC, pCRow3, LDC

	mov	pA, origPA			// pA = start of A array

.Ldgemm_kernel_L4_M8_BEGIN:

	mov	counterI, origM
	asr 	counterI, counterI, #3		// counterI = counterI / 8
	cmp 	counterI, #0
	ble	.Ldgemm_kernel_L4_M4_BEGIN

	.align 5
.Ldgemm_kernel_L4_M8_20:

	mov	pB, origPB

	asr 	counterL , origK, #7		// L = K / 128
	cmp	counterL , #2			// is there at least 4 to do?
	blt	.Ldgemm_kernel_L4_M8_32

	KERNEL8x4_I
	KERNEL8x4_M2
	KERNEL8x4_M1_M2_x32
	KERNEL8x4_M1_M2_x16
	KERNEL8x4_M1_M2_x8
	KERNEL8x4_M1_M2_x4
	KERNEL8x4_M1_M2_x2
	KERNEL8x4_M1_M2_x1

	subs	counterL, counterL, #2		// subtract 2
	ble	.Ldgemm_kernel_L4_M8_22a

	.align 5
.Ldgemm_kernel_L4_M8_22:

	KERNEL8x4_M1_M2_x64

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M8_22

	.align 5
.Ldgemm_kernel_L4_M8_22a:

	KERNEL8x4_M1_M2_x32
	KERNEL8x4_M1_M2_x16
	KERNEL8x4_M1_M2_x8
	KERNEL8x4_M1_M2_x4
	KERNEL8x4_M1_M2_x2
	KERNEL8x4_M1_M2_x1
	KERNEL8x4_M1
	KERNEL8x4_E

	b	 .Ldgemm_kernel_L4_M8_44

	.align 5
.Ldgemm_kernel_L4_M8_32:

	tst	counterL, #1
	ble	.Ldgemm_kernel_L4_M8_40

	KERNEL8x4_I
	KERNEL8x4_M2
	KERNEL8x4_M1_M2_x32
	KERNEL8x4_M1_M2_x16
	KERNEL8x4_M1_M2_x8
	KERNEL8x4_M1_M2_x4
	KERNEL8x4_M1_M2_x2
	KERNEL8x4_M1
	KERNEL8x4_E

	b	.Ldgemm_kernel_L4_M8_44

.Ldgemm_kernel_L4_M8_40:

	INIT8x4

.Ldgemm_kernel_L4_M8_44:

	ands	counterL , origK, #127
	ble	.Ldgemm_kernel_L4_M8_100

	.align 5
.Ldgemm_kernel_L4_M8_46:

	KERNEL8x4_SUB

	subs	counterL, counterL, #1
	bne	.Ldgemm_kernel_L4_M8_46

.Ldgemm_kernel_L4_M8_100:
	prfm	PLDL2KEEP, [pCRow0, C_PRE_SIZE]
	prfm	PLDL2KEEP, [pCRow1, C_PRE_SIZE]
	prfm	PLDL2KEEP, [pCRow2, C_PRE_SIZE]
	prfm	PLDL2KEEP, [pCRow3, C_PRE_SIZE]
	prfm	PLDL1KEEP, [pA]
	prfm	PLDL1KEEP, [pA, #64]
	prfm	PLDL1KEEP, [origPB]

	SAVE8x4

.Ldgemm_kernel_L4_M8_END:
	subs	counterI, counterI, #1
	bne	.Ldgemm_kernel_L4_M8_20

.Ldgemm_kernel_L4_M4_BEGIN:

	mov	counterI, origM
	tst	counterI , #7
	ble	.Ldgemm_kernel_L4_END

	tst	counterI, #4
	ble	.Ldgemm_kernel_L4_M2_BEGIN

.Ldgemm_kernel_L4_M4_20:

	INIT4x4

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L4_M4_40

	.align 5
.Ldgemm_kernel_L4_M4_22:

	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M4_22

.Ldgemm_kernel_L4_M4_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L4_M4_100

.Ldgemm_kernel_L4_M4_42:

	KERNEL4x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M4_42

.Ldgemm_kernel_L4_M4_100:

	SAVE4x4

.Ldgemm_kernel_L4_M4_END:

.Ldgemm_kernel_L4_M2_BEGIN:

	mov	counterI, origM
	tst	counterI , #3
	ble	.Ldgemm_kernel_L4_END

	tst	counterI, #2			// counterI = counterI / 2
	ble	.Ldgemm_kernel_L4_M1_BEGIN

.Ldgemm_kernel_L4_M2_20:

	INIT2x4

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L4_M2_40

	.align 5
.Ldgemm_kernel_L4_M2_22:

	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x4_SUB

	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x4_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M2_22


.Ldgemm_kernel_L4_M2_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L4_M2_100

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]
.Ldgemm_kernel_L4_M2_42:

	KERNEL2x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M2_42

.Ldgemm_kernel_L4_M2_100:

	SAVE2x4

.Ldgemm_kernel_L4_M2_END:


.Ldgemm_kernel_L4_M1_BEGIN:

	tst	counterI, #1			// counterI = counterI % 2
	ble	.Ldgemm_kernel_L4_END

.Ldgemm_kernel_L4_M1_20:

	INIT1x4

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L4_M1_40

	.align 5
.Ldgemm_kernel_L4_M1_22:
	KERNEL1x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x4_SUB
	KERNEL1x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x4_SUB

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	KERNEL1x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x4_SUB
	KERNEL1x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x4_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M1_22


.Ldgemm_kernel_L4_M1_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L4_M1_100

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
.Ldgemm_kernel_L4_M1_42:

	KERNEL1x4_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L4_M1_42

.Ldgemm_kernel_L4_M1_100:

	SAVE1x4

.Ldgemm_kernel_L4_END:

	lsl	temp, origK, #5
	add	origPB, origPB, temp		// B = B + K * 4 * 8

	subs	counterJ, counterJ , #1		// j--
	bgt	.Ldgemm_kernel_L4_BEGIN


/******************************************************************************/

.Ldgemm_kernel_L2_BEGIN:   // less than 2 left in N direction

	mov	counterJ , origN
	tst	counterJ , #3
	ble	.Ldgemm_kernel_L999   // error, N was less than 4?

	tst	counterJ , #2
	ble	.Ldgemm_kernel_L1_BEGIN

	mov	pCRow0, pC
	add	pCRow1, pCRow0, LDC

	add	pC, pCRow1, LDC

	mov	pA, origPA			// pA = A

.Ldgemm_kernel_L2_M8_BEGIN:

	mov	counterI, origM
	asr 	counterI, counterI, #3		// counterI = counterI / 8
	cmp	counterI, #0
	ble	.Ldgemm_kernel_L2_M4_BEGIN

	.align 5
.Ldgemm_kernel_L2_M8_20:

	INIT8x2

	mov	pB, origPB

	asr	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL,#0
	ble	.Ldgemm_kernel_L2_M8_40

	.align 5
.Ldgemm_kernel_L2_M8_22:
	KERNEL8x2_SUB
	KERNEL8x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL8x2_SUB
	KERNEL8x2_SUB

	KERNEL8x2_SUB
	KERNEL8x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL8x2_SUB
	KERNEL8x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M8_22

.Ldgemm_kernel_L2_M8_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L2_M8_100

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE_64]
.Ldgemm_kernel_L2_M8_42:

	KERNEL8x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M8_42

.Ldgemm_kernel_L2_M8_100:

	SAVE8x2

.Ldgemm_kernel_L2_M8_END:

	subs	counterI, counterI, #1
	bgt	.Ldgemm_kernel_L2_M8_20

.Ldgemm_kernel_L2_M4_BEGIN:

	mov	counterI, origM
	tst	counterI , #7
	ble	.Ldgemm_kernel_L2_END

	tst	counterI, #4			// counterI = counterI / 2
	ble	.Ldgemm_kernel_L2_M2_BEGIN

.Ldgemm_kernel_L2_M4_20:

	INIT4x2

	mov	pB, origPB

	asr	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL,#0
	ble	.Ldgemm_kernel_L2_M4_40

	.align 5
.Ldgemm_kernel_L2_M4_22:
	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x2_SUB

	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M4_22


.Ldgemm_kernel_L2_M4_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L2_M4_100

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE_64]
.Ldgemm_kernel_L2_M4_42:

	KERNEL4x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M4_42

.Ldgemm_kernel_L2_M4_100:

	SAVE4x2

.Ldgemm_kernel_L2_M4_END:


.Ldgemm_kernel_L2_M2_BEGIN:

	mov	counterI, origM
	tst	counterI , #3
	ble	.Ldgemm_kernel_L2_END

	tst	counterI, #2			// counterI = counterI / 2
	ble	.Ldgemm_kernel_L2_M1_BEGIN

.Ldgemm_kernel_L2_M2_20:

	INIT2x2

	mov	pB, origPB

	asr	counterL , origK, #3		// counterL = counterL / 8
        cmp	counterL,#0
	ble	.Ldgemm_kernel_L2_M2_40

.Ldgemm_kernel_L2_M2_22:

	KERNEL2x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x2_SUB
	KERNEL2x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x2_SUB

	KERNEL2x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL2x2_SUB
	KERNEL2x2_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M2_22

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE_64]
.Ldgemm_kernel_L2_M2_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L2_M2_100

.Ldgemm_kernel_L2_M2_42:

	KERNEL2x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M2_42

.Ldgemm_kernel_L2_M2_100:

	SAVE2x2

.Ldgemm_kernel_L2_M2_END:


.Ldgemm_kernel_L2_M1_BEGIN:

	tst	counterI, #1			// counterI = counterI % 2
	ble	.Ldgemm_kernel_L2_END

.Ldgemm_kernel_L2_M1_20:

	INIT1x2

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
        cmp     counterL, #0
	ble	.Ldgemm_kernel_L2_M1_40

.Ldgemm_kernel_L2_M1_22:
	KERNEL1x2_SUB
	KERNEL1x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x2_SUB
	KERNEL1x2_SUB

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	KERNEL1x2_SUB
	KERNEL1x2_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x2_SUB
	KERNEL1x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M1_22

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE_64]
.Ldgemm_kernel_L2_M1_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L2_M1_100

.Ldgemm_kernel_L2_M1_42:

	KERNEL1x2_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L2_M1_42

.Ldgemm_kernel_L2_M1_100:

	SAVE1x2

.Ldgemm_kernel_L2_END:
	add	origPB, origPB, origK, lsl #4	// B = B + K * 2 * 8

/******************************************************************************/

.Ldgemm_kernel_L1_BEGIN:

	mov	counterJ , origN
	tst	counterJ , #1
	ble	.Ldgemm_kernel_L999 // done

	mov	pCRow0, pC			// pCRow0 = C
	add	pC , pC , LDC			// Update pC to point to next

	mov	pA, origPA			// pA = A

.Ldgemm_kernel_L1_M8_BEGIN:

	mov	counterI, origM
	asr 	counterI, counterI, #3		// counterI = counterI / 8
	cmp	counterI, #0
	ble	.Ldgemm_kernel_L1_M4_BEGIN

	.align 5
.Ldgemm_kernel_L1_M8_20:

	INIT8x1

	mov	pB, origPB
	asr	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L1_M8_40

	.align 5
.Ldgemm_kernel_L1_M8_22:
	KERNEL8x1_SUB
	KERNEL8x1_SUB
	KERNEL8x1_SUB
	KERNEL8x1_SUB

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	KERNEL8x1_SUB
	KERNEL8x1_SUB
	KERNEL8x1_SUB
	KERNEL8x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M8_22


.Ldgemm_kernel_L1_M8_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L1_M8_100

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
.Ldgemm_kernel_L1_M8_42:

	KERNEL8x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M8_42

.Ldgemm_kernel_L1_M8_100:

	SAVE8x1

.Ldgemm_kernel_L1_M8_END:

	subs	counterI, counterI, #1
	bgt	.Ldgemm_kernel_L1_M8_20

.Ldgemm_kernel_L1_M4_BEGIN:

	mov	counterI, origM
	tst	counterI , #7
	ble	.Ldgemm_kernel_L1_END

	tst	counterI, #4			// counterI = counterI / 2
	ble	.Ldgemm_kernel_L1_M2_BEGIN

.Ldgemm_kernel_L1_M4_20:

	INIT4x1

	mov	pB, origPB
	asr	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L1_M4_40

	.align 5
.Ldgemm_kernel_L1_M4_22:
	KERNEL4x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x1_SUB
	KERNEL4x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x1_SUB

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	KERNEL4x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x1_SUB
	KERNEL4x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL4x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M4_22


.Ldgemm_kernel_L1_M4_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L1_M4_100

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
.Ldgemm_kernel_L1_M4_42:

	KERNEL4x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M4_42

.Ldgemm_kernel_L1_M4_100:

	SAVE4x1

.Ldgemm_kernel_L1_M4_END:

.Ldgemm_kernel_L1_M2_BEGIN:

	mov	counterI, origM
	tst	counterI , #3
	ble	.Ldgemm_kernel_L1_END

	tst	counterI, #2			// counterI = counterI / 2
	ble	.Ldgemm_kernel_L1_M1_BEGIN

.Ldgemm_kernel_L1_M2_20:

	INIT2x1

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L1_M2_40

.Ldgemm_kernel_L1_M2_22:

	KERNEL2x1_SUB
	KERNEL2x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x1_SUB
	KERNEL2x1_SUB

	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]

	KERNEL2x1_SUB
	KERNEL2x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL2x1_SUB
	KERNEL2x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M2_22

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE_64]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
.Ldgemm_kernel_L1_M2_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L1_M2_100

.Ldgemm_kernel_L1_M2_42:

	KERNEL2x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M2_42

.Ldgemm_kernel_L1_M2_100:

	SAVE2x1

.Ldgemm_kernel_L1_M2_END:


.Ldgemm_kernel_L1_M1_BEGIN:

	tst	counterI, #1			// counterI = counterI % 2
	ble	.Ldgemm_kernel_L1_END

.Ldgemm_kernel_L1_M1_20:

	INIT1x1

	mov	pB, origPB

	asr 	counterL , origK, #3		// counterL = counterL / 8
	cmp	counterL , #0
	ble	.Ldgemm_kernel_L1_M1_40


.Ldgemm_kernel_L1_M1_22:
	KERNEL1x1_SUB
	KERNEL1x1_SUB
	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	KERNEL1x1_SUB
	KERNEL1x1_SUB

	KERNEL1x1_SUB
	KERNEL1x1_SUB
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
	KERNEL1x1_SUB
	KERNEL1x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M1_22


.Ldgemm_kernel_L1_M1_40:

	ands	counterL , origK, #7		// counterL = counterL % 8
	ble	.Ldgemm_kernel_L1_M1_100

	prfm	PLDL1KEEP, [pA, A_PRE_SIZE]
	prfm	PLDL1KEEP, [pB, B_PRE_SIZE]
.Ldgemm_kernel_L1_M1_42:

	KERNEL1x1_SUB

	subs	counterL, counterL, #1
	bgt	.Ldgemm_kernel_L1_M1_42

.Ldgemm_kernel_L1_M1_100:

	SAVE1x1


.Ldgemm_kernel_L1_END:


.Ldgemm_kernel_L999:
	mov	x0, #0				// set return value
	ldp	d8, d9, [sp, #(0 * 16)]
	ldp	d10, d11, [sp, #(1 * 16)]
	ldp	d12, d13, [sp, #(2 * 16)]
	ldp	d14, d15, [sp, #(3 * 16)]
	ldp	d16, d17, [sp, #(4 * 16)]
	ldp	x18, x19, [sp, #(5 * 16)]
	ldp	x20, x21, [sp, #(6 * 16)]
	ldp	x22, x23, [sp, #(7 * 16)]
	ldp	x24, x25, [sp, #(8 * 16)]
	ldp	x26, x27, [sp, #(9 * 16)]
	ldr	x28, [sp, #(10 * 16)]
	add	sp, sp, #(11*16)
	ret

	EPILOGUE